From fa61927c6b0b1e7053bb3508e044d9f2fe22a79b Mon Sep 17 00:00:00 2001 From: holger krekel Date: Tue, 24 Jul 2012 12:10:04 +0200 Subject: [PATCH] introduce @pytest.mark.setup decorated function, extend newexamples.txt and draft a V4 resources API doc. --- _pytest/__init__.py | 2 +- _pytest/main.py | 32 ++- _pytest/python.py | 28 ++- doc/en/example/newexamples.txt | 139 ++++++++++-- doc/en/resources.txt | 372 +++++++++++---------------------- setup.py | 2 +- testing/test_python.py | 123 ++++++++++- 7 files changed, 410 insertions(+), 288 deletions(-) diff --git a/_pytest/__init__.py b/_pytest/__init__.py index aa0e0f06c..435f64efb 100644 --- a/_pytest/__init__.py +++ b/_pytest/__init__.py @@ -1,2 +1,2 @@ # -__version__ = '2.3.0.dev3' +__version__ = '2.3.0.dev4' diff --git a/_pytest/main.py b/_pytest/main.py index fbd7fd2f3..4fdf208e9 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -8,6 +8,8 @@ import os, sys, imp from _pytest.monkeypatch import monkeypatch from py._code.code import TerminalRepr +from _pytest.mark import MarkInfo + tracebackcutdir = py.path.local(_pytest.__file__).dirpath() # exitcodes for the command line @@ -422,6 +424,7 @@ class FuncargManager: self.arg2facspec = {} session.config.pluginmanager.register(self, "funcmanage") self._holderobjseen = set() + self.setuplist = [] ### XXX this hook should be called for historic events like pytest_configure ### so that we don't have to do the below pytest_collection hook @@ -445,6 +448,9 @@ class FuncargManager: def pytest_generate_tests(self, metafunc): funcargnames = list(metafunc.funcargnames) + setuplist, allargnames = self.getsetuplist(metafunc.parentid) + #print "setuplist, allargnames", setuplist, allargnames + funcargnames.extend(allargnames) seen = set() while funcargnames: argname = funcargnames.pop(0) @@ -465,6 +471,8 @@ class FuncargManager: newfuncargnames.remove("request") funcargnames.extend(newfuncargnames) + + def _parsefactories(self, holderobj, nodeid): if holderobj in self._holderobjseen: return @@ -473,20 +481,36 @@ class FuncargManager: for name in dir(holderobj): #print "check", holderobj, name obj = getattr(holderobj, name) + if not callable(obj): + continue # funcarg factories either have a pytest_funcarg__ prefix # or are "funcarg" marked if hasattr(obj, "funcarg"): - if name.startswith(self._argprefix): - argname = name[len(self._argprefix):] - else: - argname = name + assert not name.startswith(self._argprefix) + argname = name elif name.startswith(self._argprefix): argname = name[len(self._argprefix):] else: + # no funcargs. check if we have a setup function. + setup = getattr(obj, "setup", None) + if setup is not None and isinstance(setup, MarkInfo): + self.setuplist.append((nodeid, obj)) continue faclist = self.arg2facspec.setdefault(argname, []) faclist.append((nodeid, obj)) + def getsetuplist(self, nodeid): + l = [] + allargnames = set() + for baseid, setup in self.setuplist: + #print "check", baseid, setup + if nodeid.startswith(baseid): + funcargnames = getfuncargnames(setup) + l.append((setup, funcargnames)) + allargnames.update(funcargnames) + return l, allargnames + + def getfactorylist(self, argname, nodeid, function, raising=True): try: factorydef = self.arg2facspec[argname] diff --git a/_pytest/python.py b/_pytest/python.py index 7821ff47b..74d156752 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -834,6 +834,8 @@ class Function(FunctionMixin, pytest.Item): def setup(self): super(Function, self).setup() fillfuncargs(self) + if hasattr(self, "_request"): + self._request._callsetup() def __eq__(self, other): try: @@ -990,6 +992,22 @@ class FuncargRequest: return val + def _callsetup(self): + setuplist, allnames = self.funcargmanager.getsetuplist( + self._pyfuncitem.nodeid) + for setupfunc, funcargnames in setuplist: + kwargs = {} + for name in funcargnames: + if name == "request": + kwargs[name] = self + else: + kwargs[name] = self.getfuncargvalue(name) + scope = readscope(setupfunc, "setup") + if scope is None: + setupfunc(**kwargs) + else: + self.cached_setup(lambda: setupfunc(**kwargs), scope=scope) + def getfuncargvalue(self, argname): """ Retrieve a function argument by name for this test function invocation. This allows one function argument factory @@ -1030,10 +1048,8 @@ class FuncargRequest: mp.setattr(self, 'param', param, raising=False) # implemenet funcarg marker scope - marker = getattr(funcargfactory, "funcarg", None) - scope = None - if marker is not None: - scope = marker.kwargs.get("scope") + scope = readscope(funcargfactory, "funcarg") + if scope is not None: __tracebackhide__ = True if scopemismatch(self.scope, scope): @@ -1106,3 +1122,7 @@ def slice_kwargs(names, kwargs): new_kwargs[name] = kwargs[name] return new_kwargs +def readscope(func, markattr): + marker = getattr(func, markattr, None) + if marker is not None: + return marker.kwargs.get("scope") diff --git a/doc/en/example/newexamples.txt b/doc/en/example/newexamples.txt index 8736cd3ea..b09d9a51e 100644 --- a/doc/en/example/newexamples.txt +++ b/doc/en/example/newexamples.txt @@ -48,7 +48,7 @@ If you run the tests:: ================================= FAILURES ================================= ________________________________ test_ehlo _________________________________ - smtp = + smtp = def test_ehlo(smtp): response = smtp.ehlo() @@ -60,7 +60,7 @@ If you run the tests:: test_module.py:5: AssertionError ________________________________ test_noop _________________________________ - smtp = + smtp = def test_noop(smtp): response = smtp.noop() @@ -69,7 +69,7 @@ If you run the tests:: E assert 0 test_module.py:10: AssertionError - 2 failed in 0.14 seconds + 2 failed in 0.27 seconds you will see the two ``assert 0`` failing and can see that the same (session-scoped) object was passed into the two test functions. @@ -100,7 +100,7 @@ another run:: ================================= FAILURES ================================= __________________________ test_ehlo[merlinux.eu] __________________________ - smtp = + smtp = def test_ehlo(smtp): response = smtp.ehlo() @@ -112,7 +112,7 @@ another run:: test_module.py:5: AssertionError ________________________ test_ehlo[mail.python.org] ________________________ - smtp = + smtp = def test_ehlo(smtp): response = smtp.ehlo() @@ -123,7 +123,7 @@ another run:: test_module.py:4: AssertionError __________________________ test_noop[merlinux.eu] __________________________ - smtp = + smtp = def test_noop(smtp): response = smtp.noop() @@ -134,7 +134,7 @@ another run:: test_module.py:10: AssertionError ________________________ test_noop[mail.python.org] ________________________ - smtp = + smtp = def test_noop(smtp): response = smtp.noop() @@ -143,9 +143,9 @@ another run:: E assert 0 test_module.py:10: AssertionError - 4 failed in 5.70 seconds - closing - closing + 4 failed in 6.91 seconds + closing + closing We get four failures because we are running the two tests twice with different ``smtp`` instantiations as defined on the factory. @@ -157,7 +157,7 @@ You can look at what tests pytest collects without running them:: $ py.test --collectonly =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev3 + platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev4 plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov collecting ... collected 4 items @@ -174,10 +174,113 @@ And you can run without output capturing and minimized failure reporting to chec collecting ... collected 4 items FFFF ================================= FAILURES ================================= - /home/hpk/tmp/doc-exec-330/test_module.py:5: assert 0 - /home/hpk/tmp/doc-exec-330/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN' - /home/hpk/tmp/doc-exec-330/test_module.py:10: assert 0 - /home/hpk/tmp/doc-exec-330/test_module.py:10: assert 0 - 4 failed in 6.02 seconds - closing - closing + /home/hpk/tmp/doc-exec-361/test_module.py:5: assert 0 + /home/hpk/tmp/doc-exec-361/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN' + /home/hpk/tmp/doc-exec-361/test_module.py:10: assert 0 + /home/hpk/tmp/doc-exec-361/test_module.py:10: assert 0 + 4 failed in 6.83 seconds + closing + closing + +.. _`new_setup`: + +``@pytest.mark.setup``: xUnit on steroids +-------------------------------------------------------------------- + +.. regendoc:wipe + +.. versionadded:: 2.3 + +The ``@pytest.mark.setup`` marker allows + +* to mark a function as a setup/fixture method; the function can itself + receive funcargs +* to set a scope which determines the level of caching and how often + the setup function is going to be called. + +Here is a simple example which configures a global funcarg without +the test needing to have it in its signature:: + + # content of conftest.py + import pytest + + @pytest.mark.funcarg(scope="module") + def resource(request, tmpdir): + def fin(): + print "finalize", tmpdir + request.addfinalizer(fin) + print "created resource", tmpdir + return tmpdir + +And the test file contains a setup function using this resource:: + + # content of test_module.py + import pytest + + @pytest.mark.setup(scope="function") + def setresource(resource): + global myresource + myresource = resource + + def test_1(): + assert myresource + print "using myresource", myresource + + def test_2(): + assert myresource + print "using myresource", myresource + +Let's run this module:: + + $ py.test -qs + collecting ... collected 2 items + .. + 2 passed in 0.24 seconds + created resource /home/hpk/tmp/pytest-3715/test_10 + using myresource /home/hpk/tmp/pytest-3715/test_10 + using myresource /home/hpk/tmp/pytest-3715/test_10 + finalize /home/hpk/tmp/pytest-3715/test_10 + +The two test functions will see the same resource instance because it has +a module life cycle or scope. + +The resource funcarg can later add parametrization without any test +or setup code needing to change:: + + # content of conftest.py + import pytest + + @pytest.mark.funcarg(scope="module", params=["aaa", "bbb"]) + def resource(request, tmpdir): + newtmp = tmpdir.join(request.param) + def fin(): + print "finalize", newtmp + request.addfinalizer(fin) + print "created resource", newtmp + return newtmp + +Running this will run four tests:: + + $ py.test -qs + collecting ... collected 4 items + .... + 4 passed in 0.24 seconds + created resource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa + using myresource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa + created resource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb + using myresource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb + using myresource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa + using myresource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb + finalize /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb + finalize /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa + +Each parameter causes the creation of a respective resource and the +unchanged test module uses it in its ``@setup`` decorated method. + +.. note:: + + Currently, parametrized tests are sorted by test function location + so a test function will execute multiple times with different parametrized + funcargs. If you have class/module/session scoped funcargs and + they cause global side effects this can cause problems because the + code under test may not be prepared to deal with it. diff --git a/doc/en/resources.txt b/doc/en/resources.txt index a9f043e90..521308ba4 100644 --- a/doc/en/resources.txt +++ b/doc/en/resources.txt @@ -1,43 +1,55 @@ -V3: Creating and working with parametrized test resources +V4: Creating and working with parametrized resources =============================================================== **Target audience**: Reading this document requires basic knowledge of python testing, xUnit setup methods and the basic pytest funcarg mechanism, see http://pytest.org/latest/funcargs.html -**Abstract**: pytest-2.X provides more powerful and more flexible funcarg -and setup machinery. It does so by introducing a new @funcarg and a -new @setup marker which allows to define scoping and parametrization -parameters. If using ``@funcarg``, following the ``pytest_funcarg__`` -naming pattern becomes optional. Functions decorated with ``@setup`` -are called independenlty from the definition of funcargs but can -access funcarg values if needed. This allows for ultimate flexibility -in designing your test fixtures and their parametrization. Also, -you can now use ``py.test --collectonly`` to inspect your fixture -setup. Nonwithstanding these extensions, pre-existing test suites -and plugins written to work for previous pytest versions shall run unmodified. +**Abstract**: pytest-2.X provides yet more powerful and flexible +fixture machinery by introducing: + +* a new ``@pytest.mark.funcarg`` marker to define funcarg factories and their + scoping and parametrization. No special ``pytest_funcarg__`` naming there. + +* a new ``@pytest.mark.setup`` marker to define setup functions and their + scoping. + +* directly use funcargs through funcarg factory signatures + +Both funcarg factories and setup functions can be defined in test modules, +classes, conftest.py files and installed plugins. + +The introduction of these two markers lifts several prior limitations +and allows to easily define and implement complex testing scenarios. + +Nonwithstanding these extensions, already existing test suites and plugins +written to work for previous pytest versions shall run unmodified. -**Changes**: This V3 draft is based on incorporating and thinking about -feedback provided by Floris Bruynooghe, Carl Meyer and Samuele Pedroni. -It remains as draft documentation, pending further refinements and -changes according to implementation or backward compatibility issues. -The main changes to V2 are: +**Changes**: This V4 draft is based on incorporating and thinking about +feedback on previous versions provided by Floris Bruynooghe, Carl Meyer, +Ronny Pfannschmidt and Samuele Pedroni. It remains as draft +documentation, pending further refinements and changes according to +implementation or backward compatibility issues. The main changes are: -* Collapse funcarg factory decorator into a single "@funcarg" one. - You can specify scopes and params with it. Moreover, if you supply - a "name" you do not need to follow the "pytest_funcarg__NAME" naming - pattern. Keeping with "funcarg" naming arguable now makes more - sense since the main interface using these resources are test and - setup functions. Keeping it probably causes the least semantic friction. +* Collapse funcarg factory decorators into a single "@funcarg" one. + You can specify scopes and params with it. When using the decorator + the "pytest_funcarg__" prefix becomes optional. -* Drop setup_directory/setup_session and introduce a new @setup - decorator similar to the @funcarg one but accepting funcargs. +* funcarg factories can now use funcargs themselves -* cosnider the extended setup_X funcargs for dropping because - the new @setup decorator probably is more flexible and introduces - less implementation complexity. +* Drop setup/directory scope from this draft + +* introduce a new @setup decorator similar to the @funcarg one + except that setup-markers cannot define parametriation themselves. + Instead they can easily depend on a parametrized funcarg (which + must not be visible at test function signatures). + +* drop consideration of setup_X support for funcargs because + it is less flexible and probably causes more implementation + troubles than the current @setup approach which can share + a lot of logic with the @funcarg one. .. currentmodule:: _pytest @@ -78,17 +90,13 @@ There are some problems with this approach: ``extrakey`` parameter containing ``request.param`` to the :py:func:`~python.Request.cached_setup` call. -3. the current implementation is inefficient: it performs factory discovery - each time a "db" argument is required. This discovery wrongly happens at - setup-time. +3. there is no way how you can make use of funcarg factories + in xUnit setup methods. -4. there is no way how you can use funcarg factories, let alone - parametrization, when your tests use the xUnit setup_X approach. +4. A non-parametrized funcarg factory cannot use a parametrized + funcarg resource if it isn't stated in the test function signature. -5. there is no way to specify a per-directory scope for caching. - -In the following sections, API extensions are presented to solve -each of these problems. +The following sections address the advances which solve all of these problems. Direct scoping of funcarg factories @@ -158,7 +166,7 @@ factory function. Direct usage of funcargs with funcargs factories ---------------------------------------------------------- -.. note:: Not Implemented - unclear if to. +.. note:: Implemented. You can now directly use funcargs in funcarg factories. Example:: @@ -168,33 +176,39 @@ You can now directly use funcargs in funcarg factories. Example:: Apart from convenience it also solves an issue when your factory depends on a parametrized funcarg. Previously, a call to -``request.getfuncargvalue()`` would not allow pytest to know -at collection time about the fact that a required resource is -actually parametrized. +``request.getfuncargvalue()`` happens at test execution time and +thus pytest would not know at collection time about the fact that +a required resource is parametrized. + +No ``pytest_funcarg__`` prefix when using @funcarg decorator +------------------------------------------------------------------- -The "pytest_funcarg__" prefix becomes optional ------------------------------------------------------ .. note:: Implemented -When using the ``@funcarg`` decorator you do not need to use -the ``pytest_funcarg__`` prefix any more:: +When using the ``@funcarg`` decorator the name of the function +does not need to (and in fact cannot) use the ``pytest_funcarg__`` +naming:: @pytest.mark.funcarg def db(request): ... The name under which the funcarg resource can be requested is ``db``. -Any ``pytest_funcarg__`` prefix will be stripped. Note that a an -unqualified funcarg-marker implies a scope of "function" meaning -that the funcarg factory will be called for each test function invocation. + +You can also use the "old" non-decorator way of specifying funcarg factories +aka:: + + def pytest_funcarg__db(request): + ... + +It is recommended to use the funcarg-decorator, however. +solving per-session setup / the new @setup marker +-------------------------------------------------------------- -support for a new @setup marker ------------------------------------------------------- - -.. note:: Not-Implemented, still under consideration if to. +.. note:: Implemented, at least working for basic situations. pytest for a long time offered a pytest_configure and a pytest_sessionstart hook which are often used to setup global resources. This suffers from @@ -212,9 +226,7 @@ several problems: fact that this hook is actually used for reporting, in particular the test-header with platform/custom information. -4. there is no direct way how you can restrict setup to a directory scope. - -Moreover, it is today not easy to define scoped setup from plugins or +Moreover, it is today not easy to define a scoped setup from plugins or conftest files other than to implement a ``pytest_runtest_setup()`` hook and caring for scoping/caching yourself. And it's virtually impossible to do this with parametrization as ``pytest_runtest_setup()`` is called @@ -222,222 +234,76 @@ during test execution and parametrization happens at collection time. It follows that pytest_configure/session/runtest_setup are often not appropriate for implementing common fixture needs. Therefore, -pytest-2.X introduces a new "@pytest.mark.setup" marker, accepting -the same parameters as the @funcargs decorator. The difference is -that the decorated function can accept function arguments itself -Example:: - - # content of conftest.py - import pytest - @pytest.mark.setup(scope="session") - def mysetup(db): - ... +pytest-2.X introduces a new "@pytest.mark.setup" marker which takes +an optional "scope" parameter. -This ``mysetup`` function is going to be executed when the first -test in the directory tree executes. It is going to be executed once -per-session and it receives the ``db`` funcarg which must be of same -of higher scope; you e. g. generally cannot use a per-module or per-function -scoped resource in a session-scoped setup function. - -You can also use ``@setup`` inside a test module or class:: - - # content of test_module.py - import pytest - - @pytest.mark.setup(scope="module", params=[1,2,3]) - def modes(tmpdir, request): - # ... - -This would execute the ``modes`` function once for each parameter -which will be put at ``request.param``. This request object offers -the ``addfinalizer(func)`` helper which allows to register a function -which will be executed when test functions within the specified scope -finished execution. - -.. note:: - - For each scope, the funcargs will be setup and then the setup functions - will be called. This allows @setup-decorated functions to depend - on already setup funcarg values by accessing ``request.funcargs``. - -Using funcarg resources in xUnit setup methods ------------------------------------------------------------- - -.. note:: Not implemented. Not clear if to. - -XXX Consider this feature in contrast to the @setup feature - probably -introducing one of them is better and the @setup decorator is more flexible. - -For a long time, pytest has recommended the usage of funcarg -factories as a primary means for managing resources in your test run. -It is a better approach than the jUnit-based approach in many cases, even -more with the new pytest-2.X features, because the funcarg resource factory -provides a single place to determine scoping and parametrization. Your tests -do not need to encode setup/teardown details in every test file's -setup_module/class/method. - -However, the jUnit methods originally introduced by pytest to Python, -remain popoular with nose and unittest-based test suites. Without question, -there are large existing test suites using this paradigm. pytest-2.X -recognizes this fact and now offers direct integration with funcarg resources. Here is a basic example for getting a per-module tmpdir:: - - def setup_module(mod, tmpdir): - mod.tmpdir = tmpdir - -This will trigger pytest's funcarg mechanism to create a value of -"tmpdir" which can then be used throughout the module as a global. - -The new extension to setup_X methods also works in case a resource is -parametrized. For example, let's consider an setup_class example using -our "db" resource:: - - class TestClass: - def setup_class(cls, db): - cls.db = db - # perform some extra things on db - # so that test methods can work with it - -With pytest-2.X the setup* methods will be discovered at collection-time, -allowing to seemlessly integrate this approach with parametrization, -allowing the factory specification to determine all details. The -setup_class itself does not itself need to be aware of the fact that -"db" might be a mysql/PG database. -Note that if the specified resource is provided only as a per-testfunction -resource, collection would early on report a ScopingMismatch error. - - -the "directory" caching scope --------------------------------------------- - -.. note:: Not implemented. - -All API accepting a scope (:py:func:`cached_setup()` and -the new funcarg/setup decorators) now also accept a "directory" -specification. This allows to restrict/cache resource values on a -per-directory level. +See :ref:`new_setup` for examples. funcarg and setup discovery now happens at collection time --------------------------------------------------------------------- -.. note:: Partially implemented - collectonly shows no extra information +.. note:: + Partially implemented - collectonly shows no extra information however. -pytest-2.X takes care to discover funcarg factories and setup_X methods +pytest-2.X takes care to discover funcarg factories and @setup methods at collection time. This is more efficient especially for large test suites. Moreover, a call to "py.test --collectonly" should be able to show a lot of setup-information and thus presents a nice method to get an overview of resource management in your project. -Implementation level -=================================================================== -To implement the above new features, pytest-2.X grows some new hooks and -methods. At the time of writing V2 and without actually implementing -it, it is not clear how much of this new internal API will also be -exposed and advertised e. g. for plugin writers. +Sorting tests by funcarg scopes +------------------------------------------- -The main effort, however, will lie in revising what is done at -collection and what at test setup time. All funcarg factories and -xUnit setup methods need to be discovered at collection time -for the above mechanism to work. Additionally all test function -signatures need to be parsed in order to know which resources are -used. On the plus side, all previously collected fixtures and -test functions only need to be called, no discovery is neccessary -is required anymore. +.. note:: Not implemented, Under consideration. -the "request" object incorporates scope-specific behaviour ------------------------------------------------------------------- +pytest by default sorts test items by their source location. +For class/module/session scoped funcargs it is not always +desirable to have multiple active funcargs. Sometimes, +the application under test may not even be able to handle it +because it relies on global state/side effects related to those +resources. -funcarg factories receive a request object to help with implementing -finalization and inspection of the requesting-context. If there is -no scoping is in effect, nothing much will change of the API behaviour. -However, with scoping the request object represents the according context. -Let's consider this example:: +Therefore, pytest-2.3 tries to minimize the number of active +resources and re-orders test items accordingly. Consider the following +example:: - @pytest.mark.factory_scope("class") - def pytest_funcarg__db(request): - # ... - request.getfuncargvalue(...) - # - request.addfinalizer(db) - -Due to the class-scope, the request object will: - -- provide a ``None`` value for the ``request.function`` attribute. -- default to per-class finalization with the addfinalizer() call. -- raise a ScopeMismatchError if a more broadly scoped factory - wants to use a more tighly scoped factory (e.g. per-function) - -In fact, the request object is likely going to provide a "node" -attribute, denoting the current collection node on which it internally -operates. (Prior to pytest-2.3 there already was an internal -_pyfuncitem). - -As these are rather intuitive extensions, not much friction is expected -for test/plugin writers using the new scoping and parametrization mechanism. -It's, however, a serious internal effort to reorganize the pytest -implementation. - - -node.register_factory/getresource() methods --------------------------------------------------------- - -In order to implement factory- and setup-method discovery at -collection time, a new node API will be introduced to allow -for factory registration and a getresource() call to obtain -created values. The exact details of this API remain subject -to experimentation. The basic idea is to introduce two new -methods to the Session class which is already available on all nodes -through the ``node.session`` attribute:: - - class Session: - def register_resource_factory(self, name, factory_or_list, scope): - """ register a resource factory for the given name. - - :param name: Name of the resource. - :factory_or_list: a function or a list of functions creating - one or multiple resource values. - :param scope: a node instance. The factory will be only visisble - available for all descendant nodes. - specify the "session" instance for global availability - """ - - def getresource(self, name, node): - """ get a named resource for the give node. - - This method looks up a matching funcarg resource factory - and calls it. - """ - -.. todo:: - - XXX While this new API (or some variant of it) may suffices to implement - all of the described new usage-level features, it remains unclear how the - existing "@parametrize" or "metafunc.parametrize()" calls will map to it. - These parametrize-approaches tie resource parametrization to the - function/funcargs-usage rather than to the factories. - - - -ISSUES --------------------------- - -decorating a parametrized funcarg factory:: - - @pytest.mark.funcarg(scope="session", params=["mysql", "pg"]) - def db(request): + @pytest.mark.funcarg(scope="module", params=[1,2]) + def arg(request): + ... + @pytest.mark.funcarg(scope="function", params=[1,2]) + def otherarg(request): ... - class TestClass: - @pytest.mark.funcarg(scope="function") - def something(self, request): - session_db = request.getfuncargvalue("db") - ... -Here the function-scoped "something" factory uses the session-scoped -"db" factory to perform some additional steps. The dependency, however, -is only visible at setup-time, when the factory actually gets called. + def test_0(otherarg): + pass + def test_1(arg): + pass + def test_2(arg, otherarg): + pass -In order to allow parametrization at collection-time I see two ways: +if arg.1, arg.2, otherarg.1, otherarg.2 denote the respective +parametrized funcarg instances this will re-order test +execution like follows:: + + test_0(otherarg.1) + test_0(otherarg.2) + test_1(arg.1) + test_2(arg.1, otherarg.1) + test_2(arg.1, otherarg.2) + test_1(arg.2) + test_2(arg.2, otherarg.1) + test_2(arg.2, otherarg.2) + +Moreover, test_2(arg.1) will execute any registered teardowns for +the arg.1 resource after the test finished execution. + +.. note:: + + XXX it's quite unclear at the moment how to implement. + If we have a 1000 tests requiring different sets of parametrized + resources with different scopes, how to re-order accordingly? + It even seems difficult to express the expectation in a + concise manner. -- allow specifying dependencies in the funcarg-marker -- allow funcargs for factories as well diff --git a/setup.py b/setup.py index 164f80351..ec03a36d6 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def main(): name='pytest', description='py.test: simple powerful testing with Python', long_description = long_description, - version='2.3.0.dev3', + version='2.3.0.dev4', url='http://pytest.org', license='MIT license', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], diff --git a/testing/test_python.py b/testing/test_python.py index 2e681630f..5ad6bf37e 100644 --- a/testing/test_python.py +++ b/testing/test_python.py @@ -1746,12 +1746,121 @@ class TestFuncargManager: reprec = testdir.inline_run("-s") reprec.assertoutcome(passed=1) +class TestSetupDiscovery: + def pytest_funcarg__testdir(self, request): + testdir = request.getfuncargvalue("testdir") + testdir.makeconftest(""" + import pytest + @pytest.mark.setup + def perfunction(request): + pass + @pytest.mark.setup + def perfunction2(request): + pass + + def pytest_funcarg__fm(request): + return request.funcargmanager + + def pytest_funcarg__item(request): + return request._pyfuncitem + """) + return testdir + + def test_parsefactories_conftest(self, testdir): + testdir.makepyfile(""" + def test_check_setup(item, fm): + setuplist, allnames = fm.getsetuplist(item.nodeid) + assert len(setuplist) == 2 + assert setuplist[0][0].__name__ == "perfunction" + assert "request" in setuplist[0][1] + assert setuplist[1][0].__name__ == "perfunction2" + assert "request" in setuplist[1][1] + """) + reprec = testdir.inline_run("-s") + reprec.assertoutcome(passed=1) + + +class TestSetupManagement: + def test_funcarg_and_setup(self, testdir): + testdir.makepyfile(""" + import pytest + l = [] + @pytest.mark.funcarg(scope="module") + def arg(request): + l.append(1) + return 0 + @pytest.mark.setup(scope="class") + def something(request, arg): + l.append(2) + + def test_hello(arg): + assert len(l) == 2 + assert l == [1,2] + assert arg == 0 + + def test_hello2(arg): + assert len(l) == 2 + assert l == [1,2] + assert arg == 0 + """) + reprec = testdir.inline_run() + reprec.assertoutcome(passed=2) + + def test_setup_uses_parametrized_resource(self, testdir): + testdir.makepyfile(""" + import pytest + l = [] + @pytest.mark.funcarg(params=[1,2]) + def arg(request): + return request.param + + @pytest.mark.setup + def something(request, arg): + l.append(arg) + + def test_hello(): + if len(l) == 1: + assert l == [1] + elif len(l) == 2: + assert l == [1, 2] + else: + 0/0 + + """) + reprec = testdir.inline_run("-s") + reprec.assertoutcome(passed=2) + + def test_session_parametrized_function_setup(self, testdir): + testdir.makepyfile(""" + import pytest + + l = [] + + @pytest.mark.funcarg(scope="session", params=[1,2]) + def arg(request): + return request.param + + @pytest.mark.setup(scope="function") + def append(request, arg): + if request.function.__name__ == "test_some": + l.append(arg) + + def test_some(): + pass + + def test_result(arg): + assert len(l) == 2 + assert l == [1,2] + """) + reprec = testdir.inline_run("-s") + reprec.assertoutcome(passed=4) + class TestFuncargMarker: def test_parametrize(self, testdir): testdir.makepyfile(""" import pytest @pytest.mark.funcarg(params=["a", "b", "c"]) - def pytest_funcarg__arg(request): + def arg(request): return request.param l = [] def test_param(arg): @@ -1767,7 +1876,7 @@ class TestFuncargMarker: import pytest l = [] @pytest.mark.funcarg(scope="module") - def pytest_funcarg__arg(request): + def arg(request): l.append(1) return 1 @@ -1789,7 +1898,7 @@ class TestFuncargMarker: import pytest l = [] @pytest.mark.funcarg(scope="module") - def pytest_funcarg__arg(request): + def arg(request): l.append(1) return 1 @@ -1812,7 +1921,7 @@ class TestFuncargMarker: finalized = [] created = [] @pytest.mark.funcarg(scope="module") - def pytest_funcarg__arg(request): + def arg(request): created.append(1) assert request.scope == "module" request.addfinalizer(lambda: finalized.append(1)) @@ -1851,14 +1960,14 @@ class TestFuncargMarker: finalized = [] created = [] @pytest.mark.funcarg(scope="function") - def pytest_funcarg__arg(request): + def arg(request): pass """) testdir.makepyfile( test_mod1=""" import pytest @pytest.mark.funcarg(scope="session") - def pytest_funcarg__arg(request): + def arg(request): %s def test_1(arg): pass @@ -1894,7 +2003,7 @@ class TestFuncargMarker: testdir.makepyfile(""" import pytest @pytest.mark.funcarg(scope="module", params=["a", "b", "c"]) - def pytest_funcarg__arg(request): + def arg(request): return request.param l = [] def test_param(arg):