introduce @pytest.mark.setup decorated function,

extend newexamples.txt and draft a V4 resources API doc.

_pytest/__init__.py

@@ -1,2 +1,2 @@
 #
-__version__ = '2.3.0.dev3'
+__version__ = '2.3.0.dev4'

_pytest/main.py

@@ -8,6 +8,8 @@ import os, sys, imp
 from _pytest.monkeypatch import monkeypatch
 from py._code.code import TerminalRepr
 
+from _pytest.mark import MarkInfo
+
 tracebackcutdir = py.path.local(_pytest.__file__).dirpath()
 
 # exitcodes for the command line
@@ -422,6 +424,7 @@ class FuncargManager:
         self.arg2facspec = {}
         session.config.pluginmanager.register(self, "funcmanage")
         self._holderobjseen = set()
+        self.setuplist = []
 
     ### XXX this hook should be called for historic events like pytest_configure
     ### so that we don't have to do the below pytest_collection hook
@@ -445,6 +448,9 @@ class FuncargManager:
 
     def pytest_generate_tests(self, metafunc):
         funcargnames = list(metafunc.funcargnames)
+        setuplist, allargnames = self.getsetuplist(metafunc.parentid)
+        #print "setuplist, allargnames", setuplist, allargnames
+        funcargnames.extend(allargnames)
         seen = set()
         while funcargnames:
             argname = funcargnames.pop(0)
@@ -465,6 +471,8 @@ class FuncargManager:
                 newfuncargnames.remove("request")
                 funcargnames.extend(newfuncargnames)
 
+
+
     def _parsefactories(self, holderobj, nodeid):
         if holderobj in self._holderobjseen:
             return
@@ -473,20 +481,36 @@ class FuncargManager:
         for name in dir(holderobj):
             #print "check", holderobj, name
             obj = getattr(holderobj, name)
+            if not callable(obj):
+                continue
             # funcarg factories either have a pytest_funcarg__ prefix
             # or are "funcarg" marked
             if hasattr(obj, "funcarg"):
-                if name.startswith(self._argprefix):
+                assert not name.startswith(self._argprefix)
-                    argname = name[len(self._argprefix):]
-                else:
                 argname = name
             elif name.startswith(self._argprefix):
                 argname = name[len(self._argprefix):]
             else:
+                # no funcargs. check if we have a setup function.
+                setup = getattr(obj, "setup", None)
+                if setup is not None and isinstance(setup, MarkInfo):
+                    self.setuplist.append((nodeid, obj))
                 continue
             faclist = self.arg2facspec.setdefault(argname, [])
             faclist.append((nodeid, obj))
 
+    def getsetuplist(self, nodeid):
+        l = []
+        allargnames = set()
+        for baseid, setup in self.setuplist:
+            #print "check", baseid, setup
+            if nodeid.startswith(baseid):
+                funcargnames = getfuncargnames(setup)
+                l.append((setup, funcargnames))
+                allargnames.update(funcargnames)
+        return l, allargnames
+
+
     def getfactorylist(self, argname, nodeid, function, raising=True):
         try:
             factorydef = self.arg2facspec[argname]

_pytest/python.py

@@ -834,6 +834,8 @@ class Function(FunctionMixin, pytest.Item):
     def setup(self):
         super(Function, self).setup()
         fillfuncargs(self)
+        if hasattr(self, "_request"):
+            self._request._callsetup()
 
     def __eq__(self, other):
         try:
@@ -990,6 +992,22 @@ class FuncargRequest:
         return val
 
 
+    def _callsetup(self):
+        setuplist, allnames = self.funcargmanager.getsetuplist(
+                                                    self._pyfuncitem.nodeid)
+        for setupfunc, funcargnames in setuplist:
+            kwargs = {}
+            for name in funcargnames:
+                if name == "request":
+                    kwargs[name] = self
+                else:
+                    kwargs[name] = self.getfuncargvalue(name)
+            scope = readscope(setupfunc, "setup")
+            if scope is None:
+                setupfunc(**kwargs)
+            else:
+                self.cached_setup(lambda: setupfunc(**kwargs), scope=scope)
+
     def getfuncargvalue(self, argname):
         """ Retrieve a function argument by name for this test
         function invocation.  This allows one function argument factory
@@ -1030,10 +1048,8 @@ class FuncargRequest:
             mp.setattr(self, 'param', param, raising=False)
 
         # implemenet funcarg marker scope
-        marker = getattr(funcargfactory, "funcarg", None)
+        scope = readscope(funcargfactory, "funcarg")
-        scope = None
+
-        if marker is not None:
-            scope = marker.kwargs.get("scope")
         if scope is not None:
             __tracebackhide__ = True
             if scopemismatch(self.scope, scope):
@@ -1106,3 +1122,7 @@ def slice_kwargs(names, kwargs):
         new_kwargs[name] = kwargs[name]
     return new_kwargs
 
+def readscope(func, markattr):
+    marker = getattr(func, markattr, None)
+    if marker is not None:
+        return marker.kwargs.get("scope")

doc/en/example/newexamples.txt

@@ -48,7 +48,7 @@ If you run the tests::
     ================================= FAILURES =================================
     ________________________________ test_ehlo _________________________________
     
-    smtp = <smtplib.SMTP instance at 0x28599e0>
+    smtp = <smtplib.SMTP instance at 0x20ba7e8>
     
         def test_ehlo(smtp):
             response = smtp.ehlo()
@@ -60,7 +60,7 @@ If you run the tests::
     test_module.py:5: AssertionError
     ________________________________ test_noop _________________________________
     
-    smtp = <smtplib.SMTP instance at 0x28599e0>
+    smtp = <smtplib.SMTP instance at 0x20ba7e8>
     
         def test_noop(smtp):
             response = smtp.noop()
@@ -69,7 +69,7 @@ If you run the tests::
     E       assert 0
     
     test_module.py:10: AssertionError
-    2 failed in 0.14 seconds
+    2 failed in 0.27 seconds
 
 you will see the two ``assert 0`` failing and can see that
 the same (session-scoped) object was passed into the two test functions.
@@ -100,7 +100,7 @@ another run::
     ================================= FAILURES =================================
     __________________________ test_ehlo[merlinux.eu] __________________________
     
-    smtp = <smtplib.SMTP instance at 0x2bf3d40>
+    smtp = <smtplib.SMTP instance at 0x2a51830>
     
         def test_ehlo(smtp):
             response = smtp.ehlo()
@@ -112,7 +112,7 @@ another run::
     test_module.py:5: AssertionError
     ________________________ test_ehlo[mail.python.org] ________________________
     
-    smtp = <smtplib.SMTP instance at 0x2bf9170>
+    smtp = <smtplib.SMTP instance at 0x2a56c20>
     
         def test_ehlo(smtp):
             response = smtp.ehlo()
@@ -123,7 +123,7 @@ another run::
     test_module.py:4: AssertionError
     __________________________ test_noop[merlinux.eu] __________________________
     
-    smtp = <smtplib.SMTP instance at 0x2bf3d40>
+    smtp = <smtplib.SMTP instance at 0x2a51830>
     
         def test_noop(smtp):
             response = smtp.noop()
@@ -134,7 +134,7 @@ another run::
     test_module.py:10: AssertionError
     ________________________ test_noop[mail.python.org] ________________________
     
-    smtp = <smtplib.SMTP instance at 0x2bf9170>
+    smtp = <smtplib.SMTP instance at 0x2a56c20>
     
         def test_noop(smtp):
             response = smtp.noop()
@@ -143,9 +143,9 @@ another run::
     E       assert 0
     
     test_module.py:10: AssertionError
-    4 failed in 5.70 seconds
+    4 failed in 6.91 seconds
-    closing <smtplib.SMTP instance at 0x2bf9170>
+    closing <smtplib.SMTP instance at 0x2a56c20>
-    closing <smtplib.SMTP instance at 0x2bf3d40>
+    closing <smtplib.SMTP instance at 0x2a51830>
 
 We get four failures because we are running the two tests twice with
 different ``smtp`` instantiations as defined on the factory.
@@ -157,7 +157,7 @@ You can look at what tests pytest collects without running them::
 
     $ py.test --collectonly
     =========================== test session starts ============================
-    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev3
+    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev4
     plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
     collecting ... collected 4 items
     <Module 'test_module.py'>
@@ -174,10 +174,113 @@ And you can run without output capturing and minimized failure reporting to chec
     collecting ... collected 4 items
     FFFF
     ================================= FAILURES =================================
-    /home/hpk/tmp/doc-exec-330/test_module.py:5: assert 0
+    /home/hpk/tmp/doc-exec-361/test_module.py:5: assert 0
-    /home/hpk/tmp/doc-exec-330/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
+    /home/hpk/tmp/doc-exec-361/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
-    /home/hpk/tmp/doc-exec-330/test_module.py:10: assert 0
+    /home/hpk/tmp/doc-exec-361/test_module.py:10: assert 0
-    /home/hpk/tmp/doc-exec-330/test_module.py:10: assert 0
+    /home/hpk/tmp/doc-exec-361/test_module.py:10: assert 0
-    4 failed in 6.02 seconds
+    4 failed in 6.83 seconds
-    closing <smtplib.SMTP instance at 0x1f5ef38>
+    closing <smtplib.SMTP instance at 0x236da28>
-    closing <smtplib.SMTP instance at 0x1f5acf8>
+    closing <smtplib.SMTP instance at 0x23687e8>
+
+.. _`new_setup`:
+
+``@pytest.mark.setup``: xUnit on steroids
+--------------------------------------------------------------------
+
+.. regendoc:wipe
+
+.. versionadded:: 2.3
+
+The ``@pytest.mark.setup`` marker allows 
+
+* to mark a function as a setup/fixture method; the function can itself
+  receive funcargs
+* to set a scope which determines the level of caching and how often
+  the setup function is going to be called.
+
+Here is a simple example which configures a global funcarg without
+the test needing to have it in its signature::
+
+    # content of conftest.py
+    import pytest
+
+    @pytest.mark.funcarg(scope="module")
+    def resource(request, tmpdir):
+        def fin():
+            print "finalize", tmpdir
+        request.addfinalizer(fin)
+        print "created resource", tmpdir
+        return tmpdir
+
+And the test file contains a setup function using this resource::
+
+    # content of test_module.py
+    import pytest
+
+    @pytest.mark.setup(scope="function")
+    def setresource(resource):
+        global myresource
+        myresource = resource
+
+    def test_1():
+        assert myresource
+        print "using myresource", myresource
+
+    def test_2():
+        assert myresource
+        print "using myresource", myresource
+
+Let's run this module::
+
+    $ py.test -qs
+    collecting ... collected 2 items
+    ..
+    2 passed in 0.24 seconds
+    created resource /home/hpk/tmp/pytest-3715/test_10
+    using myresource /home/hpk/tmp/pytest-3715/test_10
+    using myresource /home/hpk/tmp/pytest-3715/test_10
+    finalize /home/hpk/tmp/pytest-3715/test_10
+
+The two test functions will see the same resource instance because it has
+a module life cycle or scope.  
+
+The resource funcarg can later add parametrization without any test
+or setup code needing to change::
+
+    # content of conftest.py
+    import pytest
+
+    @pytest.mark.funcarg(scope="module", params=["aaa", "bbb"])
+    def resource(request, tmpdir):
+        newtmp = tmpdir.join(request.param)
+        def fin():
+            print "finalize", newtmp
+        request.addfinalizer(fin)
+        print "created resource", newtmp
+        return newtmp
+
+Running this will run four tests::
+
+    $ py.test -qs
+    collecting ... collected 4 items
+    ....
+    4 passed in 0.24 seconds
+    created resource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa
+    using myresource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa
+    created resource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb
+    using myresource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb
+    using myresource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa
+    using myresource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb
+    finalize /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb
+    finalize /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa
+
+Each parameter causes the creation of a respective resource and the
+unchanged test module uses it in its ``@setup`` decorated method.
+
+.. note::
+
+   Currently, parametrized tests are sorted by test function location
+   so a test function will execute multiple times with different parametrized
+   funcargs.  If you have class/module/session scoped funcargs and
+   they cause global side effects this can cause problems because the
+   code under test may not be prepared to deal with it.

doc/en/resources.txt

@@ -1,43 +1,55 @@
 
-V3: Creating and working with parametrized test resources
+V4: Creating and working with parametrized resources
 ===============================================================
 
 **Target audience**: Reading this document requires basic knowledge of 
 python testing, xUnit setup methods and the basic pytest funcarg mechanism,
 see http://pytest.org/latest/funcargs.html 
 
-**Abstract**: pytest-2.X provides more powerful and more flexible funcarg
+**Abstract**: pytest-2.X provides yet more powerful and flexible 
-and setup machinery.  It does so by introducing a new @funcarg and a
+fixture machinery by introducing:
-new @setup marker which allows to define scoping and parametrization
+
-parameters.  If using ``@funcarg``, following the ``pytest_funcarg__`` 
+* a new ``@pytest.mark.funcarg`` marker to define funcarg factories and their
-naming pattern becomes optional.  Functions decorated with ``@setup`` 
+  scoping and parametrization.  No special ``pytest_funcarg__`` naming there.
-are called independenlty from the definition of funcargs but can 
+
-access funcarg values if needed.  This allows for ultimate flexibility
+* a new ``@pytest.mark.setup`` marker to define setup functions and their 
-in designing your test fixtures and their parametrization. Also,
+  scoping.
-you can now use ``py.test --collectonly`` to inspect your fixture 
+
-setup.  Nonwithstanding these extensions, pre-existing test suites 
+* directly use funcargs through funcarg factory signatures
-and plugins written to work for previous pytest versions shall run unmodified.
+
+Both funcarg factories and setup functions can be defined in test modules,
+classes, conftest.py files and installed plugins.
+
+The introduction of these two markers lifts several prior limitations
+and allows to easily define and implement complex testing scenarios.
+
+Nonwithstanding these extensions, already existing test suites and plugins
+written to work for previous pytest versions shall run unmodified.
 
 
-**Changes**: This V3 draft is based on incorporating and thinking about
+**Changes**: This V4 draft is based on incorporating and thinking about
-feedback provided by Floris Bruynooghe, Carl Meyer and Samuele Pedroni.
+feedback on previous versions provided by Floris Bruynooghe, Carl Meyer,
-It remains as draft documentation, pending further refinements and
+Ronny Pfannschmidt and Samuele Pedroni.  It remains as draft
-changes according to implementation or backward compatibility issues.
+documentation, pending further refinements and changes according to
-The main changes to V2 are:
+implementation or backward compatibility issues.  The main changes are:
 
-* Collapse funcarg factory decorator into a single "@funcarg" one.
+* Collapse funcarg factory decorators into a single "@funcarg" one.
-  You can specify scopes and params with it.  Moreover, if you supply
+  You can specify scopes and params with it.  When using the decorator
-  a "name" you do not need to follow the "pytest_funcarg__NAME" naming
+  the "pytest_funcarg__" prefix becomes optional.
-  pattern.  Keeping with "funcarg" naming arguable now makes more
-  sense since the main interface using these resources are test and
-  setup functions. Keeping it probably causes the least semantic friction.
 
-* Drop setup_directory/setup_session and introduce a new @setup
+* funcarg factories can now use funcargs themselves
-  decorator similar to the @funcarg one but accepting funcargs.
 
-* cosnider the extended setup_X funcargs for dropping because
+* Drop setup/directory scope from this draft
-  the new @setup decorator probably is more flexible and introduces
+
-  less implementation complexity.
+* introduce a new @setup decorator similar to the @funcarg one
+  except that setup-markers cannot define parametriation themselves.
+  Instead they can easily depend on a parametrized funcarg (which
+  must not be visible at test function signatures).
+
+* drop consideration of setup_X support for funcargs because
+  it is less flexible and probably causes more implementation
+  troubles than the current @setup approach which can share
+  a lot of logic with the @funcarg one.
 
 .. currentmodule:: _pytest
 
@@ -78,17 +90,13 @@ There are some problems with this approach:
    ``extrakey`` parameter containing ``request.param`` to the 
    :py:func:`~python.Request.cached_setup` call.
 
-3. the current implementation is inefficient: it performs factory discovery
+3. there is no way how you can make use of funcarg factories
-   each time a "db" argument is required.  This discovery wrongly happens at 
+   in xUnit setup methods.
-   setup-time.
 
-4. there is no way how you can use funcarg factories, let alone 
+4. A non-parametrized funcarg factory cannot use a parametrized 
-   parametrization, when your tests use the xUnit setup_X approach.
+   funcarg resource if it isn't stated in the test function signature.
 
-5. there is no way to specify a per-directory scope for caching.
+The following sections address the advances which solve all of these problems.
-
-In the following sections, API extensions are presented to solve 
-each of these problems. 
 
 
 Direct scoping of funcarg factories
@@ -158,7 +166,7 @@ factory function.
 Direct usage of funcargs with funcargs factories
 ----------------------------------------------------------
 
-.. note:: Not Implemented - unclear if to.
+.. note:: Implemented.
 
 You can now directly use funcargs in funcarg factories.  Example::
 
@@ -168,33 +176,39 @@ You can now directly use funcargs in funcarg factories.  Example::
 
 Apart from convenience it also solves an issue when your factory
 depends on a parametrized funcarg.  Previously, a call to 
-``request.getfuncargvalue()`` would not allow pytest to know
+``request.getfuncargvalue()`` happens at test execution time and
-at collection time about the fact that a required resource is
+thus pytest would not know at collection time about the fact that 
-actually parametrized.
+a required resource is parametrized.
+
+No ``pytest_funcarg__`` prefix when using @funcarg decorator
+-------------------------------------------------------------------
 
-The "pytest_funcarg__" prefix becomes optional
------------------------------------------------------
 
 .. note:: Implemented
 
-When using the ``@funcarg`` decorator you do not need to use
+When using the ``@funcarg`` decorator the name of the function
-the ``pytest_funcarg__`` prefix any more::
+does not need to (and in fact cannot) use the ``pytest_funcarg__``
+naming::
 
     @pytest.mark.funcarg
     def db(request):
         ...
 
 The name under which the funcarg resource can be requested is ``db``.
-Any ``pytest_funcarg__`` prefix will be stripped. Note that a an
+
-unqualified funcarg-marker implies a scope of "function" meaning
+You can also use the "old" non-decorator way of specifying funcarg factories 
-that the funcarg factory will be called for each test function invocation.
+aka::
+
+    def pytest_funcarg__db(request):
+        ...
+
+It is recommended to use the funcarg-decorator, however.
 
 
+solving per-session setup / the new @setup marker
+--------------------------------------------------------------
 
-support for a new @setup marker
+.. note:: Implemented, at least working for basic situations.
-------------------------------------------------------
-
-.. note:: Not-Implemented, still under consideration if to.
 
 pytest for a long time offered a pytest_configure and a pytest_sessionstart
 hook which are often used to setup global resources.  This suffers from
@@ -212,9 +226,7 @@ several problems:
    fact that this hook is actually used for reporting, in particular
    the test-header with platform/custom information.
 
-4. there is no direct way how you can restrict setup to a directory scope.
+Moreover, it is today not easy to define a scoped setup from plugins or
-
-Moreover, it is today not easy to define scoped setup from plugins or
 conftest files other than to implement a ``pytest_runtest_setup()`` hook
 and caring for scoping/caching yourself.  And it's virtually impossible
 to do this with parametrization as ``pytest_runtest_setup()`` is called
@@ -222,222 +234,76 @@ during test execution and parametrization happens at collection time.
 
 It follows that pytest_configure/session/runtest_setup are often not
 appropriate for implementing common fixture needs.  Therefore, 
-pytest-2.X introduces a new "@pytest.mark.setup" marker, accepting
+pytest-2.X introduces a new "@pytest.mark.setup" marker which takes
-the same parameters as the @funcargs decorator.  The difference is
+an optional "scope" parameter.  
-that the decorated function can accept function arguments itself
-Example::
 
-    # content of conftest.py
+See :ref:`new_setup` for examples.
-    import pytest
-    @pytest.mark.setup(scope="session")
-    def mysetup(db):
-        ...
-
-This ``mysetup`` function is going to be executed when the first
-test in the directory tree executes.  It is going to be executed once
-per-session and it receives the ``db`` funcarg which must be of same
-of higher scope; you e. g. generally cannot use a per-module or per-function 
-scoped resource in a session-scoped setup function.
-
-You can also use ``@setup`` inside a test module or class::
-
-    # content of test_module.py
-    import pytest
-
-    @pytest.mark.setup(scope="module", params=[1,2,3])
-    def modes(tmpdir, request):
-        # ...
-
-This would execute the ``modes`` function once for each parameter
-which will be put at ``request.param``.  This request object offers
-the ``addfinalizer(func)`` helper which allows to register a function
-which will be executed when test functions within the specified scope 
-finished execution.
-
-.. note::
-   
-  For each scope, the funcargs will be setup and then the setup functions
-  will be called.  This allows @setup-decorated functions to depend
-  on already setup funcarg values by accessing ``request.funcargs``.
-
-Using funcarg resources in xUnit setup methods
-------------------------------------------------------------
-
-.. note:: Not implemented. Not clear if to.
-
-XXX Consider this feature in contrast to the @setup feature - probably
-introducing one of them is better and the @setup decorator is more flexible.
-
-For a long time, pytest has recommended the usage of funcarg 
-factories as a primary means for managing resources in your test run.
-It is a better approach than the jUnit-based approach in many cases, even 
-more with the new pytest-2.X features, because the funcarg resource factory
-provides a single place to determine scoping and parametrization.  Your tests 
-do not need to encode setup/teardown details in every test file's 
-setup_module/class/method.  
-
-However, the jUnit methods originally introduced by pytest to Python,
-remain popoular with nose and unittest-based test suites.  Without question,
-there are large existing test suites using this paradigm.  pytest-2.X
-recognizes this fact and now offers direct integration with funcarg resources.  Here is a basic example for getting a per-module tmpdir::
-
-    def setup_module(mod, tmpdir):
-        mod.tmpdir = tmpdir
-
-This will trigger pytest's funcarg mechanism to create a value of
-"tmpdir" which can then be used throughout the module as a global.
-
-The new extension to setup_X methods also works in case a resource is 
-parametrized. For example, let's consider an setup_class example using
-our "db" resource::
-
-    class TestClass:
-        def setup_class(cls, db):
-            cls.db = db
-            # perform some extra things on db
-            # so that test methods can work with it
-
-With pytest-2.X the setup* methods will be discovered at collection-time,
-allowing to seemlessly integrate this approach with parametrization,
-allowing the factory specification to determine all details. The
-setup_class itself does not itself need to be aware of the fact that 
-"db" might be a mysql/PG database.
-Note that if the specified resource is provided only as a per-testfunction
-resource, collection would early on report a ScopingMismatch error.
-
-
-the "directory" caching scope
---------------------------------------------
-
-.. note:: Not implemented.
-
-All API accepting a scope (:py:func:`cached_setup()`  and
-the new funcarg/setup decorators) now also accept a "directory"
-specification.  This allows to restrict/cache resource values on a
-per-directory level.
 
 funcarg and setup discovery now happens at collection time
 ---------------------------------------------------------------------
 
-.. note:: Partially implemented - collectonly shows no extra information
+.. note:: 
+    Partially implemented - collectonly shows no extra information however.
 
-pytest-2.X takes care to discover funcarg factories and setup_X methods
+pytest-2.X takes care to discover funcarg factories and @setup methods
 at collection time.  This is more efficient especially for large test suites. 
 Moreover, a call to "py.test --collectonly" should be able to show
 a lot of setup-information and thus presents a nice method to get an
 overview of resource management in your project.
 
-Implementation level 
-===================================================================
 
-To implement the above new features, pytest-2.X grows some new hooks and
+Sorting tests by funcarg scopes
-methods.  At the time of writing V2 and without actually implementing
+-------------------------------------------
-it, it is not clear how much of this new internal API will also be
-exposed and advertised e. g. for plugin writers. 
 
-The main effort, however, will lie in revising what is done at
+.. note:: Not implemented, Under consideration.
-collection and what at test setup time.  All funcarg factories and
-xUnit setup methods need to be discovered at collection time
-for the above mechanism to work.  Additionally all test function
-signatures need to be parsed in order to know which resources are
-used.  On the plus side, all previously collected fixtures and
-test functions only need to be called, no discovery is neccessary
-is required anymore.
 
-the "request" object incorporates scope-specific behaviour
+pytest by default sorts test items by their source location.
-------------------------------------------------------------------
+For class/module/session scoped funcargs it is not always
+desirable to have multiple active funcargs.  Sometimes,
+the application under test may not even be able to handle it
+because it relies on global state/side effects related to those 
+resources.
 
-funcarg factories receive a request object to help with implementing
+Therefore, pytest-2.3 tries to minimize the number of active
-finalization and inspection of the requesting-context.  If there is
+resources and re-orders test items accordingly.  Consider the following
-no scoping is in effect, nothing much will change of the API behaviour.
+example::
-However, with scoping the request object represents the according context.
-Let's consider this example::
 
-    @pytest.mark.factory_scope("class")
+    @pytest.mark.funcarg(scope="module", params=[1,2])
-    def pytest_funcarg__db(request):
+    def arg(request):
-        # ...
-        request.getfuncargvalue(...)
-        #
-        request.addfinalizer(db)
-
-Due to the class-scope, the request object will:
-
-- provide a ``None`` value for the ``request.function`` attribute. 
-- default to per-class finalization with the addfinalizer() call.
-- raise a ScopeMismatchError if a more broadly scoped factory
-  wants to use a more tighly scoped factory (e.g. per-function)
-
-In fact, the request object is likely going to provide a "node" 
-attribute, denoting the current collection node on which it internally
-operates.  (Prior to pytest-2.3 there already was an internal
-_pyfuncitem).
-
-As these are rather intuitive extensions, not much friction is expected 
-for test/plugin writers using the new scoping and parametrization mechanism. 
-It's, however, a serious internal effort to reorganize the pytest 
-implementation.
-
-
-node.register_factory/getresource() methods
---------------------------------------------------------
-
-In order to implement factory- and setup-method discovery at
-collection time, a new node API will be introduced to allow
-for factory registration and a getresource() call to obtain
-created values.  The exact details of this API remain subject
-to experimentation. The basic idea is to introduce two new
-methods to the Session class which is already available on all nodes
-through the ``node.session`` attribute::
-
-    class Session:
-        def register_resource_factory(self, name, factory_or_list, scope):
-            """ register a resource factory for the given name.
-
-            :param name: Name of the resource.
-            :factory_or_list: a function or a list of functions creating
-                              one or multiple resource values.
-            :param scope: a node instance. The factory will be only visisble
-                          available for all descendant nodes.
-                          specify the "session" instance for global availability
-            """
-
-        def getresource(self, name, node):
-            """ get a named resource for the give node.
-
-            This method looks up a matching funcarg resource factory 
-            and calls it.
-            """
-
-.. todo::
-
-    XXX While this new API (or some variant of it) may suffices to implement
-    all of the described new usage-level features, it remains unclear how the
-    existing "@parametrize" or "metafunc.parametrize()" calls will map to it.
-    These parametrize-approaches tie resource parametrization to the 
-    function/funcargs-usage rather than to the factories. 
-
-
-
-ISSUES
---------------------------
-
-decorating a parametrized funcarg factory::
-
-    @pytest.mark.funcarg(scope="session", params=["mysql", "pg"])
-    def db(request):
         ...
-    class TestClass:
+    @pytest.mark.funcarg(scope="function", params=[1,2])
-        @pytest.mark.funcarg(scope="function")
+    def otherarg(request):
-        def something(self, request):
-            session_db = request.getfuncargvalue("db")
         ...
 
-Here the function-scoped "something" factory uses the session-scoped
+    def test_0(otherarg):
-"db" factory to perform some additional steps.  The dependency, however,
+        pass
-is only visible at setup-time, when the factory actually gets called.
+    def test_1(arg):
+        pass
+    def test_2(arg, otherarg):
+        pass
 
-In order to allow parametrization at collection-time I see two ways:
+if arg.1, arg.2, otherarg.1, otherarg.2 denote the respective
+parametrized funcarg instances this will re-order test
+execution like follows::
+
+    test_0(otherarg.1)
+    test_0(otherarg.2)
+    test_1(arg.1) 
+    test_2(arg.1, otherarg.1)
+    test_2(arg.1, otherarg.2)
+    test_1(arg.2)
+    test_2(arg.2, otherarg.1)
+    test_2(arg.2, otherarg.2)
+
+Moreover, test_2(arg.1) will execute any registered teardowns for
+the arg.1 resource after the test finished execution.
+
+.. note::
+
+    XXX it's quite unclear at the moment how to implement.
+    If we have a 1000 tests requiring different sets of parametrized
+    resources with different scopes, how to re-order accordingly?  
+    It even seems difficult to express the expectation in a 
+    concise manner.
 
-- allow specifying dependencies in the funcarg-marker 
-- allow funcargs for factories as well 
 

setup.py

@@ -24,7 +24,7 @@ def main():
         name='pytest',
         description='py.test: simple powerful testing with Python',
         long_description = long_description,
-        version='2.3.0.dev3',
+        version='2.3.0.dev4',
         url='http://pytest.org',
         license='MIT license',
         platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'],

testing/test_python.py

@@ -1746,12 +1746,121 @@ class TestFuncargManager:
         reprec = testdir.inline_run("-s")
         reprec.assertoutcome(passed=1)
 
+class TestSetupDiscovery:
+    def pytest_funcarg__testdir(self, request):
+        testdir = request.getfuncargvalue("testdir")
+        testdir.makeconftest("""
+            import pytest
+            @pytest.mark.setup
+            def perfunction(request):
+                pass
+            @pytest.mark.setup
+            def perfunction2(request):
+                pass
+
+            def pytest_funcarg__fm(request):
+                return request.funcargmanager
+
+            def pytest_funcarg__item(request):
+                return request._pyfuncitem
+        """)
+        return testdir
+
+    def test_parsefactories_conftest(self, testdir):
+        testdir.makepyfile("""
+            def test_check_setup(item, fm):
+                setuplist, allnames = fm.getsetuplist(item.nodeid)
+                assert len(setuplist) == 2
+                assert setuplist[0][0].__name__ == "perfunction"
+                assert "request" in setuplist[0][1]
+                assert setuplist[1][0].__name__ == "perfunction2"
+                assert "request" in setuplist[1][1]
+        """)
+        reprec = testdir.inline_run("-s")
+        reprec.assertoutcome(passed=1)
+
+
+class TestSetupManagement:
+    def test_funcarg_and_setup(self, testdir):
+        testdir.makepyfile("""
+            import pytest
+            l = []
+            @pytest.mark.funcarg(scope="module")
+            def arg(request):
+                l.append(1)
+                return 0
+            @pytest.mark.setup(scope="class")
+            def something(request, arg):
+                l.append(2)
+
+            def test_hello(arg):
+                assert len(l) == 2
+                assert l == [1,2]
+                assert arg == 0
+
+            def test_hello2(arg):
+                assert len(l) == 2
+                assert l == [1,2]
+                assert arg == 0
+        """)
+        reprec = testdir.inline_run()
+        reprec.assertoutcome(passed=2)
+
+    def test_setup_uses_parametrized_resource(self, testdir):
+        testdir.makepyfile("""
+            import pytest
+            l = []
+            @pytest.mark.funcarg(params=[1,2])
+            def arg(request):
+                return request.param
+
+            @pytest.mark.setup
+            def something(request, arg):
+                l.append(arg)
+
+            def test_hello():
+                if len(l) == 1:
+                    assert l == [1]
+                elif len(l) == 2:
+                    assert l == [1, 2]
+                else:
+                    0/0
+
+        """)
+        reprec = testdir.inline_run("-s")
+        reprec.assertoutcome(passed=2)
+
+    def test_session_parametrized_function_setup(self, testdir):
+        testdir.makepyfile("""
+            import pytest
+
+            l = []
+
+            @pytest.mark.funcarg(scope="session", params=[1,2])
+            def arg(request):
+               return request.param
+
+            @pytest.mark.setup(scope="function")
+            def append(request, arg):
+                if request.function.__name__ == "test_some":
+                    l.append(arg)
+
+            def test_some():
+                pass
+
+            def test_result(arg):
+                assert len(l) == 2
+                assert l == [1,2]
+        """)
+        reprec = testdir.inline_run("-s")
+        reprec.assertoutcome(passed=4)
+
 class TestFuncargMarker:
     def test_parametrize(self, testdir):
         testdir.makepyfile("""
             import pytest
             @pytest.mark.funcarg(params=["a", "b", "c"])
-            def pytest_funcarg__arg(request):
+            def arg(request):
                 return request.param
             l = []
             def test_param(arg):
@@ -1767,7 +1876,7 @@ class TestFuncargMarker:
             import pytest
             l = []
             @pytest.mark.funcarg(scope="module")
-            def pytest_funcarg__arg(request):
+            def arg(request):
                 l.append(1)
                 return 1
 
@@ -1789,7 +1898,7 @@ class TestFuncargMarker:
             import pytest
             l = []
             @pytest.mark.funcarg(scope="module")
-            def pytest_funcarg__arg(request):
+            def arg(request):
                 l.append(1)
                 return 1
 
@@ -1812,7 +1921,7 @@ class TestFuncargMarker:
             finalized = []
             created = []
             @pytest.mark.funcarg(scope="module")
-            def pytest_funcarg__arg(request):
+            def arg(request):
                 created.append(1)
                 assert request.scope == "module"
                 request.addfinalizer(lambda: finalized.append(1))
@@ -1851,14 +1960,14 @@ class TestFuncargMarker:
             finalized = []
             created = []
             @pytest.mark.funcarg(scope="function")
-            def pytest_funcarg__arg(request):
+            def arg(request):
                 pass
         """)
         testdir.makepyfile(
             test_mod1="""
                 import pytest
                 @pytest.mark.funcarg(scope="session")
-                def pytest_funcarg__arg(request):
+                def arg(request):
                     %s
                 def test_1(arg):
                     pass
@@ -1894,7 +2003,7 @@ class TestFuncargMarker:
         testdir.makepyfile("""
             import pytest
             @pytest.mark.funcarg(scope="module", params=["a", "b", "c"])
-            def pytest_funcarg__arg(request):
+            def arg(request):
                 return request.param
             l = []
             def test_param(arg):