minimize active parametrized non-function scoped resources by

- re-ordering at collection time
- modifying setup/teardown

_pytest/__init__.py

@@ -1,2 +1,2 @@
 #
-__version__ = '2.3.0.dev4'
+__version__ = '2.3.0.dev5'

_pytest/impl

@@ -1,4 +1,187 @@
 
+the new @setup functions 
+--------------------------------------
+
+Consider a given @setup-marked function::
+
+    @pytest.mark.setup(maxscope=SCOPE)
+    def mysetup(request, arg1, arg2, ...)
+        ...
+        request.addfinalizer(fin)
+        ...
+
+then FUNCARGSET denotes the set of (arg1, arg2, ...) funcargs and
+all of its dependent funcargs.  The mysetup function will execute
+for any matching test item once per scope.  
+
+The scope is determined as the minimum scope of all scopes of the args
+in FUNCARGSET and the given "maxscope". 
+
+If mysetup has been called and no finalizers have been called it is
+called "active".
+
+Furthermore the following rules apply:
+
+- if an arg value in FUNCARGSET is about to be torn down, the 
+  mysetup-registered finalizers will execute as well.
+
+- There will never be two active mysetup invocations.
+
+Example 1, session scope::
+
+    @pytest.mark.funcarg(scope="session", params=[1,2])
+    def db(request):
+        request.addfinalizer(db_finalize)
+
+    @pytest.mark.setup
+    def mysetup(request, db):
+        request.addfinalizer(mysetup_finalize)
+        ...
+
+And a given test module:
+
+    def test_something():
+        ...
+    def test_otherthing():
+        pass
+
+Here is what happens::
+
+    db(request) executes with request.param == 1
+        mysetup(request, db) executes
+            test_something() executes
+            test_otherthing() executes
+            mysetup_finalize() executes
+    db_finalize() executes
+    db(request) executes with request.param == 2
+        mysetup(request, db) executes
+            test_something() executes
+            test_otherthing() executes
+        mysetup_finalize() executes
+    db_finalize() executes
+
+Example 2, session/function scope::
+
+    @pytest.mark.funcarg(scope="session", params=[1,2])
+    def db(request):
+        request.addfinalizer(db_finalize)
+
+    @pytest.mark.setup(scope="function")
+    def mysetup(request, db):
+        ...
+        request.addfinalizer(mysetup_finalize)
+        ...
+
+And a given test module:
+
+    def test_something():
+        ...
+    def test_otherthing():
+        pass
+
+Here is what happens::
+
+    db(request) executes with request.param == 1
+        mysetup(request, db) executes
+            test_something() executes
+        mysetup_finalize() executes
+        mysetup(request, db) executes
+            test_otherthing() executes
+        mysetup_finalize() executes
+    db_finalize() executes
+    db(request) executes with request.param == 2
+        mysetup(request, db) executes
+            test_something() executes
+        mysetup_finalize() executes
+        mysetup(request, db) executes
+            test_otherthing() executes
+        mysetup_finalize() executes
+    db_finalize() executes
+
+
+Example 3 - funcargs session-mix
+----------------------------------------
+
+Similar with funcargs, an example::
+
+    @pytest.mark.funcarg(scope="session", params=[1,2])
+    def db(request):
+        request.addfinalizer(db_finalize)
+
+    @pytest.mark.funcarg(scope="function")
+    def table(request, db):
+        ...
+        request.addfinalizer(table_finalize)
+        ...
+
+And a given test module:
+
+    def test_something(table):
+        ...
+    def test_otherthing(table):
+        pass
+    def test_thirdthing():
+        pass
+
+Here is what happens::
+
+    db(request) executes with param == 1
+        table(request, db)
+            test_something(table)
+        table_finalize()
+        table(request, db)
+            test_otherthing(table)
+        table_finalize()
+    db_finalize
+    db(request) executes with param == 2
+        table(request, db)
+            test_something(table)
+        table_finalize()
+        table(request, db)
+            test_otherthing(table)
+        table_finalize()
+    db_finalize
+    test_thirdthing()
+    
+Data structures
+--------------------
+
+pytest internally maintains a dict of active funcargs with cache, param,
+finalizer, (scopeitem?) information:
+
+    active_funcargs = dict()
+
+if a parametrized "db" is activated:
+    
+    active_funcargs["db"] = FuncargInfo(dbvalue, paramindex, 
+                                        FuncargFinalize(...), scopeitem)
+
+if a test is torn down and the next test requires a differently 
+parametrized "db":
+
+    for argname in item.callspec.params:
+        if argname in active_funcargs:
+            funcarginfo = active_funcargs[argname]
+            if funcarginfo.param != item.callspec.params[argname]:
+                funcarginfo.callfinalizer()
+                del node2funcarg[funcarginfo.scopeitem]
+                del active_funcargs[argname]
+    nodes_to_be_torn_down = ...
+    for node in nodes_to_be_torn_down:
+        if node in node2funcarg:
+            argname = node2funcarg[node]
+            active_funcargs[argname].callfinalizer()
+            del node2funcarg[node]
+            del active_funcargs[argname]
+
+if a test is setup requiring a "db" funcarg:
+
+    if "db" in active_funcargs:
+        return active_funcargs["db"][0]
+    funcarginfo = setup_funcarg()
+    active_funcargs["db"] = funcarginfo
+    node2funcarg[funcarginfo.scopeitem] = "db"
+
 Implementation plan for resources
 ------------------------------------------
 

_pytest/main.py

@@ -465,13 +465,48 @@ class FuncargManager:
                 marker = getattr(fac, "funcarg", None)
                 if marker is not None:
                     params = marker.kwargs.get("params")
+                    scope = marker.kwargs.get("scope", "function")
                     if params is not None:
-                        metafunc.parametrize(argname, params, indirect=True)
+                        metafunc.parametrize(argname, params, indirect=True,
+                                             scope=scope)
                 newfuncargnames = getfuncargnames(fac)
                 newfuncargnames.remove("request")
                 funcargnames.extend(newfuncargnames)
 
+    def pytest_collection_modifyitems(self, items):
+        # separate parametrized setups
+        def sortparam(item1, item2):
+            try:
+                cs1 = item1.callspec
+                cs2 = item2.callspec
+                common = set(cs1.params).intersection(cs2.params)
+            except AttributeError:
+                pass
+            else:
+                if common:
+                    common = list(common)
+                    common.sort(key=lambda x: cs1._arg2scopenum[x])
+                    for x in common:
+                        res = cmp(cs1.params[x], cs2.params[x])
+                        if res != 0:
+                            return res
+            return 0  # leave previous order
+        items.sort(cmp=sortparam)
 
+    def pytest_runtest_teardown(self, item, nextitem):
+        try:
+            cs1 = item.callspec
+        except AttributeError:
+            return
+        for name in cs1.params:
+            try:
+                if name in nextitem.callspec.params and \
+                    cs1.params[name] == nextitem.callspec.params[name]:
+                    continue
+            except AttributeError:
+                pass
+            key = (name, cs1.params[name])
+            item.session._setupstate._callfinalizers(key)
 
     def _parsefactories(self, holderobj, nodeid):
         if holderobj in self._holderobjseen:

_pytest/python.py

@@ -499,11 +499,13 @@ class CallSpec2(object):
         self._globalid = _notexists
         self._globalid_args = set()
         self._globalparam = _notexists
+        self._arg2scopenum = {}  # used for sorting parametrized resources
 
     def copy(self, metafunc):
         cs = CallSpec2(self.metafunc)
         cs.funcargs.update(self.funcargs)
         cs.params.update(self.params)
+        cs._arg2scopenum.update(self._arg2scopenum)
         cs._idlist = list(self._idlist)
         cs._globalid = self._globalid
         cs._globalid_args = self._globalid_args
@@ -526,10 +528,11 @@ class CallSpec2(object):
     def id(self):
         return "-".join(map(str, filter(None, self._idlist)))
 
-    def setmulti(self, valtype, argnames, valset, id):
+    def setmulti(self, valtype, argnames, valset, id, scopenum=0):
         for arg,val in zip(argnames, valset):
             self._checkargnotcontained(arg)
             getattr(self, valtype)[arg] = val
+            self._arg2scopenum[arg] = scopenum
         self._idlist.append(id)
 
     def setall(self, funcargs, id, param):
@@ -556,8 +559,10 @@ class Metafunc:
         self.module = module
         self._calls = []
         self._ids = py.builtin.set()
+        self._arg2scopenum = {}
 
-    def parametrize(self, argnames, argvalues, indirect=False, ids=None):
+    def parametrize(self, argnames, argvalues, indirect=False, ids=None,
+        scope="function"):
         """ Add new invocations to the underlying test function using the list
         of argvalues for the given argnames.  Parametrization is performed
         during the collection phase.  If you need to setup expensive resources
@@ -581,6 +586,7 @@ class Metafunc:
         if not isinstance(argnames, (tuple, list)):
             argnames = (argnames,)
             argvalues = [(val,) for val in argvalues]
+        scopenum = scopes.index(scope)
         if not indirect:
             #XXX should we also check for the opposite case?
             for arg in argnames:
@@ -595,7 +601,8 @@ class Metafunc:
             for i, valset in enumerate(argvalues):
                 assert len(valset) == len(argnames)
                 newcallspec = callspec.copy(self)
-                newcallspec.setmulti(valtype, argnames, valset, ids[i])
+                newcallspec.setmulti(valtype, argnames, valset, ids[i],
+                                     scopenum)
                 newcalls.append(newcallspec)
         self._calls = newcalls
 
@@ -995,6 +1002,7 @@ class FuncargRequest:
     def _callsetup(self):
         setuplist, allnames = self.funcargmanager.getsetuplist(
                                                     self._pyfuncitem.nodeid)
+        mp = monkeypatch()
         for setupfunc, funcargnames in setuplist:
             kwargs = {}
             for name in funcargnames:
@@ -1002,11 +1010,16 @@ class FuncargRequest:
                     kwargs[name] = self
                 else:
                     kwargs[name] = self.getfuncargvalue(name)
+
             scope = readscope(setupfunc, "setup")
-            if scope is None:
+            mp.setattr(self, 'scope', scope)
-                setupfunc(**kwargs)
+            try:
-            else:
+                if scope is None:
-                self.cached_setup(lambda: setupfunc(**kwargs), scope=scope)
+                    setupfunc(**kwargs)
+                else:
+                    self.cached_setup(lambda: setupfunc(**kwargs), scope=scope)
+            finally:
+                mp.undo()
 
     def getfuncargvalue(self, argname):
         """ Retrieve a function argument by name for this test
@@ -1047,7 +1060,7 @@ class FuncargRequest:
         else:
             mp.setattr(self, 'param', param, raising=False)
 
-        # implemenet funcarg marker scope
+        # implement funcarg marker scope
         scope = readscope(funcargfactory, "funcarg")
 
         if scope is not None:
@@ -1100,7 +1113,12 @@ class FuncargRequest:
         self._addfinalizer(finalizer, scope=self.scope)
 
     def _addfinalizer(self, finalizer, scope):
-        colitem = self._getscopeitem(scope)
+        if scope != "function" and hasattr(self, "param"):
+            # parametrized resources are sorted by param
+            # so we rather store finalizers per (argname, param)
+            colitem = (self._currentarg, self.param)
+        else:
+            colitem = self._getscopeitem(scope)
         self._pyfuncitem.session._setupstate.addfinalizer(
             finalizer=finalizer, colitem=colitem)
 

_pytest/runner.py

@@ -294,6 +294,8 @@ class SetupState(object):
         """ attach a finalizer to the given colitem.
         if colitem is None, this will add a finalizer that
         is called at the end of teardown_all().
+        if colitem is a tuple, it will be used as a key
+        and needs an explicit call to _callfinalizers(key) later on.
         """
         assert hasattr(finalizer, '__call__')
         #assert colitem in self.stack
@@ -311,15 +313,17 @@ class SetupState(object):
 
     def _teardown_with_finalization(self, colitem):
         self._callfinalizers(colitem)
-        if colitem:
+        if hasattr(colitem, "teardown"):
             colitem.teardown()
         for colitem in self._finalizers:
-            assert colitem is None or colitem in self.stack
+            assert colitem is None or colitem in self.stack \
+             or isinstance(colitem, tuple)
 
     def teardown_all(self):
         while self.stack:
             self._pop_and_teardown()
-        self._teardown_with_finalization(None)
+        for key in list(self._finalizers):
+            self._teardown_with_finalization(key)
         assert not self._finalizers
 
     def teardown_exact(self, item, nextitem):

doc/en/example/newexamples.txt

@@ -48,7 +48,7 @@ If you run the tests::
     ================================= FAILURES =================================
     ________________________________ test_ehlo _________________________________
     
-    smtp = <smtplib.SMTP instance at 0x20ba7e8>
+    smtp = <smtplib.SMTP instance at 0x2c0f170>
     
         def test_ehlo(smtp):
             response = smtp.ehlo()
@@ -60,7 +60,7 @@ If you run the tests::
     test_module.py:5: AssertionError
     ________________________________ test_noop _________________________________
     
-    smtp = <smtplib.SMTP instance at 0x20ba7e8>
+    smtp = <smtplib.SMTP instance at 0x2c0f170>
     
         def test_noop(smtp):
             response = smtp.noop()
@@ -69,7 +69,7 @@ If you run the tests::
     E       assert 0
     
     test_module.py:10: AssertionError
-    2 failed in 0.27 seconds
+    2 failed in 0.21 seconds
 
 you will see the two ``assert 0`` failing and can see that
 the same (session-scoped) object was passed into the two test functions.
@@ -98,9 +98,31 @@ another run::
     collecting ... collected 4 items
     FFFF
     ================================= FAILURES =================================
+    ________________________ test_ehlo[mail.python.org] ________________________
+    
+    smtp = <smtplib.SMTP instance at 0x20fca70>
+    
+        def test_ehlo(smtp):
+            response = smtp.ehlo()
+            assert response[0] == 250
+    >       assert "merlinux" in response[1]
+    E       assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
+    
+    test_module.py:4: AssertionError
+    ________________________ test_noop[mail.python.org] ________________________
+    
+    smtp = <smtplib.SMTP instance at 0x20fca70>
+    
+        def test_noop(smtp):
+            response = smtp.noop()
+            assert response[0] == 250
+    >       assert 0  # for demo purposes
+    E       assert 0
+    
+    test_module.py:10: AssertionError
     __________________________ test_ehlo[merlinux.eu] __________________________
     
-    smtp = <smtplib.SMTP instance at 0x2a51830>
+    smtp = <smtplib.SMTP instance at 0x2104bd8>
     
         def test_ehlo(smtp):
             response = smtp.ehlo()
@@ -110,20 +132,9 @@ another run::
     E       assert 0
     
     test_module.py:5: AssertionError
-    ________________________ test_ehlo[mail.python.org] ________________________
-    
-    smtp = <smtplib.SMTP instance at 0x2a56c20>
-    
-        def test_ehlo(smtp):
-            response = smtp.ehlo()
-            assert response[0] == 250
-    >       assert "merlinux" in response[1]
-    E       assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
-    
-    test_module.py:4: AssertionError
     __________________________ test_noop[merlinux.eu] __________________________
     
-    smtp = <smtplib.SMTP instance at 0x2a51830>
+    smtp = <smtplib.SMTP instance at 0x2104bd8>
     
         def test_noop(smtp):
             response = smtp.noop()
@@ -132,20 +143,7 @@ another run::
     E       assert 0
     
     test_module.py:10: AssertionError
-    ________________________ test_noop[mail.python.org] ________________________
+    4 failed in 6.48 seconds
-    
-    smtp = <smtplib.SMTP instance at 0x2a56c20>
-    
-        def test_noop(smtp):
-            response = smtp.noop()
-            assert response[0] == 250
-    >       assert 0  # for demo purposes
-    E       assert 0
-    
-    test_module.py:10: AssertionError
-    4 failed in 6.91 seconds
-    closing <smtplib.SMTP instance at 0x2a56c20>
-    closing <smtplib.SMTP instance at 0x2a51830>
 
 We get four failures because we are running the two tests twice with
 different ``smtp`` instantiations as defined on the factory.
@@ -157,14 +155,14 @@ You can look at what tests pytest collects without running them::
 
     $ py.test --collectonly
     =========================== test session starts ============================
-    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev4
+    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev5
     plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
     collecting ... collected 4 items
     <Module 'test_module.py'>
-      <Function 'test_ehlo[merlinux.eu]'>
       <Function 'test_ehlo[mail.python.org]'>
-      <Function 'test_noop[merlinux.eu]'>
       <Function 'test_noop[mail.python.org]'>
+      <Function 'test_ehlo[merlinux.eu]'>
+      <Function 'test_noop[merlinux.eu]'>
     
     =============================  in 0.02 seconds =============================
 
@@ -174,13 +172,13 @@ And you can run without output capturing and minimized failure reporting to chec
     collecting ... collected 4 items
     FFFF
     ================================= FAILURES =================================
-    /home/hpk/tmp/doc-exec-361/test_module.py:5: assert 0
+    /home/hpk/tmp/doc-exec-386/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
-    /home/hpk/tmp/doc-exec-361/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
+    /home/hpk/tmp/doc-exec-386/test_module.py:10: assert 0
-    /home/hpk/tmp/doc-exec-361/test_module.py:10: assert 0
+    /home/hpk/tmp/doc-exec-386/test_module.py:5: assert 0
-    /home/hpk/tmp/doc-exec-361/test_module.py:10: assert 0
+    /home/hpk/tmp/doc-exec-386/test_module.py:10: assert 0
-    4 failed in 6.83 seconds
+    4 failed in 6.45 seconds
-    closing <smtplib.SMTP instance at 0x236da28>
+    closing <smtplib.SMTP instance at 0x29d0a28>
-    closing <smtplib.SMTP instance at 0x23687e8>
+    closing <smtplib.SMTP instance at 0x29d8878>
 
 .. _`new_setup`:
 
@@ -217,8 +215,9 @@ And the test file contains a setup function using this resource::
     # content of test_module.py
     import pytest
 
-    @pytest.mark.setup(scope="function")
+    @pytest.mark.setup(scope="module")
     def setresource(resource):
+        print "setupresource", resource
         global myresource
         myresource = resource
 
@@ -236,10 +235,11 @@ Let's run this module::
     collecting ... collected 2 items
     ..
     2 passed in 0.24 seconds
-    created resource /home/hpk/tmp/pytest-3715/test_10
+    created resource /home/hpk/tmp/pytest-3875/test_10
-    using myresource /home/hpk/tmp/pytest-3715/test_10
+    setupresource /home/hpk/tmp/pytest-3875/test_10
-    using myresource /home/hpk/tmp/pytest-3715/test_10
+    using myresource /home/hpk/tmp/pytest-3875/test_10
-    finalize /home/hpk/tmp/pytest-3715/test_10
+    using myresource /home/hpk/tmp/pytest-3875/test_10
+    finalize /home/hpk/tmp/pytest-3875/test_10
 
 The two test functions will see the same resource instance because it has
 a module life cycle or scope.  
@@ -264,23 +264,22 @@ Running this will run four tests::
     $ py.test -qs
     collecting ... collected 4 items
     ....
-    4 passed in 0.24 seconds
+    4 passed in 0.25 seconds
-    created resource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa
+    created resource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
-    using myresource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa
+    setupresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
-    created resource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb
+    using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
-    using myresource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb
+    using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
-    using myresource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa
+    finalize /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
-    using myresource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb
+    created resource /home/hpk/tmp/pytest-3876/test_1_bbb_0/bbb
-    finalize /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb
+    using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
-    finalize /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa
+    using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
+    finalize /home/hpk/tmp/pytest-3876/test_1_bbb_0/bbb
 
 Each parameter causes the creation of a respective resource and the
 unchanged test module uses it in its ``@setup`` decorated method.
 
 .. note::
 
-   Currently, parametrized tests are sorted by test function location
+   Parametrized Resources will be grouped together during test execution.
-   so a test function will execute multiple times with different parametrized
+   Moreover, any added finalizers will be run before the next parametrized
-   funcargs.  If you have class/module/session scoped funcargs and
+   resource is being setup.
-   they cause global side effects this can cause problems because the
-   code under test may not be prepared to deal with it.

doc/en/resources.txt

@@ -51,6 +51,8 @@ implementation or backward compatibility issues.  The main changes are:
   troubles than the current @setup approach which can share
   a lot of logic with the @funcarg one.
 
+* tests are grouped by any parametrized resource
+
 .. currentmodule:: _pytest
 
 
@@ -252,58 +254,82 @@ a lot of setup-information and thus presents a nice method to get an
 overview of resource management in your project.
 
 
-Sorting tests by funcarg scopes
+Grouping tests by resource parameters
--------------------------------------------
+----------------------------------------------------------
 
-.. note:: Not implemented, Under consideration.
+.. note:: Implemented.
 
-pytest by default sorts test items by their source location.
+pytest usually sorts test items by their source location.
-For class/module/session scoped funcargs it is not always
+With pytest-2.X tests are first grouped by resource parameters.
-desirable to have multiple active funcargs.  Sometimes,
+If you have a parametrized resource, then all the tests using it
-the application under test may not even be able to handle it
+will first execute with it.  Then any finalizers are called and then
-because it relies on global state/side effects related to those 
+the next parametrized resource instance is created and its tests are run.
-resources.
+Among other things, this allows to have per-session parametrized setups
+including ones which affect global state of an application.
 
-Therefore, pytest-2.3 tries to minimize the number of active
+The following example uses two parametrized funcargs, one of which is 
-resources and re-orders test items accordingly.  Consider the following
+scoped on a per-module basis::
-example::
+
+    # content of test_module.py
+    import pytest
+
+    @pytest.mark.funcarg(scope="module", params=["mod1", "mod2"])
+    def modarg(request):
+        param = request.param
+        print "create", param
+        def fin():
+            print "fin", param
+        request.addfinalizer(fin)
+        return param
 
-    @pytest.mark.funcarg(scope="module", params=[1,2])
-    def arg(request):
-        ...
     @pytest.mark.funcarg(scope="function", params=[1,2])
     def otherarg(request):
-        ...
+        return request.param
 
     def test_0(otherarg):
-        pass
+        print "  test0", otherarg
-    def test_1(arg):
+    def test_1(modarg):
-        pass
+        print "  test1", modarg
-    def test_2(arg, otherarg):
+    def test_2(otherarg, modarg):
-        pass
+        print "  test2", otherarg, modarg
 
-if arg.1, arg.2, otherarg.1, otherarg.2 denote the respective
+If you run the tests in verbose mode and with looking at captured output::
-parametrized funcarg instances this will re-order test
-execution like follows::
 
-    test_0(otherarg.1)
+    $ py.test -v -s
-    test_0(otherarg.2)
+    =========================== test session starts ============================
-    test_1(arg.1) 
+    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev5 -- /home/hpk/venv/1/bin/python
-    test_2(arg.1, otherarg.1)
+    cachedir: /home/hpk/tmp/doc-exec-382/.cache
-    test_2(arg.1, otherarg.2)
+    plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
-    test_1(arg.2)
+    collecting ... collected 8 items
-    test_2(arg.2, otherarg.1)
+    
-    test_2(arg.2, otherarg.2)
+    test_module.py:16: test_0[1] PASSED
+    test_module.py:16: test_0[2] PASSED
+    test_module.py:18: test_1[mod1] PASSED
+    test_module.py:20: test_2[1-mod1] PASSED
+    test_module.py:20: test_2[2-mod1] PASSED
+    test_module.py:18: test_1[mod2] PASSED
+    test_module.py:20: test_2[1-mod2] PASSED
+    test_module.py:20: test_2[2-mod2] PASSED
+    
+    ========================= 8 passed in 0.03 seconds =========================
+      test0 1
+      test0 2
+    create mod1
+      test1 mod1
+      test2 1 mod1
+      test2 2 mod1
+    fin mod1
+    create mod2
+      test1 mod2
+      test2 1 mod2
+      test2 2 mod2
+    fin mod2
 
-Moreover, test_2(arg.1) will execute any registered teardowns for
+You can see that that the parametrized ``modarg`` resource lead to
-the arg.1 resource after the test finished execution.
+a re-ordering of test execution. Moreover, the finalizer for the
+"mod1" parametrized resource was executed before the "mod2" resource 
+was setup with a different parameter.
 
 .. note::
 
-    XXX it's quite unclear at the moment how to implement.
+    The current implementation is experimental.
-    If we have a 1000 tests requiring different sets of parametrized
-    resources with different scopes, how to re-order accordingly?  
-    It even seems difficult to express the expectation in a 
-    concise manner.
-
-

setup.py

@@ -24,7 +24,7 @@ def main():
         name='pytest',
         description='py.test: simple powerful testing with Python',
         long_description = long_description,
-        version='2.3.0.dev4',
+        version='2.3.0.dev5',
         url='http://pytest.org',
         license='MIT license',
         platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'],

testing/test_python.py

@@ -1695,7 +1695,7 @@ class TestResourceIntegrationFunctional:
         """)
         result = testdir.runpytest("-v")
         assert result.ret == 1
-        result.stdout.fnmatch_lines([
+        result.stdout.fnmatch_lines_random([
             "*test_function*basic*PASSED",
             "*test_function*advanced*FAILED",
         ])
@@ -1849,10 +1849,10 @@ class TestSetupManagement:
                 pass
 
             def test_result(arg):
-                assert len(l) == 2
+                assert len(l) == arg
-                assert l == [1,2]
+                assert l[:arg] == [1,2][:arg]
         """)
-        reprec = testdir.inline_run("-s")
+        reprec = testdir.inline_run("-v", "-s")
         reprec.assertoutcome(passed=4)
 
 class TestFuncargMarker:
@@ -2013,3 +2013,170 @@ class TestFuncargMarker:
         """)
         reprec = testdir.inline_run()
         reprec.assertoutcome(passed=4)
+
+    def test_scope_mismatch(self, testdir):
+        testdir.makeconftest("""
+            import pytest
+            @pytest.mark.funcarg(scope="function")
+            def arg(request):
+                pass
+        """)
+        testdir.makepyfile("""
+            import pytest
+            @pytest.mark.funcarg(scope="session")
+            def arg(request, arg):
+                pass
+            def test_mismatch(arg):
+                pass
+        """)
+        result = testdir.runpytest()
+        result.stdout.fnmatch_lines([
+            "*ScopeMismatch*",
+            "*1 error*",
+        ])
+
+    def test_parametrize_separated_order(self, testdir):
+        testdir.makepyfile("""
+            import pytest
+
+            @pytest.mark.funcarg(scope="module", params=[1, 2])
+            def arg(request):
+                return request.param
+
+            l = []
+            def test_1(arg):
+                l.append(arg)
+            def test_2(arg):
+                l.append(arg)
+            def test_3():
+                assert len(l) == 4
+                assert l[0] == l[1]
+                assert l[2] == l[3]
+
+        """)
+        reprec = testdir.inline_run("-v")
+        reprec.assertoutcome(passed=5)
+
+    def test_parametrize_separated_order_higher_scope_first(self, testdir):
+        testdir.makepyfile("""
+            import pytest
+
+            @pytest.mark.funcarg(scope="function", params=[1, 2])
+            def arg(request):
+                param = request.param
+                request.addfinalizer(lambda: l.append("fin:%s" % param))
+                l.append("create:%s" % param)
+                return request.param
+
+            @pytest.mark.funcarg(scope="module", params=["mod1", "mod2"])
+            def modarg(request):
+                param = request.param
+                request.addfinalizer(lambda: l.append("fin:%s" % param))
+                l.append("create:%s" % param)
+                return request.param
+
+            l = []
+            def test_1(arg):
+                l.append("test1")
+            def test_2(modarg):
+                l.append("test2")
+            def test_3(arg, modarg):
+                l.append("test3")
+            def test_4(modarg, arg):
+                l.append("test4")
+            def test_5():
+                assert len(l) == 12 * 3
+                import pprint
+                pprint.pprint(l)
+                assert l == [
+                    'create:1', 'test1', 'fin:1',
+                    'create:2', 'test1', 'fin:2',
+                    'create:mod1', 'test2', 'create:1', 'test3', 'fin:1',
+                    'create:1', 'test4', 'fin:1', 'create:2', 'test3', 'fin:2',
+                    'create:2', 'test4', 'fin:mod1', 'fin:2',
+
+                    'create:mod2', 'test2', 'create:1', 'test3', 'fin:1',
+                    'create:1', 'test4', 'fin:1', 'create:2', 'test3', 'fin:2',
+                    'create:2', 'test4', 'fin:mod2', 'fin:2',
+                ]
+
+        """)
+        reprec = testdir.inline_run("-v")
+        reprec.assertoutcome(passed=12+1)
+
+    def test_parametrize_separated_lifecycle(self, testdir):
+        testdir.makepyfile("""
+            import pytest
+
+            @pytest.mark.funcarg(scope="module", params=[1, 2])
+            def arg(request):
+                x = request.param
+                request.addfinalizer(lambda: l.append("fin%s" % x))
+                return request.param
+
+            l = []
+            def test_1(arg):
+                l.append(arg)
+            def test_2(arg):
+                l.append(arg)
+            def test_3():
+                assert len(l) == 6
+                assert l[0] == l[1]
+                assert l[2] == "fin1"
+                assert l[3] == l[4]
+                assert l[5] == "fin2"
+
+        """)
+        reprec = testdir.inline_run("-v")
+        reprec.assertoutcome(passed=5)
+
+    def test_parametrize_function_scoped_finalizers_called(self, testdir):
+        testdir.makepyfile("""
+            import pytest
+
+            @pytest.mark.funcarg(scope="function", params=[1, 2])
+            def arg(request):
+                x = request.param
+                request.addfinalizer(lambda: l.append("fin%s" % x))
+                return request.param
+
+            l = []
+            def test_1(arg):
+                l.append(arg)
+            def test_2(arg):
+                l.append(arg)
+            def test_3():
+                assert len(l) == 8
+                assert l == [1, "fin1", 1, "fin1", 2, "fin2", 2, "fin2"]
+        """)
+        reprec = testdir.inline_run("-v")
+        reprec.assertoutcome(passed=5)
+
+    def test_parametrize_setup_function(self, testdir):
+        testdir.makepyfile("""
+            import pytest
+
+            @pytest.mark.funcarg(scope="module", params=[1, 2])
+            def arg(request):
+                return request.param
+
+            @pytest.mark.setup(scope="module")
+            def mysetup(request, arg):
+                request.addfinalizer(lambda: l.append("fin%s" % arg))
+                l.append("setup%s" % arg)
+
+            l = []
+            def test_1(arg):
+                l.append(arg)
+            def test_2(arg):
+                l.append(arg)
+            def test_3():
+                import pprint
+                pprint.pprint(l)
+                assert l == ["setup1", 1, 1, "fin1",
+                             "setup2", 2, 2, "fin2",]
+
+        """)
+        reprec = testdir.inline_run("-v")
+        reprec.assertoutcome(passed=5)
+