- enhance ordering of tests using parametrized resources

- introduce a refined way to perform finalization for setup functions
  which does not use cached_setup() anymore

_pytest/__init__.py

@@ -1,2 +1,2 @@
 #
-__version__ = '2.3.0.dev5'
+__version__ = '2.3.0.dev6'

_pytest/impl

@@ -1,3 +1,29 @@
+Sorting per-resource
+-----------------------------
+
+for any given set of items:
+
+- collect items per session-scoped parametrized funcarg
+- re-order until items no parametrizations are mixed
+
+  examples:
+
+        test()
+        test1(s1)
+        test1(s2)     
+        test2()
+        test3(s1)
+        test3(s2)
+
+  gets sorted to:
+
+        test()
+        test2()
+        test1(s1)
+        test3(s1)
+        test1(s2)
+        test3(s2)
+ 
 
 the new @setup functions 
 --------------------------------------

_pytest/main.py

@@ -425,6 +425,7 @@ class FuncargManager:
         session.config.pluginmanager.register(self, "funcmanage")
         self._holderobjseen = set()
         self.setuplist = []
+        self._arg2finish = {}
 
     ### XXX this hook should be called for historic events like pytest_configure
     ### so that we don't have to do the below pytest_collection hook
@@ -469,23 +470,7 @@ class FuncargManager:
 
     def pytest_collection_modifyitems(self, items):
         # separate parametrized setups
-        def sortparam(item1, item2):
+        items[:] = parametrize_sorted(items, set(), {}, 0)
-            try:
-                cs1 = item1.callspec
-                cs2 = item2.callspec
-                common = set(cs1.params).intersection(cs2.params)
-            except AttributeError:
-                pass
-            else:
-                if common:
-                    common = list(common)
-                    common.sort(key=lambda x: cs1._arg2scopenum[x])
-                    for x in common:
-                        res = cmp(cs1.params[x], cs2.params[x])
-                        if res != 0:
-                            return res
-            return 0  # leave previous order
-        items.sort(cmp=sortparam)
 
     def pytest_runtest_teardown(self, item, nextitem):
         try:
@@ -501,6 +486,10 @@ class FuncargManager:
                 pass
             key = (name, cs1.params[name])
             item.session._setupstate._callfinalizers(key)
+            l = self._arg2finish.get(name)
+            if l is not None:
+                for fin in l:
+                    fin()
 
     def _parsefactories(self, holderobj, nodeid):
         if holderobj in self._holderobjseen:
@@ -577,17 +566,59 @@ class FuncargManager:
         msg += "\n use 'py.test --funcargs [testpath]' for help on them."
         raise FuncargLookupError(function, msg)
 
+    def ensure_setupcalls(self, request):
+        setuplist, allnames = self.getsetuplist(request._pyfuncitem.nodeid)
+        for setupcall in setuplist:
+            if setupcall.active:
+                continue
+            setuprequest = SetupRequest(request, setupcall)
+            kwargs = {}
+            for name in setupcall.funcargnames:
+                if name == "request":
+                    kwargs[name] = setuprequest
+                else:
+                    kwargs[name] = request.getfuncargvalue(name)
+            scope = setupcall.scope or "function"
+            scol = setupcall.scopeitem = request._getscopeitem(scope)
+            self.session._setupstate.addfinalizer(setupcall.finish, scol)
+            for argname in setupcall.funcargnames: # XXX all deps?
+                self.addargfinalizer(setupcall.finish, argname)
+            setupcall.execute(kwargs)
 
-class FactoryDef:
+    def addargfinalizer(self, finalizer, argname):
-    """ A container for a factory definition. """
+        l = self._arg2finish.setdefault(argname, [])
-    def __init__(self, funcargmanager, baseid, argname, func, scope, params):
+        l.append(finalizer)
-        self.funcargmanager = funcargmanager
+
-        self.baseid = baseid
+    def removefinalizer(self, finalizer):
-        self.func = func
+        for l in self._arg2finish.values():
-        self.argname = argname
+            try:
-        self.scope = scope
+                l.remove(finalizer)
-        self.params = params
+            except ValueError:
-        self.funcargnames = getfuncargnames(func)
+                pass
+
+def rprop(attr, doc=None):
+    if doc is None:
+        doc = "%r of underlying test item"
+    return property(lambda x: getattr(x._request, attr), doc=doc)
+
+class SetupRequest:
+    def __init__(self, request, setupcall):
+        self._request = request
+        self._setupcall = setupcall
+        self._finalizers = []
+
+    # no getfuncargvalue(), cached_setup, applymarker helpers here
+    # on purpose
+
+    function = rprop("function")
+    cls = rprop("cls")
+    instance = rprop("instance")
+    fspath = rprop("fspath")
+    keywords = rprop("keywords")
+    config = rprop("config", "pytest config object.")
+
+    def addfinalizer(self, finalizer):
+        self._setupcall.addfinalizer(finalizer)
 
 class SetupCall:
     """ a container/helper for managing calls to setup functions. """
@@ -601,20 +632,32 @@ class SetupCall:
         self._finalizer = []
 
     def execute(self, kwargs):
-        #assert not self.active
+        assert not self.active
         self.active = True
-        mp = monkeypatch()
+        self.func(**kwargs)
-        #if "request" in kwargs:
-        #    request = kwargs["request"]
-        #    def addfinalizer(func):
-        #        #scopeitem = request._getscopeitem(scope)
-        #        self._finalizer.append(func)
-        #    mp.setattr(request, "addfinalizer", addfinalizer)
-        try:
-            self.func(**kwargs)
-        finally:
-            mp.undo()
 
+    def addfinalizer(self, finalizer):
+        assert self.active
+        self._finalizer.append(finalizer)
+
+    def finish(self):
+        while self._finalizer:
+            func = self._finalizer.pop()
+            func()
+        # check neccesity of next commented call
+        self.funcargmanager.removefinalizer(self.finish)
+        self.active = False
+
+class FactoryDef:
+    """ A container for a factory definition. """
+    def __init__(self, funcargmanager, baseid, argname, func, scope, params):
+        self.funcargmanager = funcargmanager
+        self.baseid = baseid
+        self.func = func
+        self.argname = argname
+        self.scope = scope
+        self.params = params
+        self.funcargnames = getfuncargnames(func)
 
 class NoMatch(Exception):
     """ raised if matching cannot locate a matching names. """
@@ -899,3 +942,75 @@ def readscope(func, markattr):
     marker = getattr(func, markattr, None)
     if marker is not None:
         return marker.kwargs.get("scope")
+
+# algorithm for sorting on a per-parametrized resource setup basis
+
+def parametrize_sorted(items, ignore, cache, scopenum):
+    if scopenum >= 3:
+        return items
+    newitems = []
+    olditems = []
+    slicing_argparam = None
+    for item in items:
+        argparamlist = getfuncargparams(item, ignore, scopenum, cache)
+        if slicing_argparam is None and argparamlist:
+            slicing_argparam = argparamlist[0]
+            slicing_index = len(olditems)
+        if slicing_argparam in argparamlist:
+            newitems.append(item)
+        else:
+            olditems.append(item)
+    if newitems:
+        newignore = ignore.copy()
+        newignore.add(slicing_argparam)
+        newitems = parametrize_sorted(newitems + olditems[slicing_index:],
+                                      newignore, cache, scopenum)
+        old1 = parametrize_sorted(olditems[:slicing_index], newignore,
+                                  cache, scopenum+1)
+        return old1 + newitems
+    else:
+        olditems = parametrize_sorted(olditems, ignore, cache, scopenum+1)
+    return olditems + newitems
+
+def getfuncargparams(item, ignore, scopenum, cache):
+    """ return list of (arg,param) tuple, sorted by broader scope first. """
+    assert scopenum < 3  # function
+    try:
+        cs = item.callspec
+    except AttributeError:
+        return []
+    if scopenum == 0:
+        argparams = [x for x in cs.params.items() if x not in ignore
+                        and cs._arg2scopenum[x[0]] == scopenum]
+    elif scopenum == 1:  # module
+        argparams = []
+        for argname, param in cs.params.items():
+            if cs._arg2scopenum[argname] == scopenum:
+                key = (argname, param, item.fspath)
+                if key in ignore:
+                    continue
+                argparams.append(key)
+    elif scopenum == 2:  # class
+        argparams = []
+        for argname, param in cs.params.items():
+            if cs._arg2scopenum[argname] == scopenum:
+                l = cache.setdefault(item.fspath, [])
+                try:
+                    i = l.index(item.cls)
+                except ValueError:
+                    i = len(l)
+                    l.append(item.cls)
+                key = (argname, param, item.fspath, i)
+                if key in ignore:
+                    continue
+                argparams.append(key)
+    #elif scopenum == 3:
+    #    argparams = []
+    #    for argname, param in cs.params.items():
+    #        if cs._arg2scopenum[argname] == scopenum:
+    #            key = (argname, param, getfslineno(item.obj))
+    #            if key in ignore:
+    #                continue
+    #            argparams.append(key)
+    return argparams
+

_pytest/python.py

@@ -1001,26 +1001,7 @@ class FuncargRequest:
 
 
     def _callsetup(self):
-        setuplist, allnames = self.funcargmanager.getsetuplist(
+        self.funcargmanager.ensure_setupcalls(self)
-                                                    self._pyfuncitem.nodeid)
-        mp = monkeypatch()
-        for setupcall in setuplist:
-            kwargs = {}
-            for name in setupcall.funcargnames:
-                if name == "request":
-                    kwargs[name] = self
-                else:
-                    kwargs[name] = self.getfuncargvalue(name)
-
-            mp.setattr(self, 'scope', setupcall.scope)
-            try:
-                if setupcall.scope is None:
-                    setupcall.execute(kwargs)
-                else:
-                    self.cached_setup(lambda: setupcall.execute(kwargs),
-                                      scope=setupcall.scope)
-            finally:
-                mp.undo()
 
     def getfuncargvalue(self, argname):
         """ Retrieve a function argument by name for this test

doc/en/example/newexamples.txt

@@ -48,7 +48,7 @@ If you run the tests::
     ================================= FAILURES =================================
     ________________________________ test_ehlo _________________________________
     
-    smtp = <smtplib.SMTP instance at 0x2c0f170>
+    smtp = <smtplib.SMTP instance at 0x2b8ebd8>
     
         def test_ehlo(smtp):
             response = smtp.ehlo()
@@ -60,7 +60,7 @@ If you run the tests::
     test_module.py:5: AssertionError
     ________________________________ test_noop _________________________________
     
-    smtp = <smtplib.SMTP instance at 0x2c0f170>
+    smtp = <smtplib.SMTP instance at 0x2b8ebd8>
     
         def test_noop(smtp):
             response = smtp.noop()
@@ -69,7 +69,7 @@ If you run the tests::
     E       assert 0
     
     test_module.py:10: AssertionError
-    2 failed in 0.21 seconds
+    2 failed in 0.26 seconds
 
 you will see the two ``assert 0`` failing and can see that
 the same (session-scoped) object was passed into the two test functions.
@@ -98,31 +98,9 @@ another run::
     collecting ... collected 4 items
     FFFF
     ================================= FAILURES =================================
-    ________________________ test_ehlo[mail.python.org] ________________________
-    
-    smtp = <smtplib.SMTP instance at 0x20fca70>
-    
-        def test_ehlo(smtp):
-            response = smtp.ehlo()
-            assert response[0] == 250
-    >       assert "merlinux" in response[1]
-    E       assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
-    
-    test_module.py:4: AssertionError
-    ________________________ test_noop[mail.python.org] ________________________
-    
-    smtp = <smtplib.SMTP instance at 0x20fca70>
-    
-        def test_noop(smtp):
-            response = smtp.noop()
-            assert response[0] == 250
-    >       assert 0  # for demo purposes
-    E       assert 0
-    
-    test_module.py:10: AssertionError
     __________________________ test_ehlo[merlinux.eu] __________________________
     
-    smtp = <smtplib.SMTP instance at 0x2104bd8>
+    smtp = <smtplib.SMTP instance at 0x2ee5200>
     
         def test_ehlo(smtp):
             response = smtp.ehlo()
@@ -134,7 +112,7 @@ another run::
     test_module.py:5: AssertionError
     __________________________ test_noop[merlinux.eu] __________________________
     
-    smtp = <smtplib.SMTP instance at 0x2104bd8>
+    smtp = <smtplib.SMTP instance at 0x2ee5200>
     
         def test_noop(smtp):
             response = smtp.noop()
@@ -143,7 +121,29 @@ another run::
     E       assert 0
     
     test_module.py:10: AssertionError
-    4 failed in 6.48 seconds
+    ________________________ test_ehlo[mail.python.org] ________________________
+    
+    smtp = <smtplib.SMTP instance at 0x2eee5a8>
+    
+        def test_ehlo(smtp):
+            response = smtp.ehlo()
+            assert response[0] == 250
+    >       assert "merlinux" in response[1]
+    E       assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
+    
+    test_module.py:4: AssertionError
+    ________________________ test_noop[mail.python.org] ________________________
+    
+    smtp = <smtplib.SMTP instance at 0x2eee5a8>
+    
+        def test_noop(smtp):
+            response = smtp.noop()
+            assert response[0] == 250
+    >       assert 0  # for demo purposes
+    E       assert 0
+    
+    test_module.py:10: AssertionError
+    4 failed in 6.94 seconds
 
 We get four failures because we are running the two tests twice with
 different ``smtp`` instantiations as defined on the factory.
@@ -159,10 +159,10 @@ You can look at what tests pytest collects without running them::
     plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
     collecting ... collected 4 items
     <Module 'test_module.py'>
-      <Function 'test_ehlo[mail.python.org]'>
-      <Function 'test_noop[mail.python.org]'>
       <Function 'test_ehlo[merlinux.eu]'>
       <Function 'test_noop[merlinux.eu]'>
+      <Function 'test_ehlo[mail.python.org]'>
+      <Function 'test_noop[mail.python.org]'>
     
     =============================  in 0.02 seconds =============================
 
@@ -172,13 +172,13 @@ And you can run without output capturing and minimized failure reporting to chec
     collecting ... collected 4 items
     FFFF
     ================================= FAILURES =================================
-    /home/hpk/tmp/doc-exec-386/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
+    /home/hpk/tmp/doc-exec-389/test_module.py:5: assert 0
-    /home/hpk/tmp/doc-exec-386/test_module.py:10: assert 0
+    /home/hpk/tmp/doc-exec-389/test_module.py:10: assert 0
-    /home/hpk/tmp/doc-exec-386/test_module.py:5: assert 0
+    /home/hpk/tmp/doc-exec-389/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
-    /home/hpk/tmp/doc-exec-386/test_module.py:10: assert 0
+    /home/hpk/tmp/doc-exec-389/test_module.py:10: assert 0
-    4 failed in 6.45 seconds
+    4 failed in 9.99 seconds
-    closing <smtplib.SMTP instance at 0x29d0a28>
+    closing <smtplib.SMTP instance at 0x2a61560>
-    closing <smtplib.SMTP instance at 0x29d8878>
+    closing <smtplib.SMTP instance at 0x2a6b248>
 
 .. _`new_setup`:
 
@@ -191,8 +191,11 @@ And you can run without output capturing and minimized failure reporting to chec
 
 The ``@pytest.mark.setup`` marker allows 
 
+* to define setup-functions close to test code or in conftest.py files
+  or plugins.
 * to mark a function as a setup/fixture method; the function can itself
-  receive funcargs
+  receive funcargs and will execute multiple times if the funcargs
+  are parametrized
 * to set a scope which determines the level of caching and how often
   the setup function is going to be called.
 
@@ -234,12 +237,12 @@ Let's run this module::
     $ py.test -qs
     collecting ... collected 2 items
     ..
-    2 passed in 0.24 seconds
+    2 passed in 0.62 seconds
-    created resource /home/hpk/tmp/pytest-3875/test_10
+    created resource /home/hpk/tmp/pytest-4224/test_10
-    setupresource /home/hpk/tmp/pytest-3875/test_10
+    setupresource /home/hpk/tmp/pytest-4224/test_10
-    using myresource /home/hpk/tmp/pytest-3875/test_10
+    using myresource /home/hpk/tmp/pytest-4224/test_10
-    using myresource /home/hpk/tmp/pytest-3875/test_10
+    using myresource /home/hpk/tmp/pytest-4224/test_10
-    finalize /home/hpk/tmp/pytest-3875/test_10
+    finalize /home/hpk/tmp/pytest-4224/test_10
 
 The two test functions will see the same resource instance because it has
 a module life cycle or scope.  
@@ -265,15 +268,16 @@ Running this will run four tests::
     collecting ... collected 4 items
     ....
     4 passed in 0.25 seconds
-    created resource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
+    created resource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa
-    setupresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
+    setupresource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa
-    using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
+    using myresource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa
-    using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
+    using myresource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa
-    finalize /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
+    finalize /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa
-    created resource /home/hpk/tmp/pytest-3876/test_1_bbb_0/bbb
+    created resource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb
-    using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
+    setupresource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb
-    using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
+    using myresource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb
-    finalize /home/hpk/tmp/pytest-3876/test_1_bbb_0/bbb
+    using myresource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb
+    finalize /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb
 
 Each parameter causes the creation of a respective resource and the
 unchanged test module uses it in its ``@setup`` decorated method.

doc/en/resources.txt

@@ -51,7 +51,7 @@ implementation or backward compatibility issues.  The main changes are:
   troubles than the current @setup approach which can share
   a lot of logic with the @funcarg one.
 
-* tests are grouped by any parametrized resource
+* tests are grouped by parametrized funcargs
 
 .. currentmodule:: _pytest
 
@@ -259,13 +259,13 @@ Grouping tests by resource parameters
 
 .. note:: Implemented.
 
-pytest usually sorts test items by their source location.
+pytest used to always sort test items by their source location.
-With pytest-2.X tests are first grouped by resource parameters.
+With pytest-2.X tests are first grouped by funcarg parameters.
-If you have a parametrized resource, then all the tests using it
+If you have a parametrized funcarg, then all the tests using it
 will first execute with it.  Then any finalizers are called and then
 the next parametrized resource instance is created and its tests are run.
-Among other things, this allows to have per-session parametrized setups
+Among other things, this eases testing of applications which create 
-including ones which affect global state of an application.
+and use global state.
 
 The following example uses two parametrized funcargs, one of which is 
 scoped on a per-module basis::
@@ -293,12 +293,12 @@ scoped on a per-module basis::
     def test_2(otherarg, modarg):
         print "  test2", otherarg, modarg
 
-If you run the tests in verbose mode and with looking at captured output::
+Let's run the tests in verbose mode and with looking at the print-output::
 
     $ py.test -v -s
     =========================== test session starts ============================
     platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev5 -- /home/hpk/venv/1/bin/python
-    cachedir: /home/hpk/tmp/doc-exec-382/.cache
+    cachedir: /home/hpk/tmp/doc-exec-388/.cache
     plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
     collecting ... collected 8 items
     
@@ -326,9 +326,8 @@ If you run the tests in verbose mode and with looking at captured output::
     fin mod2
 
 You can see that that the parametrized ``modarg`` resource lead to
-a re-ordering of test execution. Moreover, the finalizer for the
+a re-ordering of test execution. The finalizer for the "mod1" parametrized 
-"mod1" parametrized resource was executed before the "mod2" resource 
+resource was executed before the "mod2" resource was setup.
-was setup with a different parameter.
 
 .. note::
 

setup.py

@@ -24,7 +24,7 @@ def main():
         name='pytest',
         description='py.test: simple powerful testing with Python',
         long_description = long_description,
-        version='2.3.0.dev5',
+        version='2.3.0.dev6',
         url='http://pytest.org',
         license='MIT license',
         platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'],

testing/test_python.py

@@ -1559,33 +1559,8 @@ def test_issue117_sessionscopeteardown(testdir):
     ])
 
 class TestRequestAPI:
-    @pytest.mark.xfail(reason="reverted refactoring")
+    @pytest.mark.xfail(reason="consider flub feedback")
-    def test_addfinalizer_cachedsetup_getfuncargvalue(self, testdir):
+    def test_setup_can_query_funcargs(self, testdir):
-        testdir.makeconftest("""
-            l = []
-            def pytest_runtest_setup(item):
-                item.addfinalizer(lambda: l.append(1))
-                l2 = item.getfuncargvalue("l")
-                assert l2 is l
-                item.cached_setup(lambda: l.append(2), lambda val: l.append(3),
-                                  scope="function")
-            def pytest_funcarg__l(request):
-                return l
-        """)
-        testdir.makepyfile("""
-            def test_hello():
-                pass
-            def test_hello2(l):
-                assert l == [2, 3, 1, 2]
-        """)
-        result = testdir.runpytest()
-        assert result.ret == 0
-        result.stdout.fnmatch_lines([
-            "*2 passed*",
-        ])
-
-    @pytest.mark.xfail(reason="consider item's funcarg access and error conditions")
-    def test_runtest_setup_sees_filled_funcargs(self, testdir):
         testdir.makeconftest("""
             def pytest_runtest_setup(item):
                 assert not hasattr(item, "_request")
@@ -1606,9 +1581,9 @@ class TestRequestAPI:
 
         result = testdir.makeconftest("""
             import pytest
-            @pytest.mark.trylast
+            @pytest.mark.setup
-            def pytest_runtest_setup(item):
+            def mysetup(testcontext):
-                assert item.funcargs == {"a": 1, "b": 2}
+                testcontext.uses_funcarg("db")
         """)
         result = testdir.runpytest()
         assert result.ret == 0
@@ -1737,7 +1712,7 @@ class TestFuncargManager:
                     return "class"
                 def test_hello(self, item, fm):
                     faclist = fm.getfactorylist("hello", item.nodeid, item.obj)
-                    print faclist
+                    print (faclist)
                     assert len(faclist) == 3
                     assert faclist[0].func(item._request) == "conftest"
                     assert faclist[1].func(item._request) == "module"
@@ -1863,6 +1838,43 @@ class TestSetupManagement:
         reprec = testdir.inline_run("-v", "-s")
         reprec.assertoutcome(passed=4)
 
+    def test_class_function_parametrization_finalization(self, testdir):
+        p = testdir.makeconftest("""
+            import pytest
+            import pprint
+
+            l = []
+
+            @pytest.mark.funcarg(scope="function", params=[1,2])
+            def farg(request):
+                return request.param
+
+            @pytest.mark.funcarg(scope="class", params=list("ab"))
+            def carg(request):
+                return request.param
+
+            @pytest.mark.setup(scope="class")
+            def append(request, farg, carg):
+                def fin():
+                    l.append("fin_%s%s" % (carg, farg))
+                request.addfinalizer(fin)
+        """)
+        testdir.makepyfile("""
+            import pytest
+
+            class TestClass:
+                def test_1(self):
+                    pass
+            class TestClass2:
+                def test_2(self):
+                    pass
+        """)
+        reprec = testdir.inline_run("-v",)
+        reprec.assertoutcome(passed=8)
+        config = reprec.getcalls("pytest_unconfigure")[0].config
+        l = config._conftest.getconftestmodules(p)[0].l
+        assert l == ["fin_a1", "fin_a2", "fin_b1", "fin_b2"] * 2
+
 class TestFuncargMarker:
     def test_parametrize(self, testdir):
         testdir.makepyfile("""
@@ -2016,11 +2028,14 @@ class TestFuncargMarker:
             l = []
             def test_param(arg):
                 l.append(arg)
-            def test_result():
-                assert l == list("abc")
         """)
-        reprec = testdir.inline_run()
+        reprec = testdir.inline_run("-v")
-        reprec.assertoutcome(passed=4)
+        reprec.assertoutcome(passed=3)
+        l = reprec.getcalls("pytest_runtest_call")[0].item.module.l
+        assert len(l) == 3
+        assert "a" in l
+        assert "b" in l
+        assert "c" in l
 
     def test_scope_mismatch(self, testdir):
         testdir.makeconftest("""
@@ -2056,14 +2071,105 @@ class TestFuncargMarker:
                 l.append(arg)
             def test_2(arg):
                 l.append(arg)
-            def test_3():
-                assert len(l) == 4
-                assert l[0] == l[1]
-                assert l[2] == l[3]
-
         """)
         reprec = testdir.inline_run("-v")
-        reprec.assertoutcome(passed=5)
+        reprec.assertoutcome(passed=4)
+        l = reprec.getcalls("pytest_runtest_call")[0].item.module.l
+        assert l == [1,1,2,2]
+
+    def test_module_parametrized_ordering(self, testdir):
+        testdir.makeconftest("""
+            import pytest
+
+            @pytest.mark.funcarg(scope="session", params="s1 s2".split())
+            def sarg(request):
+                pass
+            @pytest.mark.funcarg(scope="module", params="m1 m2".split())
+            def marg(request):
+                pass
+        """)
+        testdir.makepyfile(test_mod1="""
+            def test_func(sarg):
+                pass
+            def test_func1(marg):
+                pass
+        """, test_mod2="""
+            def test_func2(sarg):
+                pass
+            def test_func3(sarg, marg):
+                pass
+            def test_func3b(sarg, marg):
+                pass
+            def test_func4(marg):
+                pass
+        """)
+        result = testdir.runpytest("-v")
+        result.stdout.fnmatch_lines("""
+            test_mod1.py:1: test_func[s1] PASSED
+            test_mod2.py:1: test_func2[s1] PASSED
+            test_mod2.py:3: test_func3[s1-m1] PASSED
+            test_mod2.py:5: test_func3b[s1-m1] PASSED
+            test_mod2.py:3: test_func3[s1-m2] PASSED
+            test_mod2.py:5: test_func3b[s1-m2] PASSED
+            test_mod1.py:1: test_func[s2] PASSED
+            test_mod2.py:1: test_func2[s2] PASSED
+            test_mod2.py:3: test_func3[s2-m1] PASSED
+            test_mod2.py:5: test_func3b[s2-m1] PASSED
+            test_mod2.py:7: test_func4[m1] PASSED
+            test_mod2.py:3: test_func3[s2-m2] PASSED
+            test_mod2.py:5: test_func3b[s2-m2] PASSED
+            test_mod2.py:7: test_func4[m2] PASSED
+            test_mod1.py:3: test_func1[m1] PASSED
+            test_mod1.py:3: test_func1[m2] PASSED
+        """)
+
+    def test_class_ordering(self, testdir):
+        p = testdir.makeconftest("""
+            import pytest
+
+            l = []
+
+            @pytest.mark.funcarg(scope="function", params=[1,2])
+            def farg(request):
+                return request.param
+
+            @pytest.mark.funcarg(scope="class", params=list("ab"))
+            def carg(request):
+                return request.param
+
+            @pytest.mark.setup(scope="class")
+            def append(request, farg, carg):
+                def fin():
+                    l.append("fin_%s%s" % (carg, farg))
+                request.addfinalizer(fin)
+        """)
+        testdir.makepyfile("""
+            import pytest
+
+            class TestClass2:
+                def test_1(self):
+                    pass
+                def test_2(self):
+                    pass
+            class TestClass:
+                def test_3(self):
+                    pass
+        """)
+        result = testdir.runpytest("-v")
+        result.stdout.fnmatch_lines("""
+            test_class_ordering.py:4: TestClass2.test_1[1-a] PASSED
+            test_class_ordering.py:4: TestClass2.test_1[2-a] PASSED
+            test_class_ordering.py:6: TestClass2.test_2[1-a] PASSED
+            test_class_ordering.py:6: TestClass2.test_2[2-a] PASSED
+            test_class_ordering.py:4: TestClass2.test_1[1-b] PASSED
+            test_class_ordering.py:4: TestClass2.test_1[2-b] PASSED
+            test_class_ordering.py:6: TestClass2.test_2[1-b] PASSED
+            test_class_ordering.py:6: TestClass2.test_2[2-b] PASSED
+            test_class_ordering.py:9: TestClass.test_3[1-a] PASSED
+            test_class_ordering.py:9: TestClass.test_3[2-a] PASSED
+            test_class_ordering.py:9: TestClass.test_3[1-b] PASSED
+            test_class_ordering.py:9: TestClass.test_3[2-b] PASSED
+        """)
 
     def test_parametrize_separated_order_higher_scope_first(self, testdir):
         testdir.makepyfile("""
@@ -2097,17 +2203,14 @@ class TestFuncargMarker:
                 import pprint
                 pprint.pprint(l)
                 assert l == [
-                    'create:1', 'test1', 'fin:1',
+                    'create:1', 'test1', 'fin:1', 'create:2', 'test1',
-                    'create:2', 'test1', 'fin:2',
+                    'fin:2', 'create:mod1', 'test2', 'create:1', 'test3',
-                    'create:mod1', 'test2', 'create:1', 'test3', 'fin:1',
+                    'fin:1', 'create:2', 'test3', 'fin:2', 'create:1',
-                    'create:1', 'test4', 'fin:1', 'create:2', 'test3', 'fin:2',
+                    'test4', 'fin:1', 'create:2', 'test4', 'fin:mod1',
-                    'create:2', 'test4', 'fin:mod1', 'fin:2',
+                    'fin:2', 'create:mod2', 'test2', 'create:1', 'test3',
-
+                    'fin:1', 'create:2', 'test3', 'fin:2', 'create:1',
-                    'create:mod2', 'test2', 'create:1', 'test3', 'fin:1',
+                    'test4', 'fin:1', 'create:2', 'test4', 'fin:mod2',
-                    'create:1', 'test4', 'fin:1', 'create:2', 'test3', 'fin:2',
+                'fin:2']
-                    'create:2', 'test4', 'fin:mod2', 'fin:2',
-                ]
-
         """)
         reprec = testdir.inline_run("-v")
         reprec.assertoutcome(passed=12+1)
@@ -2118,6 +2221,7 @@ class TestFuncargMarker:
 
             @pytest.mark.funcarg(scope="module", params=[1, 2])
             def arg(request):
+                request.config.l = l # to access from outer
                 x = request.param
                 request.addfinalizer(lambda: l.append("fin%s" % x))
                 return request.param
@@ -2127,16 +2231,18 @@ class TestFuncargMarker:
                 l.append(arg)
             def test_2(arg):
                 l.append(arg)
-            def test_3():
-                assert len(l) == 6
-                assert l[0] == l[1]
-                assert l[2] == "fin1"
-                assert l[3] == l[4]
-                assert l[5] == "fin2"
-
         """)
         reprec = testdir.inline_run("-v")
-        reprec.assertoutcome(passed=5)
+        reprec.assertoutcome(passed=4)
+        l = reprec.getcalls("pytest_configure")[0].config.l
+        import pprint
+        pprint.pprint(l)
+        assert len(l) == 6
+        assert l[0] == l[1] == 1
+        assert l[2] == "fin1"
+        assert l[3] == l[4] == 2
+        assert l[5] == "fin2"
+
 
     def test_parametrize_function_scoped_finalizers_called(self, testdir):
         testdir.makepyfile("""
@@ -2155,7 +2261,7 @@ class TestFuncargMarker:
                 l.append(arg)
             def test_3():
                 assert len(l) == 8
-                assert l == [1, "fin1", 1, "fin1", 2, "fin2", 2, "fin2"]
+                assert l == [1, "fin1", 2, "fin2", 1, "fin1", 2, "fin2"]
         """)
         reprec = testdir.inline_run("-v")
         reprec.assertoutcome(passed=5)
@@ -2181,10 +2287,13 @@ class TestFuncargMarker:
             def test_3():
                 import pprint
                 pprint.pprint(l)
-                assert l == ["setup1", 1, 1, "fin1",
+                if arg == 1:
-                             "setup2", 2, 2, "fin2",]
+                    assert l == ["setup1", 1, 1, ]
+                elif arg == 2:
+                    assert l == ["setup1", 1, 1, "fin1",
+                                 "setup2", 2, 2, ]
 
         """)
         reprec = testdir.inline_run("-v")
-        reprec.assertoutcome(passed=5)
+        reprec.assertoutcome(passed=6)