move funcarg factory to a new FuncargManager object at session level

_pytest/capture.py

@@ -188,7 +188,7 @@ def pytest_funcarg__capsys(request):
     which return a ``(out, err)`` tuple.
     """
     if "capfd" in request._funcargs:
-        raise request.LookupError(error_capsysfderror)
+        raise request.raiseerror(error_capsysfderror)
     return CaptureFuncarg(py.io.StdCapture)
 
 def pytest_funcarg__capfd(request):
@@ -197,7 +197,7 @@ def pytest_funcarg__capfd(request):
     which return a ``(out, err)`` tuple.
     """
     if "capsys" in request._funcargs:
-        raise request.LookupError(error_capsysfderror)
+        request.raiseerror(error_capsysfderror)
     if not hasattr(os, 'dup'):
         pytest.skip("capfd funcarg needs os.dup")
     return CaptureFuncarg(py.io.StdCaptureFD)

_pytest/impl

@@ -3,84 +3,32 @@ Implementation plan for resources
 ------------------------------------------
 
 1. Revert FuncargRequest to the old form, unmerge item/request
-2. make setup functions be discovered at collection time
+   (done)
-3. make funcarg factories be discovered at collection time
+2. make funcarg factories be discovered at collection time
-4. Introduce funcarg marker
+3. Introduce funcarg marker
-5. Introduce funcarg scope parameter
+4. Introduce funcarg scope parameter
-6. Introduce funcarg parametrize parameter
+5. Introduce funcarg parametrize parameter
+6. make setup functions be discovered at collection time
 7. (Introduce a pytest_fixture_protocol/setup_funcargs hook)
 
 methods and data structures
 --------------------------------
 
-A FuncarcDB holds all information about funcarg definitions,
+A FuncarcManager holds all information about funcarg definitions
-parametrization and the places where funcargs are required.  It can
+including parametrization and scope definitions.  It implements
-answer the following questions:
+a pytest_generate_tests hook which performs parametrization as appropriate.
-
-* given a node and a funcargname, return a paramlist so that collection 
-  can perform parametrization (parametrized nodes?)
-* given a node (possibly containing a param), perform a funcargrequest 
-  and return the value
-* if funcargname is an empty string, it matches general setup.
-
-pytest could perform 2-pass collection:
-- first perform normal collection (no parametrization at all!), populate
-  FuncargDB
-- walk through the node tree and ask FuncargDB for each node for
-  required funcargs and their parameters - clone subtrees (deepcopy) and
-  substitute the un-parametrized node with parametrized ones
 
 as a simple example, let's consider a tree where a test function requires
 a "abc" funcarg and its factory defines it as parametrized and scoped
-for Modules.  When the 2nd collection pass asks FuncargDB to return
+for Modules.  When collections hits the function item, it creates
-params for the test module, it will know that the test functions in it
+the metafunc object, and calls funcargdb.pytest_generate_tests(metafunc)
-requires "abc" and that is it parametrized and defined for module scope.
+which looks up available funcarg factories and their scope and parametrization.
-Therefore parametrization of the module node is performed, substituting
+This information is equivalent to what can be provided today directly
-the node with multiple module nodes ("test_module.py[1]", ...).
+at the function site and it should thus be relatively straight forward
-When test_module.py[1] is setup() it will call all its (parametrized)
+to implement the additional way of defining parametrization/scoping.
-factories and populate a funcargs dictionary, mapping funcargnames to values.
-When a test function below test_module.py[1] is executed, it looks up
-its required arguments from the thus populated funcargs dictionary.
-
-Let's add to this example a second funcarg "def" that has a per-function parametrization.  When the 2nd collection pass asks FuncargDB to return
-params for the test function, it will know that the test functions in it
-requires "def" and that is it parametrized and defined for function scope.
-Therefore parametrization of the function node is performed, substituting
-the node with multiple function nodes ("test_function[1]", ...).
-
-When test_function[1] is setup() it will call all its (parametrized)
-factories and populate a funcargs dictionary.  The "def" will only appear
-in the funcargs dict seen by test_function[1]. When test_function[1] 
-executes, it will use its funcargs.
-
-
-
-
-where
-
-* ``nodeidbase`` is a basestring; for all nodeids matching
-  startswith(nodeidbase) it defines a (scopecls, factorylist) tuple
-* ``scopecls`` is a node class for the which the factorylist s defined
-* ``param`` is a parametrizing parameter for the factorylist
-* ``factorylist`` is a list of factories which will be used to perform
-  a funcarg request
-* the whole list is sorted by length of nodeidbase (longest first)
 
 conftest loading:
-    each funcarg-factory will populate FuncargDefs which keeps references
+    each funcarg-factory will populate the session.funcargmanager
-    to all definitions the funcarg2 marked function or pytest_funcarg__ 
-
-
-scope can be a string or a nodenames-tuple. 
-
-        scopestring -> list of (funcargname, factorylist)
-
-        nodenames -> (funcargname, list of factories)
-
-It needs to be a list because factories can decorate
-
-For any given node and a required funcarg it is thus
-easy to lookup a list of matching factories.
 
 When a test item is collected, it grows a dictionary 
 (funcargname2factorycalllist).  A factory lookup is performed 

_pytest/main.py

@@ -2,8 +2,12 @@
 
 import py
 import pytest, _pytest
+import inspect
 import os, sys, imp
 
+from _pytest.monkeypatch import monkeypatch
+from py._code.code import TerminalRepr
+
 tracebackcutdir = py.path.local(_pytest.__file__).dirpath()
 
 # exitcodes for the command line
@@ -279,6 +283,15 @@ class Node(object):
         pass
 
     def _repr_failure_py(self, excinfo, style=None):
+        LE = self.session.funcargmanager.FuncargLookupError
+        if excinfo.errisinstance(LE):
+            request = excinfo.value.request
+            fspath, lineno, msg = request._pyfuncitem.reportinfo()
+            lines, _ = inspect.getsourcelines(request.function)
+            for i, line in enumerate(lines):
+                if line.strip().startswith('def'):
+                    return FuncargLookupErrorRepr(fspath, lineno, lines[:i+1],
+                                str(excinfo.value.msg))
         if self.config.option.fulltrace:
             style="long"
         else:
@@ -391,6 +404,75 @@ class Item(Node):
             self._location = location
             return location
 
+class FuncargLookupError(LookupError):
+    """ could not find a factory. """
+    def __init__(self, request, msg):
+        self.request = request
+        self.msg = msg
+
+class FuncargManager:
+    _argprefix = "pytest_funcarg__"
+    FuncargLookupError = FuncargLookupError
+
+    def __init__(self, session):
+        self.session = session
+        self.config = session.config
+        self.node2name2factory = {}
+
+    def _discoverfactories(self, request, argname):
+        node = request._pyfuncitem
+        name2factory = self.node2name2factory.setdefault(node, {})
+        if argname not in name2factory:
+            name2factory[argname] = self.config.pluginmanager.listattr(
+                    plugins=request._plugins,
+                    attrname=self._argprefix + str(argname)
+            )
+        #else: we are called recursively
+        if not name2factory[argname]:
+            self._raiselookupfailed(request, argname)
+
+    def _getfuncarg(self, request, argname):
+        node = request._pyfuncitem
+        try:
+            factorylist = self.node2name2factory[node][argname]
+        except KeyError:
+            # XXX at collection time this funcarg was not know to be a
+            # requirement, would be better if it would be known
+            self._discoverfactories(request, argname)
+            factorylist = self.node2name2factory[node][argname]
+
+        if not factorylist:
+            self._raiselookupfailed(request, argname)
+        funcargfactory = factorylist.pop()
+        oldarg = request._currentarg
+        mp = monkeypatch()
+        mp.setattr(request, '_currentarg', argname)
+        try:
+            param = node.callspec.getparam(argname)
+        except (AttributeError, ValueError):
+            pass
+        else:
+            mp.setattr(request, 'param', param, raising=False)
+        try:
+            return funcargfactory(request=request)
+        finally:
+            mp.undo()
+
+    def _raiselookupfailed(self, request, argname):
+        available = []
+        for plugin in request._plugins:
+            for name in vars(plugin):
+                if name.startswith(self._argprefix):
+                    name = name[len(self._argprefix):]
+                    if name not in available:
+                        available.append(name)
+        fspath, lineno, msg = request._pyfuncitem.reportinfo()
+        msg = "LookupError: no factory found for argument %r" % (argname,)
+        msg += "\n available funcargs: %s" %(", ".join(available),)
+        msg += "\n use 'py.test --funcargs [testpath]' for help on them."
+        raise FuncargLookupError(request, msg)
+
+
 class NoMatch(Exception):
     """ raised if matching cannot locate a matching names. """
 
@@ -407,6 +489,7 @@ class Session(FSCollector):
         self.shouldstop = False
         self.trace = config.trace.root.get("collection")
         self._norecursepatterns = config.getini("norecursedirs")
+        self.funcargmanager = FuncargManager(self)
 
     def pytest_collectstart(self):
         if self.shouldstop:
@@ -634,4 +717,18 @@ class Session(FSCollector):
 
 
 
+class FuncargLookupErrorRepr(TerminalRepr):
+    def __init__(self, filename, firstlineno, deflines, errorstring):
+        self.deflines = deflines
+        self.errorstring = errorstring
+        self.filename = filename
+        self.firstlineno = firstlineno
 
+    def toterminal(self, tw):
+        tw.line()
+        for line in self.deflines:
+            tw.line("    " + line.strip())
+        for line in self.errorstring.split("\n"):
+            tw.line("        " + line.strip(), red=True)
+        tw.line()
+        tw.line("%s:%d" % (self.filename, self.firstlineno+1))

_pytest/python.py

@@ -3,8 +3,6 @@ import py
 import inspect
 import sys
 import pytest
-from py._code.code import TerminalRepr
-from _pytest.monkeypatch import monkeypatch
 
 import _pytest
 cutdir = py.path.local(_pytest.__file__).dirpath()
@@ -278,9 +276,9 @@ class PyCollector(PyobjMixin, pytest.Collector):
         plugins = self.getplugins() + extra
         gentesthook.pcall(plugins, metafunc=metafunc)
         Function = self._getcustomclass("Function")
-        if not metafunc._calls:
-            return Function(name, parent=self)
         l = []
+        if not metafunc._calls:
+            l.append(Function(name, parent=self))
         for callspec in metafunc._calls:
             subname = "%s[%s]" %(name, callspec.id)
             function = Function(name=subname, parent=self,
@@ -423,13 +421,6 @@ class FunctionMixin(PyobjMixin):
             excinfo.traceback = ntraceback.filter()
 
     def _repr_failure_py(self, excinfo, style="long"):
-        if excinfo.errisinstance(FuncargRequest.LookupError):
-            fspath, lineno, msg = self.reportinfo()
-            lines, _ = inspect.getsourcelines(self.obj)
-            for i, line in enumerate(lines):
-                if line.strip().startswith('def'):
-                    return FuncargLookupErrorRepr(fspath, lineno,
-            lines[:i+1], str(excinfo.value))
         if excinfo.errisinstance(pytest.fail.Exception):
             if not excinfo.value.pytrace:
                 return str(excinfo.value)
@@ -441,22 +432,6 @@ class FunctionMixin(PyobjMixin):
         return self._repr_failure_py(excinfo,
             style=self.config.option.tbstyle)
 
-class FuncargLookupErrorRepr(TerminalRepr):
-    def __init__(self, filename, firstlineno, deflines, errorstring):
-        self.deflines = deflines
-        self.errorstring = errorstring
-        self.filename = filename
-        self.firstlineno = firstlineno
-
-    def toterminal(self, tw):
-        tw.line()
-        for line in self.deflines:
-            tw.line("    " + line.strip())
-        for line in self.errorstring.split("\n"):
-            tw.line("        " + line.strip(), red=True)
-        tw.line()
-        tw.line("%s:%d" % (self.filename, self.firstlineno+1))
-
 
 class Generator(FunctionMixin, PyCollector):
     def collect(self):
@@ -523,23 +498,9 @@ def fillfuncargs(function):
         try:
             request = function._request
         except AttributeError:
-            request = FuncargRequest(function)
+            request = function._request = FuncargRequest(function)
         request._fillfuncargs()
 
-def XXXfillfuncargs(node):
-    """ fill missing funcargs. """
-    node = FuncargRequest(node)
-    if node.funcargs is None:
-        node.funcargs = getattr(node, "_funcargs", {})
-    if not isinstance(node, Function) or not node._isyieldedfunction():
-        try:
-            funcargnames = node.funcargnames
-        except AttributeError:
-            funcargnames = getfuncargnames(node.function)
-        if funcargnames:
-            for argname in funcargnames:
-                node.getfuncargvalue(argname)
-
 _notexists = object()
 
 class CallSpec2(object):
@@ -711,11 +672,12 @@ def _showfuncargs_main(config, session):
     curdir = py.path.local()
     tw = py.io.TerminalWriter()
     verbose = config.getvalue("verbose")
+    argprefix = session.funcargmanager._argprefix
     for plugin in plugins:
         available = []
         for name, factory in vars(plugin).items():
-            if name.startswith(FuncargRequest._argprefix):
+            if name.startswith(argprefix):
-                name = name[len(FuncargRequest._argprefix):]
+                name = name[len(argprefix):]
                 if name not in available:
                     available.append([name, factory])
         if available:
@@ -847,11 +809,11 @@ class Function(FunctionMixin, pytest.Item):
             else:
                 self.funcargs = {}
             self._request = req = FuncargRequest(self)
+            req._discoverfactories()
         if callobj is not _dummy:
             self.obj = callobj
         startindex = int(self.cls is not None)
         self.funcargnames = getfuncargnames(self.obj, startindex=startindex)
-
         self.keywords.update(py.builtin._getfuncdict(self.obj) or {})
         if keywords:
             self.keywords.update(keywords)
@@ -912,11 +874,6 @@ class FuncargRequest:
         If no such call was done in a ``pytest_generate_tests``
         hook, the attribute will not be present.
     """
-    _argprefix = "pytest_funcarg__"
-    _argname = None
-
-    class LookupError(LookupError):
-        """ error on performing funcarg request. """
 
     def __init__(self, pyfuncitem):
         self._pyfuncitem = pyfuncitem
@@ -925,13 +882,24 @@ class FuncargRequest:
         self.getparent = pyfuncitem.getparent
         self._funcargs  = self._pyfuncitem.funcargs.copy()
         self._name2factory = {}
+        self.funcargmanager = pyfuncitem.session.funcargmanager
         self._currentarg = None
+        self.funcargnames = getfuncargnames(self.function)
+
+    def _discoverfactories(self):
+        for argname in self.funcargnames:
+            if argname not in self._funcargs:
+                self.funcargmanager._discoverfactories(self, argname)
 
     @cached_property
     def _plugins(self):
         extra = [obj for obj in (self.module, self.instance) if obj]
         return self._pyfuncitem.getplugins() + extra
 
+    def raiseerror(self, msg):
+        """ raise a FuncargLookupError with the given message. """
+        raise self.funcargmanager.FuncargLookupError(self, msg)
+
     @property
     def function(self):
         """ function object of the test invocation. """
@@ -972,14 +940,13 @@ class FuncargRequest:
         return self._pyfuncitem.fspath
 
     def _fillfuncargs(self):
-        argnames = getfuncargnames(self.function)
+        if self.funcargnames:
-        if argnames:
             assert not getattr(self._pyfuncitem, '_args', None), (
                 "yielded functions cannot have funcargs")
-        for argname in argnames:
+        for argname in self.funcargnames:
             if argname not in self._pyfuncitem.funcargs:
-                self._pyfuncitem.funcargs[argname] = self.getfuncargvalue(argname)
+                self._pyfuncitem.funcargs[argname] = \
-
+                        self.getfuncargvalue(argname)
 
     def applymarker(self, marker):
         """ Apply a marker to a single test function invocation.
@@ -1021,6 +988,7 @@ class FuncargRequest:
                 self._addfinalizer(finalizer, scope=scope)
         return val
 
+
     def getfuncargvalue(self, argname):
         """ Retrieve a function argument by name for this test
         function invocation.  This allows one function argument factory
@@ -1033,29 +1001,9 @@ class FuncargRequest:
             return self._funcargs[argname]
         except KeyError:
             pass
-        if argname not in self._name2factory:
+        val = self.funcargmanager._getfuncarg(self, argname)
-            self._name2factory[argname] = self.config.pluginmanager.listattr(
+        self._funcargs[argname] = val
-                    plugins=self._plugins,
+        return val
-                    attrname=self._argprefix + str(argname)
-            )
-        #else: we are called recursively
-        if not self._name2factory[argname]:
-            self._raiselookupfailed(argname)
-        funcargfactory = self._name2factory[argname].pop()
-        oldarg = self._currentarg
-        mp = monkeypatch()
-        mp.setattr(self, '_currentarg', argname)
-        try:
-            param = self._pyfuncitem.callspec.getparam(argname)
-        except (AttributeError, ValueError):
-            pass
-        else:
-            mp.setattr(self, 'param', param, raising=False)
-        try:
-            self._funcargs[argname] = res = funcargfactory(request=self)
-        finally:
-            mp.undo()
-        return res
 
     def _getscopeitem(self, scope):
         if scope == "function":
@@ -1084,16 +1032,3 @@ class FuncargRequest:
     def __repr__(self):
         return "<FuncargRequest for %r>" %(self._pyfuncitem)
 
-    def _raiselookupfailed(self, argname):
-        available = []
-        for plugin in self._plugins:
-            for name in vars(plugin):
-                if name.startswith(self._argprefix):
-                    name = name[len(self._argprefix):]
-                    if name not in available:
-                        available.append(name)
-        fspath, lineno, msg = self._pyfuncitem.reportinfo()
-        msg = "LookupError: no factory found for function argument %r" % (argname,)
-        msg += "\n available funcargs: %s" %(", ".join(available),)
-        msg += "\n use 'py.test --funcargs [testpath]' for help on them."
-        raise self.LookupError(msg)

testing/test_mark.py

@@ -272,7 +272,7 @@ class TestFunctional:
             import pytest
             @pytest.mark.hello("pos1", z=4)
             @pytest.mark.hello("pos0", z=3)
-            def test_func(self):
+            def test_func():
                 pass
         """)
         items, rec = testdir.inline_genitems(p)

testing/test_python.py

@@ -1,5 +1,6 @@
 import pytest, py, sys
 from _pytest import python as funcargs
+from _pytest.main import FuncargLookupError
 
 class TestModule:
     def test_failing_import(self, testdir):
@@ -301,24 +302,14 @@ class TestFunction:
         assert not f1 != f1_b
 
     def test_function_equality_with_callspec(self, testdir, tmpdir):
-        config = testdir.parseconfigure()
+        items = testdir.getitems("""
-        class callspec1:
+            import pytest
-            param = 1
+            @pytest.mark.parametrize('arg', [1,2])
-            funcargs = {}
+            def test_function(arg):
-            id = "hello"
+                pass
-        class callspec2:
+        """)
-            param = 1
+        assert items[0] != items[1]
-            funcargs = {}
+        assert not (items[0] == items[1])
-            id = "world"
-        session = testdir.Session(config)
-        def func():
-            pass
-        f5 = pytest.Function(name="name", config=config,
-            callspec=callspec1, callobj=func, session=session)
-        f5b = pytest.Function(name="name", config=config,
-            callspec=callspec2, callobj=func, session=session)
-        assert f5 != f5b
-        assert not (f5 == f5b)
 
     def test_pyfunc_call(self, testdir):
         item = testdir.getitem("def test_func(): raise ValueError")
@@ -550,33 +541,30 @@ class TestFillFuncArgs:
         assert pytest._fillfuncargs == funcargs.fillfuncargs
 
     def test_funcarg_lookupfails(self, testdir):
-        testdir.makeconftest("""
+        testdir.makepyfile("""
             def pytest_funcarg__xyzsomething(request):
                 return 42
-        """)
-        item = testdir.getitem("def test_func(some): pass")
-        exc = pytest.raises(funcargs.FuncargRequest.LookupError,
-            "funcargs.fillfuncargs(item)")
-        s = str(exc.value)
-        assert s.find("xyzsomething") != -1
 
-    def test_funcarg_lookup_default(self, testdir):
+            def test_func(some):
-        item = testdir.getitem("def test_func(some, other=42): pass")
+                pass
-        class Provider:
+        """)
-            def pytest_funcarg__some(self, request):
+        result = testdir.runpytest() # "--collectonly")
-                return request.function.__name__
+        assert result.ret != 0
-        item.config.pluginmanager.register(Provider())
+        result.stdout.fnmatch_lines([
-        funcargs.fillfuncargs(item)
+            "*def test_func(some)*",
-        assert len(item.funcargs) == 1
+            "*LookupError*",
+            "*xyzsomething*",
+        ])
 
     def test_funcarg_basic(self, testdir):
-        item = testdir.getitem("def test_func(some, other): pass")
+        item = testdir.getitem("""
-        class Provider:
+            def pytest_funcarg__some(request):
-            def pytest_funcarg__some(self, request):
                 return request.function.__name__
-            def pytest_funcarg__other(self, request):
+            def pytest_funcarg__other(request):
                 return 42
-        item.config.pluginmanager.register(Provider())
+            def test_func(some, other):
+                pass
+        """)
         funcargs.fillfuncargs(item)
         assert len(item.funcargs) == 2
         assert item.funcargs['some'] == "test_func"
@@ -612,17 +600,6 @@ class TestFillFuncArgs:
             "*1 passed*"
         ])
 
-    def test_fillfuncargs_exposed(self, testdir):
-        item = testdir.getitem("def test_func(some, other=42): pass")
-        class Provider:
-            def pytest_funcarg__some(self, request):
-                return request.function.__name__
-        item.config.pluginmanager.register(Provider())
-        if hasattr(item, '_args'):
-            del item._args
-        from _pytest.python import fillfuncargs
-        fillfuncargs(item)
-        assert len(item.funcargs) == 1
 
 class TestRequest:
     def test_request_attributes(self, testdir):
@@ -642,10 +619,12 @@ class TestRequest:
     def test_request_attributes_method(self, testdir):
         item, = testdir.getitems("""
             class TestB:
+                def pytest_funcarg__something(request):
+                    return 1
                 def test_func(self, something):
                     pass
         """)
-        req = funcargs.FuncargRequest(item)
+        req = item._request
         assert req.cls.__name__ == "TestB"
         assert req.instance.__class__ == req.cls
 
@@ -686,8 +665,8 @@ class TestRequest:
                 return l.pop()
             def test_func(something): pass
         """)
-        req = funcargs.FuncargRequest(item)
+        req = item._request
-        pytest.raises(req.LookupError, req.getfuncargvalue, "notexists")
+        pytest.raises(FuncargLookupError, req.getfuncargvalue, "notexists")
         val = req.getfuncargvalue("something")
         assert val == 1
         val = req.getfuncargvalue("something")
@@ -729,7 +708,7 @@ class TestRequest:
         """)
         result = testdir.runpytest(p)
         result.stdout.fnmatch_lines([
-            "*1 passed*1 error*"
+            "*1 error*"  # XXX the whole module collection fails
             ])
 
     def test_request_getmodulepath(self, testdir):
@@ -740,6 +719,8 @@ class TestRequest:
 
 def test_applymarker(testdir):
     item1,item2 = testdir.getitems("""
+        def pytest_funcarg__something(request):
+            pass
         class TestClass:
             def test_func1(self, something):
                 pass
@@ -756,60 +737,38 @@ def test_applymarker(testdir):
     pytest.raises(ValueError, "req1.applymarker(42)")
 
 class TestRequestCachedSetup:
-    def test_request_cachedsetup(self, testdir):
+    def test_request_cachedsetup_defaultmodule(self, testdir):
-        item1,item2 = testdir.getitems("""
+        reprec = testdir.inline_runsource("""
-            def test_func1(self, something):
+            mysetup = ["hello",].pop
-                pass
-            class TestClass:
-                def test_func2(self, something):
-                    pass
-        """)
-        req1 = funcargs.FuncargRequest(item1)
-        l = ["hello"]
-        def setup():
-            return l.pop()
-        # cached_setup's scope defaults to 'module'
-        ret1 = req1.cached_setup(setup)
-        assert ret1 == "hello"
-        ret1b = req1.cached_setup(setup)
-        assert ret1 == ret1b
-        req2 = funcargs.FuncargRequest(item2)
-        ret2 = req2.cached_setup(setup)
-        assert ret2 == ret1
 
-    def test_request_cachedsetup_class(self, testdir):
+            def pytest_funcarg__something(request):
-        item1, item2, item3, item4 = testdir.getitems("""
+                return request.cached_setup(mysetup, scope="module")
-            def test_func1(self, something):
+
-                pass
+            def test_func1(something):
-            def test_func2(self, something):
+                assert something == "hello"
-                pass
             class TestClass:
                 def test_func1a(self, something):
-                    pass
+                    assert something == "hello"
-                def test_func2b(self, something):
-                    pass
         """)
-        req1 = funcargs.FuncargRequest(item2)
+        reprec.assertoutcome(passed=2)
-        l = ["hello2", "hello"]
-        def setup():
-            return l.pop()
 
-        # module level functions setup with scope=class
+    def test_request_cachedsetup_class(self, testdir):
-        # automatically turn "class" to "module" scope
+        reprec = testdir.inline_runsource("""
-        ret1 = req1.cached_setup(setup, scope="class")
+            mysetup = ["hello", "hello2"].pop
-        assert ret1 == "hello"
-        req2 = funcargs.FuncargRequest(item2)
-        ret2 = req2.cached_setup(setup, scope="class")
-        assert ret2 == "hello"
 
-        req3 = funcargs.FuncargRequest(item3)
+            def pytest_funcarg__something(request):
-        ret3a = req3.cached_setup(setup, scope="class")
+                return request.cached_setup(mysetup, scope="class")
-        ret3b = req3.cached_setup(setup, scope="class")
+            def test_func1(something):
-        assert ret3a == "hello2"
+                assert something == "hello2"
-        assert ret3b == "hello2"
+            def test_func2(something):
-        req4 = funcargs.FuncargRequest(item4)
+                assert something == "hello2"
-        ret4 = req4.cached_setup(setup, scope="class")
+            class TestClass:
-        assert ret4 == ret3a
+                def test_func1a(self, something):
+                    assert something == "hello"
+                def test_func2b(self, something):
+                    assert something == "hello"
+        """)
+        reprec.assertoutcome(passed=4)
 
     def test_request_cachedsetup_extrakey(self, testdir):
         item1 = testdir.getitem("def test_func(): pass")
@@ -1351,7 +1310,7 @@ def test_funcarg_lookup_error(testdir):
     """)
     result = testdir.runpytest()
     result.stdout.fnmatch_lines([
-        "*ERROR at setup of test_lookup_error*",
+        "*ERROR*collecting*test_funcarg_lookup_error.py*",
         "*def test_lookup_error(unknown):*",
         "*LookupError: no factory found*unknown*",
         "*available funcargs*",

testing/test_session.py

@@ -9,24 +9,24 @@ class SessionTests:
                 assert 0
             def test_other():
                 raise ValueError(23)
-            def test_two(someargs):
+            class TestClass:
-                pass
+                def test_two(self, someargs):
+                    pass
         """)
         reprec = testdir.inline_run(tfile)
         passed, skipped, failed = reprec.listoutcomes()
         assert len(skipped) == 0
         assert len(passed) == 1
-        assert len(failed) == 3
+        assert len(failed) == 2
         end = lambda x: x.nodeid.split("::")[-1]
         assert end(failed[0]) == "test_one_one"
         assert end(failed[1]) == "test_other"
-        assert end(failed[2]) == "test_two"
         itemstarted = reprec.getcalls("pytest_itemcollected")
-        assert len(itemstarted) == 4
+        assert len(itemstarted) == 3
-        colstarted = reprec.getcalls("pytest_collectstart")
+        # XXX check for failing funcarg setup
-        assert len(colstarted) == 1 + 1
+        colreports = reprec.getcalls("pytest_collectreport")
-        col = colstarted[1].collector
+        assert len(colreports) == 4
-        assert isinstance(col, pytest.Module)
+        assert colreports[1].report.failed
 
     def test_nested_import_error(self, testdir):
         tfile = testdir.makepyfile("""