From 449b55cc7002dc31e966a7b9b5aa89e98eb3bb58 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 1 Aug 2012 09:07:32 +0200 Subject: [PATCH] - enhance ordering of tests using parametrized resources - introduce a refined way to perform finalization for setup functions which does not use cached_setup() anymore --- _pytest/__init__.py | 2 +- _pytest/impl | 26 ++++ _pytest/main.py | 193 +++++++++++++++++++++------ _pytest/python.py | 21 +-- doc/en/example/newexamples.txt | 110 +++++++-------- doc/en/resources.txt | 21 ++- setup.py | 2 +- testing/test_python.py | 237 ++++++++++++++++++++++++--------- 8 files changed, 423 insertions(+), 189 deletions(-) diff --git a/_pytest/__init__.py b/_pytest/__init__.py index b93a122d1..7b3589798 100644 --- a/_pytest/__init__.py +++ b/_pytest/__init__.py @@ -1,2 +1,2 @@ # -__version__ = '2.3.0.dev5' +__version__ = '2.3.0.dev6' diff --git a/_pytest/impl b/_pytest/impl index 5fb4263ec..889e37e5a 100644 --- a/_pytest/impl +++ b/_pytest/impl @@ -1,3 +1,29 @@ +Sorting per-resource +----------------------------- + +for any given set of items: + +- collect items per session-scoped parametrized funcarg +- re-order until items no parametrizations are mixed + + examples: + + test() + test1(s1) + test1(s2) + test2() + test3(s1) + test3(s2) + + gets sorted to: + + test() + test2() + test1(s1) + test3(s1) + test1(s2) + test3(s2) + the new @setup functions -------------------------------------- diff --git a/_pytest/main.py b/_pytest/main.py index 91cf9b23c..5f7ff466c 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -425,6 +425,7 @@ class FuncargManager: session.config.pluginmanager.register(self, "funcmanage") self._holderobjseen = set() self.setuplist = [] + self._arg2finish = {} ### XXX this hook should be called for historic events like pytest_configure ### so that we don't have to do the below pytest_collection hook @@ -469,23 +470,7 @@ class FuncargManager: def pytest_collection_modifyitems(self, items): # separate parametrized setups - def sortparam(item1, item2): - try: - cs1 = item1.callspec - cs2 = item2.callspec - common = set(cs1.params).intersection(cs2.params) - except AttributeError: - pass - else: - if common: - common = list(common) - common.sort(key=lambda x: cs1._arg2scopenum[x]) - for x in common: - res = cmp(cs1.params[x], cs2.params[x]) - if res != 0: - return res - return 0 # leave previous order - items.sort(cmp=sortparam) + items[:] = parametrize_sorted(items, set(), {}, 0) def pytest_runtest_teardown(self, item, nextitem): try: @@ -501,6 +486,10 @@ class FuncargManager: pass key = (name, cs1.params[name]) item.session._setupstate._callfinalizers(key) + l = self._arg2finish.get(name) + if l is not None: + for fin in l: + fin() def _parsefactories(self, holderobj, nodeid): if holderobj in self._holderobjseen: @@ -577,17 +566,59 @@ class FuncargManager: msg += "\n use 'py.test --funcargs [testpath]' for help on them." raise FuncargLookupError(function, msg) + def ensure_setupcalls(self, request): + setuplist, allnames = self.getsetuplist(request._pyfuncitem.nodeid) + for setupcall in setuplist: + if setupcall.active: + continue + setuprequest = SetupRequest(request, setupcall) + kwargs = {} + for name in setupcall.funcargnames: + if name == "request": + kwargs[name] = setuprequest + else: + kwargs[name] = request.getfuncargvalue(name) + scope = setupcall.scope or "function" + scol = setupcall.scopeitem = request._getscopeitem(scope) + self.session._setupstate.addfinalizer(setupcall.finish, scol) + for argname in setupcall.funcargnames: # XXX all deps? + self.addargfinalizer(setupcall.finish, argname) + setupcall.execute(kwargs) -class FactoryDef: - """ A container for a factory definition. """ - def __init__(self, funcargmanager, baseid, argname, func, scope, params): - self.funcargmanager = funcargmanager - self.baseid = baseid - self.func = func - self.argname = argname - self.scope = scope - self.params = params - self.funcargnames = getfuncargnames(func) + def addargfinalizer(self, finalizer, argname): + l = self._arg2finish.setdefault(argname, []) + l.append(finalizer) + + def removefinalizer(self, finalizer): + for l in self._arg2finish.values(): + try: + l.remove(finalizer) + except ValueError: + pass + +def rprop(attr, doc=None): + if doc is None: + doc = "%r of underlying test item" + return property(lambda x: getattr(x._request, attr), doc=doc) + +class SetupRequest: + def __init__(self, request, setupcall): + self._request = request + self._setupcall = setupcall + self._finalizers = [] + + # no getfuncargvalue(), cached_setup, applymarker helpers here + # on purpose + + function = rprop("function") + cls = rprop("cls") + instance = rprop("instance") + fspath = rprop("fspath") + keywords = rprop("keywords") + config = rprop("config", "pytest config object.") + + def addfinalizer(self, finalizer): + self._setupcall.addfinalizer(finalizer) class SetupCall: """ a container/helper for managing calls to setup functions. """ @@ -601,20 +632,32 @@ class SetupCall: self._finalizer = [] def execute(self, kwargs): - #assert not self.active + assert not self.active self.active = True - mp = monkeypatch() - #if "request" in kwargs: - # request = kwargs["request"] - # def addfinalizer(func): - # #scopeitem = request._getscopeitem(scope) - # self._finalizer.append(func) - # mp.setattr(request, "addfinalizer", addfinalizer) - try: - self.func(**kwargs) - finally: - mp.undo() + self.func(**kwargs) + def addfinalizer(self, finalizer): + assert self.active + self._finalizer.append(finalizer) + + def finish(self): + while self._finalizer: + func = self._finalizer.pop() + func() + # check neccesity of next commented call + self.funcargmanager.removefinalizer(self.finish) + self.active = False + +class FactoryDef: + """ A container for a factory definition. """ + def __init__(self, funcargmanager, baseid, argname, func, scope, params): + self.funcargmanager = funcargmanager + self.baseid = baseid + self.func = func + self.argname = argname + self.scope = scope + self.params = params + self.funcargnames = getfuncargnames(func) class NoMatch(Exception): """ raised if matching cannot locate a matching names. """ @@ -899,3 +942,75 @@ def readscope(func, markattr): marker = getattr(func, markattr, None) if marker is not None: return marker.kwargs.get("scope") + +# algorithm for sorting on a per-parametrized resource setup basis + +def parametrize_sorted(items, ignore, cache, scopenum): + if scopenum >= 3: + return items + newitems = [] + olditems = [] + slicing_argparam = None + for item in items: + argparamlist = getfuncargparams(item, ignore, scopenum, cache) + if slicing_argparam is None and argparamlist: + slicing_argparam = argparamlist[0] + slicing_index = len(olditems) + if slicing_argparam in argparamlist: + newitems.append(item) + else: + olditems.append(item) + if newitems: + newignore = ignore.copy() + newignore.add(slicing_argparam) + newitems = parametrize_sorted(newitems + olditems[slicing_index:], + newignore, cache, scopenum) + old1 = parametrize_sorted(olditems[:slicing_index], newignore, + cache, scopenum+1) + return old1 + newitems + else: + olditems = parametrize_sorted(olditems, ignore, cache, scopenum+1) + return olditems + newitems + +def getfuncargparams(item, ignore, scopenum, cache): + """ return list of (arg,param) tuple, sorted by broader scope first. """ + assert scopenum < 3 # function + try: + cs = item.callspec + except AttributeError: + return [] + if scopenum == 0: + argparams = [x for x in cs.params.items() if x not in ignore + and cs._arg2scopenum[x[0]] == scopenum] + elif scopenum == 1: # module + argparams = [] + for argname, param in cs.params.items(): + if cs._arg2scopenum[argname] == scopenum: + key = (argname, param, item.fspath) + if key in ignore: + continue + argparams.append(key) + elif scopenum == 2: # class + argparams = [] + for argname, param in cs.params.items(): + if cs._arg2scopenum[argname] == scopenum: + l = cache.setdefault(item.fspath, []) + try: + i = l.index(item.cls) + except ValueError: + i = len(l) + l.append(item.cls) + key = (argname, param, item.fspath, i) + if key in ignore: + continue + argparams.append(key) + #elif scopenum == 3: + # argparams = [] + # for argname, param in cs.params.items(): + # if cs._arg2scopenum[argname] == scopenum: + # key = (argname, param, getfslineno(item.obj)) + # if key in ignore: + # continue + # argparams.append(key) + return argparams + diff --git a/_pytest/python.py b/_pytest/python.py index a5a616661..d0bed89f3 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1001,26 +1001,7 @@ class FuncargRequest: def _callsetup(self): - setuplist, allnames = self.funcargmanager.getsetuplist( - self._pyfuncitem.nodeid) - mp = monkeypatch() - for setupcall in setuplist: - kwargs = {} - for name in setupcall.funcargnames: - if name == "request": - kwargs[name] = self - else: - kwargs[name] = self.getfuncargvalue(name) - - mp.setattr(self, 'scope', setupcall.scope) - try: - if setupcall.scope is None: - setupcall.execute(kwargs) - else: - self.cached_setup(lambda: setupcall.execute(kwargs), - scope=setupcall.scope) - finally: - mp.undo() + self.funcargmanager.ensure_setupcalls(self) def getfuncargvalue(self, argname): """ Retrieve a function argument by name for this test diff --git a/doc/en/example/newexamples.txt b/doc/en/example/newexamples.txt index 4bf3455f5..9a4902b65 100644 --- a/doc/en/example/newexamples.txt +++ b/doc/en/example/newexamples.txt @@ -48,7 +48,7 @@ If you run the tests:: ================================= FAILURES ================================= ________________________________ test_ehlo _________________________________ - smtp = + smtp = def test_ehlo(smtp): response = smtp.ehlo() @@ -60,7 +60,7 @@ If you run the tests:: test_module.py:5: AssertionError ________________________________ test_noop _________________________________ - smtp = + smtp = def test_noop(smtp): response = smtp.noop() @@ -69,7 +69,7 @@ If you run the tests:: E assert 0 test_module.py:10: AssertionError - 2 failed in 0.21 seconds + 2 failed in 0.26 seconds you will see the two ``assert 0`` failing and can see that the same (session-scoped) object was passed into the two test functions. @@ -98,31 +98,9 @@ another run:: collecting ... collected 4 items FFFF ================================= FAILURES ================================= - ________________________ test_ehlo[mail.python.org] ________________________ - - smtp = - - def test_ehlo(smtp): - response = smtp.ehlo() - assert response[0] == 250 - > assert "merlinux" in response[1] - E assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN' - - test_module.py:4: AssertionError - ________________________ test_noop[mail.python.org] ________________________ - - smtp = - - def test_noop(smtp): - response = smtp.noop() - assert response[0] == 250 - > assert 0 # for demo purposes - E assert 0 - - test_module.py:10: AssertionError __________________________ test_ehlo[merlinux.eu] __________________________ - smtp = + smtp = def test_ehlo(smtp): response = smtp.ehlo() @@ -134,7 +112,7 @@ another run:: test_module.py:5: AssertionError __________________________ test_noop[merlinux.eu] __________________________ - smtp = + smtp = def test_noop(smtp): response = smtp.noop() @@ -143,7 +121,29 @@ another run:: E assert 0 test_module.py:10: AssertionError - 4 failed in 6.48 seconds + ________________________ test_ehlo[mail.python.org] ________________________ + + smtp = + + def test_ehlo(smtp): + response = smtp.ehlo() + assert response[0] == 250 + > assert "merlinux" in response[1] + E assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN' + + test_module.py:4: AssertionError + ________________________ test_noop[mail.python.org] ________________________ + + smtp = + + def test_noop(smtp): + response = smtp.noop() + assert response[0] == 250 + > assert 0 # for demo purposes + E assert 0 + + test_module.py:10: AssertionError + 4 failed in 6.94 seconds We get four failures because we are running the two tests twice with different ``smtp`` instantiations as defined on the factory. @@ -159,10 +159,10 @@ You can look at what tests pytest collects without running them:: plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov collecting ... collected 4 items - - + + ============================= in 0.02 seconds ============================= @@ -172,13 +172,13 @@ And you can run without output capturing and minimized failure reporting to chec collecting ... collected 4 items FFFF ================================= FAILURES ================================= - /home/hpk/tmp/doc-exec-386/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN' - /home/hpk/tmp/doc-exec-386/test_module.py:10: assert 0 - /home/hpk/tmp/doc-exec-386/test_module.py:5: assert 0 - /home/hpk/tmp/doc-exec-386/test_module.py:10: assert 0 - 4 failed in 6.45 seconds - closing - closing + /home/hpk/tmp/doc-exec-389/test_module.py:5: assert 0 + /home/hpk/tmp/doc-exec-389/test_module.py:10: assert 0 + /home/hpk/tmp/doc-exec-389/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN' + /home/hpk/tmp/doc-exec-389/test_module.py:10: assert 0 + 4 failed in 9.99 seconds + closing + closing .. _`new_setup`: @@ -191,8 +191,11 @@ And you can run without output capturing and minimized failure reporting to chec The ``@pytest.mark.setup`` marker allows +* to define setup-functions close to test code or in conftest.py files + or plugins. * to mark a function as a setup/fixture method; the function can itself - receive funcargs + receive funcargs and will execute multiple times if the funcargs + are parametrized * to set a scope which determines the level of caching and how often the setup function is going to be called. @@ -234,12 +237,12 @@ Let's run this module:: $ py.test -qs collecting ... collected 2 items .. - 2 passed in 0.24 seconds - created resource /home/hpk/tmp/pytest-3875/test_10 - setupresource /home/hpk/tmp/pytest-3875/test_10 - using myresource /home/hpk/tmp/pytest-3875/test_10 - using myresource /home/hpk/tmp/pytest-3875/test_10 - finalize /home/hpk/tmp/pytest-3875/test_10 + 2 passed in 0.62 seconds + created resource /home/hpk/tmp/pytest-4224/test_10 + setupresource /home/hpk/tmp/pytest-4224/test_10 + using myresource /home/hpk/tmp/pytest-4224/test_10 + using myresource /home/hpk/tmp/pytest-4224/test_10 + finalize /home/hpk/tmp/pytest-4224/test_10 The two test functions will see the same resource instance because it has a module life cycle or scope. @@ -265,15 +268,16 @@ Running this will run four tests:: collecting ... collected 4 items .... 4 passed in 0.25 seconds - created resource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa - setupresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa - using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa - using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa - finalize /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa - created resource /home/hpk/tmp/pytest-3876/test_1_bbb_0/bbb - using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa - using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa - finalize /home/hpk/tmp/pytest-3876/test_1_bbb_0/bbb + created resource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa + setupresource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa + using myresource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa + using myresource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa + finalize /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa + created resource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb + setupresource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb + using myresource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb + using myresource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb + finalize /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb Each parameter causes the creation of a respective resource and the unchanged test module uses it in its ``@setup`` decorated method. diff --git a/doc/en/resources.txt b/doc/en/resources.txt index ee96d67ba..4c2c301a1 100644 --- a/doc/en/resources.txt +++ b/doc/en/resources.txt @@ -51,7 +51,7 @@ implementation or backward compatibility issues. The main changes are: troubles than the current @setup approach which can share a lot of logic with the @funcarg one. -* tests are grouped by any parametrized resource +* tests are grouped by parametrized funcargs .. currentmodule:: _pytest @@ -259,13 +259,13 @@ Grouping tests by resource parameters .. note:: Implemented. -pytest usually sorts test items by their source location. -With pytest-2.X tests are first grouped by resource parameters. -If you have a parametrized resource, then all the tests using it +pytest used to always sort test items by their source location. +With pytest-2.X tests are first grouped by funcarg parameters. +If you have a parametrized funcarg, then all the tests using it will first execute with it. Then any finalizers are called and then the next parametrized resource instance is created and its tests are run. -Among other things, this allows to have per-session parametrized setups -including ones which affect global state of an application. +Among other things, this eases testing of applications which create +and use global state. The following example uses two parametrized funcargs, one of which is scoped on a per-module basis:: @@ -293,12 +293,12 @@ scoped on a per-module basis:: def test_2(otherarg, modarg): print " test2", otherarg, modarg -If you run the tests in verbose mode and with looking at captured output:: +Let's run the tests in verbose mode and with looking at the print-output:: $ py.test -v -s =========================== test session starts ============================ platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev5 -- /home/hpk/venv/1/bin/python - cachedir: /home/hpk/tmp/doc-exec-382/.cache + cachedir: /home/hpk/tmp/doc-exec-388/.cache plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov collecting ... collected 8 items @@ -326,9 +326,8 @@ If you run the tests in verbose mode and with looking at captured output:: fin mod2 You can see that that the parametrized ``modarg`` resource lead to -a re-ordering of test execution. Moreover, the finalizer for the -"mod1" parametrized resource was executed before the "mod2" resource -was setup with a different parameter. +a re-ordering of test execution. The finalizer for the "mod1" parametrized +resource was executed before the "mod2" resource was setup. .. note:: diff --git a/setup.py b/setup.py index 29fafeb5c..78d9826ab 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def main(): name='pytest', description='py.test: simple powerful testing with Python', long_description = long_description, - version='2.3.0.dev5', + version='2.3.0.dev6', url='http://pytest.org', license='MIT license', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], diff --git a/testing/test_python.py b/testing/test_python.py index dda597891..36e16f5cc 100644 --- a/testing/test_python.py +++ b/testing/test_python.py @@ -1559,33 +1559,8 @@ def test_issue117_sessionscopeteardown(testdir): ]) class TestRequestAPI: - @pytest.mark.xfail(reason="reverted refactoring") - def test_addfinalizer_cachedsetup_getfuncargvalue(self, testdir): - testdir.makeconftest(""" - l = [] - def pytest_runtest_setup(item): - item.addfinalizer(lambda: l.append(1)) - l2 = item.getfuncargvalue("l") - assert l2 is l - item.cached_setup(lambda: l.append(2), lambda val: l.append(3), - scope="function") - def pytest_funcarg__l(request): - return l - """) - testdir.makepyfile(""" - def test_hello(): - pass - def test_hello2(l): - assert l == [2, 3, 1, 2] - """) - result = testdir.runpytest() - assert result.ret == 0 - result.stdout.fnmatch_lines([ - "*2 passed*", - ]) - - @pytest.mark.xfail(reason="consider item's funcarg access and error conditions") - def test_runtest_setup_sees_filled_funcargs(self, testdir): + @pytest.mark.xfail(reason="consider flub feedback") + def test_setup_can_query_funcargs(self, testdir): testdir.makeconftest(""" def pytest_runtest_setup(item): assert not hasattr(item, "_request") @@ -1606,9 +1581,9 @@ class TestRequestAPI: result = testdir.makeconftest(""" import pytest - @pytest.mark.trylast - def pytest_runtest_setup(item): - assert item.funcargs == {"a": 1, "b": 2} + @pytest.mark.setup + def mysetup(testcontext): + testcontext.uses_funcarg("db") """) result = testdir.runpytest() assert result.ret == 0 @@ -1737,7 +1712,7 @@ class TestFuncargManager: return "class" def test_hello(self, item, fm): faclist = fm.getfactorylist("hello", item.nodeid, item.obj) - print faclist + print (faclist) assert len(faclist) == 3 assert faclist[0].func(item._request) == "conftest" assert faclist[1].func(item._request) == "module" @@ -1863,6 +1838,43 @@ class TestSetupManagement: reprec = testdir.inline_run("-v", "-s") reprec.assertoutcome(passed=4) + def test_class_function_parametrization_finalization(self, testdir): + p = testdir.makeconftest(""" + import pytest + import pprint + + l = [] + + @pytest.mark.funcarg(scope="function", params=[1,2]) + def farg(request): + return request.param + + @pytest.mark.funcarg(scope="class", params=list("ab")) + def carg(request): + return request.param + + @pytest.mark.setup(scope="class") + def append(request, farg, carg): + def fin(): + l.append("fin_%s%s" % (carg, farg)) + request.addfinalizer(fin) + """) + testdir.makepyfile(""" + import pytest + + class TestClass: + def test_1(self): + pass + class TestClass2: + def test_2(self): + pass + """) + reprec = testdir.inline_run("-v",) + reprec.assertoutcome(passed=8) + config = reprec.getcalls("pytest_unconfigure")[0].config + l = config._conftest.getconftestmodules(p)[0].l + assert l == ["fin_a1", "fin_a2", "fin_b1", "fin_b2"] * 2 + class TestFuncargMarker: def test_parametrize(self, testdir): testdir.makepyfile(""" @@ -2016,11 +2028,14 @@ class TestFuncargMarker: l = [] def test_param(arg): l.append(arg) - def test_result(): - assert l == list("abc") """) - reprec = testdir.inline_run() - reprec.assertoutcome(passed=4) + reprec = testdir.inline_run("-v") + reprec.assertoutcome(passed=3) + l = reprec.getcalls("pytest_runtest_call")[0].item.module.l + assert len(l) == 3 + assert "a" in l + assert "b" in l + assert "c" in l def test_scope_mismatch(self, testdir): testdir.makeconftest(""" @@ -2056,14 +2071,105 @@ class TestFuncargMarker: l.append(arg) def test_2(arg): l.append(arg) - def test_3(): - assert len(l) == 4 - assert l[0] == l[1] - assert l[2] == l[3] - """) reprec = testdir.inline_run("-v") - reprec.assertoutcome(passed=5) + reprec.assertoutcome(passed=4) + l = reprec.getcalls("pytest_runtest_call")[0].item.module.l + assert l == [1,1,2,2] + + def test_module_parametrized_ordering(self, testdir): + testdir.makeconftest(""" + import pytest + + @pytest.mark.funcarg(scope="session", params="s1 s2".split()) + def sarg(request): + pass + @pytest.mark.funcarg(scope="module", params="m1 m2".split()) + def marg(request): + pass + """) + testdir.makepyfile(test_mod1=""" + def test_func(sarg): + pass + def test_func1(marg): + pass + """, test_mod2=""" + def test_func2(sarg): + pass + def test_func3(sarg, marg): + pass + def test_func3b(sarg, marg): + pass + def test_func4(marg): + pass + """) + result = testdir.runpytest("-v") + result.stdout.fnmatch_lines(""" + test_mod1.py:1: test_func[s1] PASSED + test_mod2.py:1: test_func2[s1] PASSED + test_mod2.py:3: test_func3[s1-m1] PASSED + test_mod2.py:5: test_func3b[s1-m1] PASSED + test_mod2.py:3: test_func3[s1-m2] PASSED + test_mod2.py:5: test_func3b[s1-m2] PASSED + test_mod1.py:1: test_func[s2] PASSED + test_mod2.py:1: test_func2[s2] PASSED + test_mod2.py:3: test_func3[s2-m1] PASSED + test_mod2.py:5: test_func3b[s2-m1] PASSED + test_mod2.py:7: test_func4[m1] PASSED + test_mod2.py:3: test_func3[s2-m2] PASSED + test_mod2.py:5: test_func3b[s2-m2] PASSED + test_mod2.py:7: test_func4[m2] PASSED + test_mod1.py:3: test_func1[m1] PASSED + test_mod1.py:3: test_func1[m2] PASSED + """) + + def test_class_ordering(self, testdir): + p = testdir.makeconftest(""" + import pytest + + l = [] + + @pytest.mark.funcarg(scope="function", params=[1,2]) + def farg(request): + return request.param + + @pytest.mark.funcarg(scope="class", params=list("ab")) + def carg(request): + return request.param + + @pytest.mark.setup(scope="class") + def append(request, farg, carg): + def fin(): + l.append("fin_%s%s" % (carg, farg)) + request.addfinalizer(fin) + """) + testdir.makepyfile(""" + import pytest + + class TestClass2: + def test_1(self): + pass + def test_2(self): + pass + class TestClass: + def test_3(self): + pass + """) + result = testdir.runpytest("-v") + result.stdout.fnmatch_lines(""" + test_class_ordering.py:4: TestClass2.test_1[1-a] PASSED + test_class_ordering.py:4: TestClass2.test_1[2-a] PASSED + test_class_ordering.py:6: TestClass2.test_2[1-a] PASSED + test_class_ordering.py:6: TestClass2.test_2[2-a] PASSED + test_class_ordering.py:4: TestClass2.test_1[1-b] PASSED + test_class_ordering.py:4: TestClass2.test_1[2-b] PASSED + test_class_ordering.py:6: TestClass2.test_2[1-b] PASSED + test_class_ordering.py:6: TestClass2.test_2[2-b] PASSED + test_class_ordering.py:9: TestClass.test_3[1-a] PASSED + test_class_ordering.py:9: TestClass.test_3[2-a] PASSED + test_class_ordering.py:9: TestClass.test_3[1-b] PASSED + test_class_ordering.py:9: TestClass.test_3[2-b] PASSED + """) def test_parametrize_separated_order_higher_scope_first(self, testdir): testdir.makepyfile(""" @@ -2097,17 +2203,14 @@ class TestFuncargMarker: import pprint pprint.pprint(l) assert l == [ - 'create:1', 'test1', 'fin:1', - 'create:2', 'test1', 'fin:2', - 'create:mod1', 'test2', 'create:1', 'test3', 'fin:1', - 'create:1', 'test4', 'fin:1', 'create:2', 'test3', 'fin:2', - 'create:2', 'test4', 'fin:mod1', 'fin:2', - - 'create:mod2', 'test2', 'create:1', 'test3', 'fin:1', - 'create:1', 'test4', 'fin:1', 'create:2', 'test3', 'fin:2', - 'create:2', 'test4', 'fin:mod2', 'fin:2', - ] - + 'create:1', 'test1', 'fin:1', 'create:2', 'test1', + 'fin:2', 'create:mod1', 'test2', 'create:1', 'test3', + 'fin:1', 'create:2', 'test3', 'fin:2', 'create:1', + 'test4', 'fin:1', 'create:2', 'test4', 'fin:mod1', + 'fin:2', 'create:mod2', 'test2', 'create:1', 'test3', + 'fin:1', 'create:2', 'test3', 'fin:2', 'create:1', + 'test4', 'fin:1', 'create:2', 'test4', 'fin:mod2', + 'fin:2'] """) reprec = testdir.inline_run("-v") reprec.assertoutcome(passed=12+1) @@ -2118,6 +2221,7 @@ class TestFuncargMarker: @pytest.mark.funcarg(scope="module", params=[1, 2]) def arg(request): + request.config.l = l # to access from outer x = request.param request.addfinalizer(lambda: l.append("fin%s" % x)) return request.param @@ -2127,16 +2231,18 @@ class TestFuncargMarker: l.append(arg) def test_2(arg): l.append(arg) - def test_3(): - assert len(l) == 6 - assert l[0] == l[1] - assert l[2] == "fin1" - assert l[3] == l[4] - assert l[5] == "fin2" - """) reprec = testdir.inline_run("-v") - reprec.assertoutcome(passed=5) + reprec.assertoutcome(passed=4) + l = reprec.getcalls("pytest_configure")[0].config.l + import pprint + pprint.pprint(l) + assert len(l) == 6 + assert l[0] == l[1] == 1 + assert l[2] == "fin1" + assert l[3] == l[4] == 2 + assert l[5] == "fin2" + def test_parametrize_function_scoped_finalizers_called(self, testdir): testdir.makepyfile(""" @@ -2155,7 +2261,7 @@ class TestFuncargMarker: l.append(arg) def test_3(): assert len(l) == 8 - assert l == [1, "fin1", 1, "fin1", 2, "fin2", 2, "fin2"] + assert l == [1, "fin1", 2, "fin2", 1, "fin1", 2, "fin2"] """) reprec = testdir.inline_run("-v") reprec.assertoutcome(passed=5) @@ -2181,10 +2287,13 @@ class TestFuncargMarker: def test_3(): import pprint pprint.pprint(l) - assert l == ["setup1", 1, 1, "fin1", - "setup2", 2, 2, "fin2",] + if arg == 1: + assert l == ["setup1", 1, 1, ] + elif arg == 2: + assert l == ["setup1", 1, 1, "fin1", + "setup2", 2, 2, ] """) reprec = testdir.inline_run("-v") - reprec.assertoutcome(passed=5) + reprec.assertoutcome(passed=6)