minimize active parametrized non-function scoped resources by
- re-ordering at collection time - modifying setup/teardown
This commit is contained in:
holger krekel
parent
fa61927c6b
commit
d68c65b493
|
|
@ -1,2 +1,2 @@
|
|||
#
|
||||
__version__ = '2.3.0.dev4'
|
||||
__version__ = '2.3.0.dev5'
|
||||
|
|
|
|||
183
_pytest/impl
183
_pytest/impl
|
|
@ -1,4 +1,187 @@
|
|||
|
||||
the new @setup functions
|
||||
--------------------------------------
|
||||
|
||||
Consider a given @setup-marked function::
|
||||
|
||||
@pytest.mark.setup(maxscope=SCOPE)
|
||||
def mysetup(request, arg1, arg2, ...)
|
||||
...
|
||||
request.addfinalizer(fin)
|
||||
...
|
||||
|
||||
then FUNCARGSET denotes the set of (arg1, arg2, ...) funcargs and
|
||||
all of its dependent funcargs. The mysetup function will execute
|
||||
for any matching test item once per scope.
|
||||
|
||||
The scope is determined as the minimum scope of all scopes of the args
|
||||
in FUNCARGSET and the given "maxscope".
|
||||
|
||||
If mysetup has been called and no finalizers have been called it is
|
||||
called "active".
|
||||
|
||||
Furthermore the following rules apply:
|
||||
|
||||
- if an arg value in FUNCARGSET is about to be torn down, the
|
||||
mysetup-registered finalizers will execute as well.
|
||||
|
||||
- There will never be two active mysetup invocations.
|
||||
|
||||
Example 1, session scope::
|
||||
|
||||
@pytest.mark.funcarg(scope="session", params=[1,2])
|
||||
def db(request):
|
||||
request.addfinalizer(db_finalize)
|
||||
|
||||
@pytest.mark.setup
|
||||
def mysetup(request, db):
|
||||
request.addfinalizer(mysetup_finalize)
|
||||
...
|
||||
|
||||
And a given test module:
|
||||
|
||||
def test_something():
|
||||
...
|
||||
def test_otherthing():
|
||||
pass
|
||||
|
||||
Here is what happens::
|
||||
|
||||
db(request) executes with request.param == 1
|
||||
mysetup(request, db) executes
|
||||
test_something() executes
|
||||
test_otherthing() executes
|
||||
mysetup_finalize() executes
|
||||
db_finalize() executes
|
||||
db(request) executes with request.param == 2
|
||||
mysetup(request, db) executes
|
||||
test_something() executes
|
||||
test_otherthing() executes
|
||||
mysetup_finalize() executes
|
||||
db_finalize() executes
|
||||
|
||||
Example 2, session/function scope::
|
||||
|
||||
@pytest.mark.funcarg(scope="session", params=[1,2])
|
||||
def db(request):
|
||||
request.addfinalizer(db_finalize)
|
||||
|
||||
@pytest.mark.setup(scope="function")
|
||||
def mysetup(request, db):
|
||||
...
|
||||
request.addfinalizer(mysetup_finalize)
|
||||
...
|
||||
|
||||
And a given test module:
|
||||
|
||||
def test_something():
|
||||
...
|
||||
def test_otherthing():
|
||||
pass
|
||||
|
||||
Here is what happens::
|
||||
|
||||
db(request) executes with request.param == 1
|
||||
mysetup(request, db) executes
|
||||
test_something() executes
|
||||
mysetup_finalize() executes
|
||||
mysetup(request, db) executes
|
||||
test_otherthing() executes
|
||||
mysetup_finalize() executes
|
||||
db_finalize() executes
|
||||
db(request) executes with request.param == 2
|
||||
mysetup(request, db) executes
|
||||
test_something() executes
|
||||
mysetup_finalize() executes
|
||||
mysetup(request, db) executes
|
||||
test_otherthing() executes
|
||||
mysetup_finalize() executes
|
||||
db_finalize() executes
|
||||
|
||||
|
||||
Example 3 - funcargs session-mix
|
||||
----------------------------------------
|
||||
|
||||
Similar with funcargs, an example::
|
||||
|
||||
@pytest.mark.funcarg(scope="session", params=[1,2])
|
||||
def db(request):
|
||||
request.addfinalizer(db_finalize)
|
||||
|
||||
@pytest.mark.funcarg(scope="function")
|
||||
def table(request, db):
|
||||
...
|
||||
request.addfinalizer(table_finalize)
|
||||
...
|
||||
|
||||
And a given test module:
|
||||
|
||||
def test_something(table):
|
||||
...
|
||||
def test_otherthing(table):
|
||||
pass
|
||||
def test_thirdthing():
|
||||
pass
|
||||
|
||||
Here is what happens::
|
||||
|
||||
db(request) executes with param == 1
|
||||
table(request, db)
|
||||
test_something(table)
|
||||
table_finalize()
|
||||
table(request, db)
|
||||
test_otherthing(table)
|
||||
table_finalize()
|
||||
db_finalize
|
||||
db(request) executes with param == 2
|
||||
table(request, db)
|
||||
test_something(table)
|
||||
table_finalize()
|
||||
table(request, db)
|
||||
test_otherthing(table)
|
||||
table_finalize()
|
||||
db_finalize
|
||||
test_thirdthing()
|
||||
|
||||
Data structures
|
||||
--------------------
|
||||
|
||||
pytest internally maintains a dict of active funcargs with cache, param,
|
||||
finalizer, (scopeitem?) information:
|
||||
|
||||
active_funcargs = dict()
|
||||
|
||||
if a parametrized "db" is activated:
|
||||
|
||||
active_funcargs["db"] = FuncargInfo(dbvalue, paramindex,
|
||||
FuncargFinalize(...), scopeitem)
|
||||
|
||||
if a test is torn down and the next test requires a differently
|
||||
parametrized "db":
|
||||
|
||||
for argname in item.callspec.params:
|
||||
if argname in active_funcargs:
|
||||
funcarginfo = active_funcargs[argname]
|
||||
if funcarginfo.param != item.callspec.params[argname]:
|
||||
funcarginfo.callfinalizer()
|
||||
del node2funcarg[funcarginfo.scopeitem]
|
||||
del active_funcargs[argname]
|
||||
nodes_to_be_torn_down = ...
|
||||
for node in nodes_to_be_torn_down:
|
||||
if node in node2funcarg:
|
||||
argname = node2funcarg[node]
|
||||
active_funcargs[argname].callfinalizer()
|
||||
del node2funcarg[node]
|
||||
del active_funcargs[argname]
|
||||
|
||||
if a test is setup requiring a "db" funcarg:
|
||||
|
||||
if "db" in active_funcargs:
|
||||
return active_funcargs["db"][0]
|
||||
funcarginfo = setup_funcarg()
|
||||
active_funcargs["db"] = funcarginfo
|
||||
node2funcarg[funcarginfo.scopeitem] = "db"
|
||||
|
||||
Implementation plan for resources
|
||||
------------------------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -465,13 +465,48 @@ class FuncargManager:
|
|||
marker = getattr(fac, "funcarg", None)
|
||||
if marker is not None:
|
||||
params = marker.kwargs.get("params")
|
||||
scope = marker.kwargs.get("scope", "function")
|
||||
if params is not None:
|
||||
metafunc.parametrize(argname, params, indirect=True)
|
||||
metafunc.parametrize(argname, params, indirect=True,
|
||||
scope=scope)
|
||||
newfuncargnames = getfuncargnames(fac)
|
||||
newfuncargnames.remove("request")
|
||||
funcargnames.extend(newfuncargnames)
|
||||
|
||||
def pytest_collection_modifyitems(self, items):
|
||||
# separate parametrized setups
|
||||
def sortparam(item1, item2):
|
||||
try:
|
||||
cs1 = item1.callspec
|
||||
cs2 = item2.callspec
|
||||
common = set(cs1.params).intersection(cs2.params)
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
if common:
|
||||
common = list(common)
|
||||
common.sort(key=lambda x: cs1._arg2scopenum[x])
|
||||
for x in common:
|
||||
res = cmp(cs1.params[x], cs2.params[x])
|
||||
if res != 0:
|
||||
return res
|
||||
return 0 # leave previous order
|
||||
items.sort(cmp=sortparam)
|
||||
|
||||
def pytest_runtest_teardown(self, item, nextitem):
|
||||
try:
|
||||
cs1 = item.callspec
|
||||
except AttributeError:
|
||||
return
|
||||
for name in cs1.params:
|
||||
try:
|
||||
if name in nextitem.callspec.params and \
|
||||
cs1.params[name] == nextitem.callspec.params[name]:
|
||||
continue
|
||||
except AttributeError:
|
||||
pass
|
||||
key = (name, cs1.params[name])
|
||||
item.session._setupstate._callfinalizers(key)
|
||||
|
||||
def _parsefactories(self, holderobj, nodeid):
|
||||
if holderobj in self._holderobjseen:
|
||||
|
|
|
|||
|
|
@ -499,11 +499,13 @@ class CallSpec2(object):
|
|||
self._globalid = _notexists
|
||||
self._globalid_args = set()
|
||||
self._globalparam = _notexists
|
||||
self._arg2scopenum = {} # used for sorting parametrized resources
|
||||
|
||||
def copy(self, metafunc):
|
||||
cs = CallSpec2(self.metafunc)
|
||||
cs.funcargs.update(self.funcargs)
|
||||
cs.params.update(self.params)
|
||||
cs._arg2scopenum.update(self._arg2scopenum)
|
||||
cs._idlist = list(self._idlist)
|
||||
cs._globalid = self._globalid
|
||||
cs._globalid_args = self._globalid_args
|
||||
|
|
@ -526,10 +528,11 @@ class CallSpec2(object):
|
|||
def id(self):
|
||||
return "-".join(map(str, filter(None, self._idlist)))
|
||||
|
||||
def setmulti(self, valtype, argnames, valset, id):
|
||||
def setmulti(self, valtype, argnames, valset, id, scopenum=0):
|
||||
for arg,val in zip(argnames, valset):
|
||||
self._checkargnotcontained(arg)
|
||||
getattr(self, valtype)[arg] = val
|
||||
self._arg2scopenum[arg] = scopenum
|
||||
self._idlist.append(id)
|
||||
|
||||
def setall(self, funcargs, id, param):
|
||||
|
|
@ -556,8 +559,10 @@ class Metafunc:
|
|||
self.module = module
|
||||
self._calls = []
|
||||
self._ids = py.builtin.set()
|
||||
self._arg2scopenum = {}
|
||||
|
||||
def parametrize(self, argnames, argvalues, indirect=False, ids=None):
|
||||
def parametrize(self, argnames, argvalues, indirect=False, ids=None,
|
||||
scope="function"):
|
||||
""" Add new invocations to the underlying test function using the list
|
||||
of argvalues for the given argnames. Parametrization is performed
|
||||
during the collection phase. If you need to setup expensive resources
|
||||
|
|
@ -581,6 +586,7 @@ class Metafunc:
|
|||
if not isinstance(argnames, (tuple, list)):
|
||||
argnames = (argnames,)
|
||||
argvalues = [(val,) for val in argvalues]
|
||||
scopenum = scopes.index(scope)
|
||||
if not indirect:
|
||||
#XXX should we also check for the opposite case?
|
||||
for arg in argnames:
|
||||
|
|
@ -595,7 +601,8 @@ class Metafunc:
|
|||
for i, valset in enumerate(argvalues):
|
||||
assert len(valset) == len(argnames)
|
||||
newcallspec = callspec.copy(self)
|
||||
newcallspec.setmulti(valtype, argnames, valset, ids[i])
|
||||
newcallspec.setmulti(valtype, argnames, valset, ids[i],
|
||||
scopenum)
|
||||
newcalls.append(newcallspec)
|
||||
self._calls = newcalls
|
||||
|
||||
|
|
@ -995,6 +1002,7 @@ class FuncargRequest:
|
|||
def _callsetup(self):
|
||||
setuplist, allnames = self.funcargmanager.getsetuplist(
|
||||
self._pyfuncitem.nodeid)
|
||||
mp = monkeypatch()
|
||||
for setupfunc, funcargnames in setuplist:
|
||||
kwargs = {}
|
||||
for name in funcargnames:
|
||||
|
|
@ -1002,11 +1010,16 @@ class FuncargRequest:
|
|||
kwargs[name] = self
|
||||
else:
|
||||
kwargs[name] = self.getfuncargvalue(name)
|
||||
|
||||
scope = readscope(setupfunc, "setup")
|
||||
if scope is None:
|
||||
setupfunc(**kwargs)
|
||||
else:
|
||||
self.cached_setup(lambda: setupfunc(**kwargs), scope=scope)
|
||||
mp.setattr(self, 'scope', scope)
|
||||
try:
|
||||
if scope is None:
|
||||
setupfunc(**kwargs)
|
||||
else:
|
||||
self.cached_setup(lambda: setupfunc(**kwargs), scope=scope)
|
||||
finally:
|
||||
mp.undo()
|
||||
|
||||
def getfuncargvalue(self, argname):
|
||||
""" Retrieve a function argument by name for this test
|
||||
|
|
@ -1047,7 +1060,7 @@ class FuncargRequest:
|
|||
else:
|
||||
mp.setattr(self, 'param', param, raising=False)
|
||||
|
||||
# implemenet funcarg marker scope
|
||||
# implement funcarg marker scope
|
||||
scope = readscope(funcargfactory, "funcarg")
|
||||
|
||||
if scope is not None:
|
||||
|
|
@ -1100,7 +1113,12 @@ class FuncargRequest:
|
|||
self._addfinalizer(finalizer, scope=self.scope)
|
||||
|
||||
def _addfinalizer(self, finalizer, scope):
|
||||
colitem = self._getscopeitem(scope)
|
||||
if scope != "function" and hasattr(self, "param"):
|
||||
# parametrized resources are sorted by param
|
||||
# so we rather store finalizers per (argname, param)
|
||||
colitem = (self._currentarg, self.param)
|
||||
else:
|
||||
colitem = self._getscopeitem(scope)
|
||||
self._pyfuncitem.session._setupstate.addfinalizer(
|
||||
finalizer=finalizer, colitem=colitem)
|
||||
|
||||
|
|
|
|||
|
|
@ -294,6 +294,8 @@ class SetupState(object):
|
|||
""" attach a finalizer to the given colitem.
|
||||
if colitem is None, this will add a finalizer that
|
||||
is called at the end of teardown_all().
|
||||
if colitem is a tuple, it will be used as a key
|
||||
and needs an explicit call to _callfinalizers(key) later on.
|
||||
"""
|
||||
assert hasattr(finalizer, '__call__')
|
||||
#assert colitem in self.stack
|
||||
|
|
@ -311,15 +313,17 @@ class SetupState(object):
|
|||
|
||||
def _teardown_with_finalization(self, colitem):
|
||||
self._callfinalizers(colitem)
|
||||
if colitem:
|
||||
if hasattr(colitem, "teardown"):
|
||||
colitem.teardown()
|
||||
for colitem in self._finalizers:
|
||||
assert colitem is None or colitem in self.stack
|
||||
assert colitem is None or colitem in self.stack \
|
||||
or isinstance(colitem, tuple)
|
||||
|
||||
def teardown_all(self):
|
||||
while self.stack:
|
||||
self._pop_and_teardown()
|
||||
self._teardown_with_finalization(None)
|
||||
for key in list(self._finalizers):
|
||||
self._teardown_with_finalization(key)
|
||||
assert not self._finalizers
|
||||
|
||||
def teardown_exact(self, item, nextitem):
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ If you run the tests::
|
|||
================================= FAILURES =================================
|
||||
________________________________ test_ehlo _________________________________
|
||||
|
||||
smtp = <smtplib.SMTP instance at 0x20ba7e8>
|
||||
smtp = <smtplib.SMTP instance at 0x2c0f170>
|
||||
|
||||
def test_ehlo(smtp):
|
||||
response = smtp.ehlo()
|
||||
|
|
@ -60,7 +60,7 @@ If you run the tests::
|
|||
test_module.py:5: AssertionError
|
||||
________________________________ test_noop _________________________________
|
||||
|
||||
smtp = <smtplib.SMTP instance at 0x20ba7e8>
|
||||
smtp = <smtplib.SMTP instance at 0x2c0f170>
|
||||
|
||||
def test_noop(smtp):
|
||||
response = smtp.noop()
|
||||
|
|
@ -69,7 +69,7 @@ If you run the tests::
|
|||
E assert 0
|
||||
|
||||
test_module.py:10: AssertionError
|
||||
2 failed in 0.27 seconds
|
||||
2 failed in 0.21 seconds
|
||||
|
||||
you will see the two ``assert 0`` failing and can see that
|
||||
the same (session-scoped) object was passed into the two test functions.
|
||||
|
|
@ -98,9 +98,31 @@ another run::
|
|||
collecting ... collected 4 items
|
||||
FFFF
|
||||
================================= FAILURES =================================
|
||||
________________________ test_ehlo[mail.python.org] ________________________
|
||||
|
||||
smtp = <smtplib.SMTP instance at 0x20fca70>
|
||||
|
||||
def test_ehlo(smtp):
|
||||
response = smtp.ehlo()
|
||||
assert response[0] == 250
|
||||
> assert "merlinux" in response[1]
|
||||
E assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
|
||||
|
||||
test_module.py:4: AssertionError
|
||||
________________________ test_noop[mail.python.org] ________________________
|
||||
|
||||
smtp = <smtplib.SMTP instance at 0x20fca70>
|
||||
|
||||
def test_noop(smtp):
|
||||
response = smtp.noop()
|
||||
assert response[0] == 250
|
||||
> assert 0 # for demo purposes
|
||||
E assert 0
|
||||
|
||||
test_module.py:10: AssertionError
|
||||
__________________________ test_ehlo[merlinux.eu] __________________________
|
||||
|
||||
smtp = <smtplib.SMTP instance at 0x2a51830>
|
||||
smtp = <smtplib.SMTP instance at 0x2104bd8>
|
||||
|
||||
def test_ehlo(smtp):
|
||||
response = smtp.ehlo()
|
||||
|
|
@ -110,20 +132,9 @@ another run::
|
|||
E assert 0
|
||||
|
||||
test_module.py:5: AssertionError
|
||||
________________________ test_ehlo[mail.python.org] ________________________
|
||||
|
||||
smtp = <smtplib.SMTP instance at 0x2a56c20>
|
||||
|
||||
def test_ehlo(smtp):
|
||||
response = smtp.ehlo()
|
||||
assert response[0] == 250
|
||||
> assert "merlinux" in response[1]
|
||||
E assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
|
||||
|
||||
test_module.py:4: AssertionError
|
||||
__________________________ test_noop[merlinux.eu] __________________________
|
||||
|
||||
smtp = <smtplib.SMTP instance at 0x2a51830>
|
||||
smtp = <smtplib.SMTP instance at 0x2104bd8>
|
||||
|
||||
def test_noop(smtp):
|
||||
response = smtp.noop()
|
||||
|
|
@ -132,20 +143,7 @@ another run::
|
|||
E assert 0
|
||||
|
||||
test_module.py:10: AssertionError
|
||||
________________________ test_noop[mail.python.org] ________________________
|
||||
|
||||
smtp = <smtplib.SMTP instance at 0x2a56c20>
|
||||
|
||||
def test_noop(smtp):
|
||||
response = smtp.noop()
|
||||
assert response[0] == 250
|
||||
> assert 0 # for demo purposes
|
||||
E assert 0
|
||||
|
||||
test_module.py:10: AssertionError
|
||||
4 failed in 6.91 seconds
|
||||
closing <smtplib.SMTP instance at 0x2a56c20>
|
||||
closing <smtplib.SMTP instance at 0x2a51830>
|
||||
4 failed in 6.48 seconds
|
||||
|
||||
We get four failures because we are running the two tests twice with
|
||||
different ``smtp`` instantiations as defined on the factory.
|
||||
|
|
@ -157,14 +155,14 @@ You can look at what tests pytest collects without running them::
|
|||
|
||||
$ py.test --collectonly
|
||||
=========================== test session starts ============================
|
||||
platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev4
|
||||
platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev5
|
||||
plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
|
||||
collecting ... collected 4 items
|
||||
<Module 'test_module.py'>
|
||||
<Function 'test_ehlo[merlinux.eu]'>
|
||||
<Function 'test_ehlo[mail.python.org]'>
|
||||
<Function 'test_noop[merlinux.eu]'>
|
||||
<Function 'test_noop[mail.python.org]'>
|
||||
<Function 'test_ehlo[merlinux.eu]'>
|
||||
<Function 'test_noop[merlinux.eu]'>
|
||||
|
||||
============================= in 0.02 seconds =============================
|
||||
|
||||
|
|
@ -174,13 +172,13 @@ And you can run without output capturing and minimized failure reporting to chec
|
|||
collecting ... collected 4 items
|
||||
FFFF
|
||||
================================= FAILURES =================================
|
||||
/home/hpk/tmp/doc-exec-361/test_module.py:5: assert 0
|
||||
/home/hpk/tmp/doc-exec-361/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
|
||||
/home/hpk/tmp/doc-exec-361/test_module.py:10: assert 0
|
||||
/home/hpk/tmp/doc-exec-361/test_module.py:10: assert 0
|
||||
4 failed in 6.83 seconds
|
||||
closing <smtplib.SMTP instance at 0x236da28>
|
||||
closing <smtplib.SMTP instance at 0x23687e8>
|
||||
/home/hpk/tmp/doc-exec-386/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
|
||||
/home/hpk/tmp/doc-exec-386/test_module.py:10: assert 0
|
||||
/home/hpk/tmp/doc-exec-386/test_module.py:5: assert 0
|
||||
/home/hpk/tmp/doc-exec-386/test_module.py:10: assert 0
|
||||
4 failed in 6.45 seconds
|
||||
closing <smtplib.SMTP instance at 0x29d0a28>
|
||||
closing <smtplib.SMTP instance at 0x29d8878>
|
||||
|
||||
.. _`new_setup`:
|
||||
|
||||
|
|
@ -217,8 +215,9 @@ And the test file contains a setup function using this resource::
|
|||
# content of test_module.py
|
||||
import pytest
|
||||
|
||||
@pytest.mark.setup(scope="function")
|
||||
@pytest.mark.setup(scope="module")
|
||||
def setresource(resource):
|
||||
print "setupresource", resource
|
||||
global myresource
|
||||
myresource = resource
|
||||
|
||||
|
|
@ -236,10 +235,11 @@ Let's run this module::
|
|||
collecting ... collected 2 items
|
||||
..
|
||||
2 passed in 0.24 seconds
|
||||
created resource /home/hpk/tmp/pytest-3715/test_10
|
||||
using myresource /home/hpk/tmp/pytest-3715/test_10
|
||||
using myresource /home/hpk/tmp/pytest-3715/test_10
|
||||
finalize /home/hpk/tmp/pytest-3715/test_10
|
||||
created resource /home/hpk/tmp/pytest-3875/test_10
|
||||
setupresource /home/hpk/tmp/pytest-3875/test_10
|
||||
using myresource /home/hpk/tmp/pytest-3875/test_10
|
||||
using myresource /home/hpk/tmp/pytest-3875/test_10
|
||||
finalize /home/hpk/tmp/pytest-3875/test_10
|
||||
|
||||
The two test functions will see the same resource instance because it has
|
||||
a module life cycle or scope.
|
||||
|
|
@ -264,23 +264,22 @@ Running this will run four tests::
|
|||
$ py.test -qs
|
||||
collecting ... collected 4 items
|
||||
....
|
||||
4 passed in 0.24 seconds
|
||||
created resource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa
|
||||
using myresource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa
|
||||
created resource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb
|
||||
using myresource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb
|
||||
using myresource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa
|
||||
using myresource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb
|
||||
finalize /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb
|
||||
finalize /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa
|
||||
4 passed in 0.25 seconds
|
||||
created resource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
|
||||
setupresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
|
||||
using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
|
||||
using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
|
||||
finalize /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
|
||||
created resource /home/hpk/tmp/pytest-3876/test_1_bbb_0/bbb
|
||||
using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
|
||||
using myresource /home/hpk/tmp/pytest-3876/test_1_aaa_0/aaa
|
||||
finalize /home/hpk/tmp/pytest-3876/test_1_bbb_0/bbb
|
||||
|
||||
Each parameter causes the creation of a respective resource and the
|
||||
unchanged test module uses it in its ``@setup`` decorated method.
|
||||
|
||||
.. note::
|
||||
|
||||
Currently, parametrized tests are sorted by test function location
|
||||
so a test function will execute multiple times with different parametrized
|
||||
funcargs. If you have class/module/session scoped funcargs and
|
||||
they cause global side effects this can cause problems because the
|
||||
code under test may not be prepared to deal with it.
|
||||
Parametrized Resources will be grouped together during test execution.
|
||||
Moreover, any added finalizers will be run before the next parametrized
|
||||
resource is being setup.
|
||||
|
|
|
|||
|
|
@ -51,6 +51,8 @@ implementation or backward compatibility issues. The main changes are:
|
|||
troubles than the current @setup approach which can share
|
||||
a lot of logic with the @funcarg one.
|
||||
|
||||
* tests are grouped by any parametrized resource
|
||||
|
||||
.. currentmodule:: _pytest
|
||||
|
||||
|
||||
|
|
@ -252,58 +254,82 @@ a lot of setup-information and thus presents a nice method to get an
|
|||
overview of resource management in your project.
|
||||
|
||||
|
||||
Sorting tests by funcarg scopes
|
||||
-------------------------------------------
|
||||
Grouping tests by resource parameters
|
||||
----------------------------------------------------------
|
||||
|
||||
.. note:: Not implemented, Under consideration.
|
||||
.. note:: Implemented.
|
||||
|
||||
pytest by default sorts test items by their source location.
|
||||
For class/module/session scoped funcargs it is not always
|
||||
desirable to have multiple active funcargs. Sometimes,
|
||||
the application under test may not even be able to handle it
|
||||
because it relies on global state/side effects related to those
|
||||
resources.
|
||||
pytest usually sorts test items by their source location.
|
||||
With pytest-2.X tests are first grouped by resource parameters.
|
||||
If you have a parametrized resource, then all the tests using it
|
||||
will first execute with it. Then any finalizers are called and then
|
||||
the next parametrized resource instance is created and its tests are run.
|
||||
Among other things, this allows to have per-session parametrized setups
|
||||
including ones which affect global state of an application.
|
||||
|
||||
Therefore, pytest-2.3 tries to minimize the number of active
|
||||
resources and re-orders test items accordingly. Consider the following
|
||||
example::
|
||||
The following example uses two parametrized funcargs, one of which is
|
||||
scoped on a per-module basis::
|
||||
|
||||
# content of test_module.py
|
||||
import pytest
|
||||
|
||||
@pytest.mark.funcarg(scope="module", params=["mod1", "mod2"])
|
||||
def modarg(request):
|
||||
param = request.param
|
||||
print "create", param
|
||||
def fin():
|
||||
print "fin", param
|
||||
request.addfinalizer(fin)
|
||||
return param
|
||||
|
||||
@pytest.mark.funcarg(scope="module", params=[1,2])
|
||||
def arg(request):
|
||||
...
|
||||
@pytest.mark.funcarg(scope="function", params=[1,2])
|
||||
def otherarg(request):
|
||||
...
|
||||
return request.param
|
||||
|
||||
def test_0(otherarg):
|
||||
pass
|
||||
def test_1(arg):
|
||||
pass
|
||||
def test_2(arg, otherarg):
|
||||
pass
|
||||
print " test0", otherarg
|
||||
def test_1(modarg):
|
||||
print " test1", modarg
|
||||
def test_2(otherarg, modarg):
|
||||
print " test2", otherarg, modarg
|
||||
|
||||
if arg.1, arg.2, otherarg.1, otherarg.2 denote the respective
|
||||
parametrized funcarg instances this will re-order test
|
||||
execution like follows::
|
||||
If you run the tests in verbose mode and with looking at captured output::
|
||||
|
||||
test_0(otherarg.1)
|
||||
test_0(otherarg.2)
|
||||
test_1(arg.1)
|
||||
test_2(arg.1, otherarg.1)
|
||||
test_2(arg.1, otherarg.2)
|
||||
test_1(arg.2)
|
||||
test_2(arg.2, otherarg.1)
|
||||
test_2(arg.2, otherarg.2)
|
||||
$ py.test -v -s
|
||||
=========================== test session starts ============================
|
||||
platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev5 -- /home/hpk/venv/1/bin/python
|
||||
cachedir: /home/hpk/tmp/doc-exec-382/.cache
|
||||
plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
|
||||
collecting ... collected 8 items
|
||||
|
||||
test_module.py:16: test_0[1] PASSED
|
||||
test_module.py:16: test_0[2] PASSED
|
||||
test_module.py:18: test_1[mod1] PASSED
|
||||
test_module.py:20: test_2[1-mod1] PASSED
|
||||
test_module.py:20: test_2[2-mod1] PASSED
|
||||
test_module.py:18: test_1[mod2] PASSED
|
||||
test_module.py:20: test_2[1-mod2] PASSED
|
||||
test_module.py:20: test_2[2-mod2] PASSED
|
||||
|
||||
========================= 8 passed in 0.03 seconds =========================
|
||||
test0 1
|
||||
test0 2
|
||||
create mod1
|
||||
test1 mod1
|
||||
test2 1 mod1
|
||||
test2 2 mod1
|
||||
fin mod1
|
||||
create mod2
|
||||
test1 mod2
|
||||
test2 1 mod2
|
||||
test2 2 mod2
|
||||
fin mod2
|
||||
|
||||
Moreover, test_2(arg.1) will execute any registered teardowns for
|
||||
the arg.1 resource after the test finished execution.
|
||||
You can see that that the parametrized ``modarg`` resource lead to
|
||||
a re-ordering of test execution. Moreover, the finalizer for the
|
||||
"mod1" parametrized resource was executed before the "mod2" resource
|
||||
was setup with a different parameter.
|
||||
|
||||
.. note::
|
||||
|
||||
XXX it's quite unclear at the moment how to implement.
|
||||
If we have a 1000 tests requiring different sets of parametrized
|
||||
resources with different scopes, how to re-order accordingly?
|
||||
It even seems difficult to express the expectation in a
|
||||
concise manner.
|
||||
|
||||
|
||||
The current implementation is experimental.
|
||||
|
|
|
|||
2
setup.py
2
setup.py
|
|
@ -24,7 +24,7 @@ def main():
|
|||
name='pytest',
|
||||
description='py.test: simple powerful testing with Python',
|
||||
long_description = long_description,
|
||||
version='2.3.0.dev4',
|
||||
version='2.3.0.dev5',
|
||||
url='http://pytest.org',
|
||||
license='MIT license',
|
||||
platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'],
|
||||
|
|
|
|||
|
|
@ -1695,7 +1695,7 @@ class TestResourceIntegrationFunctional:
|
|||
""")
|
||||
result = testdir.runpytest("-v")
|
||||
assert result.ret == 1
|
||||
result.stdout.fnmatch_lines([
|
||||
result.stdout.fnmatch_lines_random([
|
||||
"*test_function*basic*PASSED",
|
||||
"*test_function*advanced*FAILED",
|
||||
])
|
||||
|
|
@ -1849,10 +1849,10 @@ class TestSetupManagement:
|
|||
pass
|
||||
|
||||
def test_result(arg):
|
||||
assert len(l) == 2
|
||||
assert l == [1,2]
|
||||
assert len(l) == arg
|
||||
assert l[:arg] == [1,2][:arg]
|
||||
""")
|
||||
reprec = testdir.inline_run("-s")
|
||||
reprec = testdir.inline_run("-v", "-s")
|
||||
reprec.assertoutcome(passed=4)
|
||||
|
||||
class TestFuncargMarker:
|
||||
|
|
@ -2013,3 +2013,170 @@ class TestFuncargMarker:
|
|||
""")
|
||||
reprec = testdir.inline_run()
|
||||
reprec.assertoutcome(passed=4)
|
||||
|
||||
def test_scope_mismatch(self, testdir):
|
||||
testdir.makeconftest("""
|
||||
import pytest
|
||||
@pytest.mark.funcarg(scope="function")
|
||||
def arg(request):
|
||||
pass
|
||||
""")
|
||||
testdir.makepyfile("""
|
||||
import pytest
|
||||
@pytest.mark.funcarg(scope="session")
|
||||
def arg(request, arg):
|
||||
pass
|
||||
def test_mismatch(arg):
|
||||
pass
|
||||
""")
|
||||
result = testdir.runpytest()
|
||||
result.stdout.fnmatch_lines([
|
||||
"*ScopeMismatch*",
|
||||
"*1 error*",
|
||||
])
|
||||
|
||||
def test_parametrize_separated_order(self, testdir):
|
||||
testdir.makepyfile("""
|
||||
import pytest
|
||||
|
||||
@pytest.mark.funcarg(scope="module", params=[1, 2])
|
||||
def arg(request):
|
||||
return request.param
|
||||
|
||||
l = []
|
||||
def test_1(arg):
|
||||
l.append(arg)
|
||||
def test_2(arg):
|
||||
l.append(arg)
|
||||
def test_3():
|
||||
assert len(l) == 4
|
||||
assert l[0] == l[1]
|
||||
assert l[2] == l[3]
|
||||
|
||||
""")
|
||||
reprec = testdir.inline_run("-v")
|
||||
reprec.assertoutcome(passed=5)
|
||||
|
||||
def test_parametrize_separated_order_higher_scope_first(self, testdir):
|
||||
testdir.makepyfile("""
|
||||
import pytest
|
||||
|
||||
@pytest.mark.funcarg(scope="function", params=[1, 2])
|
||||
def arg(request):
|
||||
param = request.param
|
||||
request.addfinalizer(lambda: l.append("fin:%s" % param))
|
||||
l.append("create:%s" % param)
|
||||
return request.param
|
||||
|
||||
@pytest.mark.funcarg(scope="module", params=["mod1", "mod2"])
|
||||
def modarg(request):
|
||||
param = request.param
|
||||
request.addfinalizer(lambda: l.append("fin:%s" % param))
|
||||
l.append("create:%s" % param)
|
||||
return request.param
|
||||
|
||||
l = []
|
||||
def test_1(arg):
|
||||
l.append("test1")
|
||||
def test_2(modarg):
|
||||
l.append("test2")
|
||||
def test_3(arg, modarg):
|
||||
l.append("test3")
|
||||
def test_4(modarg, arg):
|
||||
l.append("test4")
|
||||
def test_5():
|
||||
assert len(l) == 12 * 3
|
||||
import pprint
|
||||
pprint.pprint(l)
|
||||
assert l == [
|
||||
'create:1', 'test1', 'fin:1',
|
||||
'create:2', 'test1', 'fin:2',
|
||||
'create:mod1', 'test2', 'create:1', 'test3', 'fin:1',
|
||||
'create:1', 'test4', 'fin:1', 'create:2', 'test3', 'fin:2',
|
||||
'create:2', 'test4', 'fin:mod1', 'fin:2',
|
||||
|
||||
'create:mod2', 'test2', 'create:1', 'test3', 'fin:1',
|
||||
'create:1', 'test4', 'fin:1', 'create:2', 'test3', 'fin:2',
|
||||
'create:2', 'test4', 'fin:mod2', 'fin:2',
|
||||
]
|
||||
|
||||
""")
|
||||
reprec = testdir.inline_run("-v")
|
||||
reprec.assertoutcome(passed=12+1)
|
||||
|
||||
def test_parametrize_separated_lifecycle(self, testdir):
|
||||
testdir.makepyfile("""
|
||||
import pytest
|
||||
|
||||
@pytest.mark.funcarg(scope="module", params=[1, 2])
|
||||
def arg(request):
|
||||
x = request.param
|
||||
request.addfinalizer(lambda: l.append("fin%s" % x))
|
||||
return request.param
|
||||
|
||||
l = []
|
||||
def test_1(arg):
|
||||
l.append(arg)
|
||||
def test_2(arg):
|
||||
l.append(arg)
|
||||
def test_3():
|
||||
assert len(l) == 6
|
||||
assert l[0] == l[1]
|
||||
assert l[2] == "fin1"
|
||||
assert l[3] == l[4]
|
||||
assert l[5] == "fin2"
|
||||
|
||||
""")
|
||||
reprec = testdir.inline_run("-v")
|
||||
reprec.assertoutcome(passed=5)
|
||||
|
||||
def test_parametrize_function_scoped_finalizers_called(self, testdir):
|
||||
testdir.makepyfile("""
|
||||
import pytest
|
||||
|
||||
@pytest.mark.funcarg(scope="function", params=[1, 2])
|
||||
def arg(request):
|
||||
x = request.param
|
||||
request.addfinalizer(lambda: l.append("fin%s" % x))
|
||||
return request.param
|
||||
|
||||
l = []
|
||||
def test_1(arg):
|
||||
l.append(arg)
|
||||
def test_2(arg):
|
||||
l.append(arg)
|
||||
def test_3():
|
||||
assert len(l) == 8
|
||||
assert l == [1, "fin1", 1, "fin1", 2, "fin2", 2, "fin2"]
|
||||
""")
|
||||
reprec = testdir.inline_run("-v")
|
||||
reprec.assertoutcome(passed=5)
|
||||
|
||||
def test_parametrize_setup_function(self, testdir):
|
||||
testdir.makepyfile("""
|
||||
import pytest
|
||||
|
||||
@pytest.mark.funcarg(scope="module", params=[1, 2])
|
||||
def arg(request):
|
||||
return request.param
|
||||
|
||||
@pytest.mark.setup(scope="module")
|
||||
def mysetup(request, arg):
|
||||
request.addfinalizer(lambda: l.append("fin%s" % arg))
|
||||
l.append("setup%s" % arg)
|
||||
|
||||
l = []
|
||||
def test_1(arg):
|
||||
l.append(arg)
|
||||
def test_2(arg):
|
||||
l.append(arg)
|
||||
def test_3():
|
||||
import pprint
|
||||
pprint.pprint(l)
|
||||
assert l == ["setup1", 1, 1, "fin1",
|
||||
"setup2", 2, 2, "fin2",]
|
||||
|
||||
""")
|
||||
reprec = testdir.inline_run("-v")
|
||||
reprec.assertoutcome(passed=5)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue