reshuffle docs, try to get a bit closer to release-relevant documentation
This commit is contained in:
holger krekel
parent
535d892f27
commit
5fd84c35dd
|
|
@ -1,2 +1,2 @@
|
||||||
#
|
#
|
||||||
__version__ = '2.3.0.dev6'
|
__version__ = '2.3.0.dev7'
|
||||||
|
|
|
||||||
|
|
@ -1355,7 +1355,8 @@ scope2props["function"] = scope2props["class"] + ("function", "keywords")
|
||||||
|
|
||||||
def scopeprop(attr, name=None, doc=None):
|
def scopeprop(attr, name=None, doc=None):
|
||||||
if doc is None:
|
if doc is None:
|
||||||
doc = "%s of underlying test context" % (attr,)
|
doc = ("%s of underlying test context, may not exist "
|
||||||
|
"if the testcontext has a higher scope" % (attr,))
|
||||||
name = name or attr
|
name = name or attr
|
||||||
def get(self):
|
def get(self):
|
||||||
if name in scope2props[self.scope]:
|
if name in scope2props[self.scope]:
|
||||||
|
|
@ -1370,6 +1371,7 @@ def rprop(attr, doc=None):
|
||||||
return property(lambda x: getattr(x._request, attr), doc=doc)
|
return property(lambda x: getattr(x._request, attr), doc=doc)
|
||||||
|
|
||||||
class TestContext(object):
|
class TestContext(object):
|
||||||
|
""" Basic objects of the current testing context. """
|
||||||
def __init__(self, request, scope):
|
def __init__(self, request, scope):
|
||||||
self._request = request
|
self._request = request
|
||||||
self.scope = scope
|
self.scope = scope
|
||||||
|
|
@ -1379,7 +1381,6 @@ class TestContext(object):
|
||||||
|
|
||||||
config = rprop("config", "pytest config object.")
|
config = rprop("config", "pytest config object.")
|
||||||
session = rprop("session", "pytest session object.")
|
session = rprop("session", "pytest session object.")
|
||||||
param = rprop("param")
|
|
||||||
|
|
||||||
function = scopeprop("function")
|
function = scopeprop("function")
|
||||||
module = scopeprop("module")
|
module = scopeprop("module")
|
||||||
|
|
@ -1400,6 +1401,8 @@ class TestContextSetup(TestContext):
|
||||||
self._setupcall.addfinalizer(finalizer)
|
self._setupcall.addfinalizer(finalizer)
|
||||||
|
|
||||||
class TestContextResource(TestContext):
|
class TestContextResource(TestContext):
|
||||||
|
param = rprop("param")
|
||||||
|
|
||||||
def __init__(self, request):
|
def __init__(self, request):
|
||||||
super(TestContextResource, self).__init__(request, request.scope)
|
super(TestContextResource, self).__init__(request, request.scope)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -17,10 +17,9 @@ to get an overview on the globally available helpers.
|
||||||
.. automodule:: pytest
|
.. automodule:: pytest
|
||||||
:members:
|
:members:
|
||||||
|
|
||||||
|
.. _builtinresources:
|
||||||
|
|
||||||
.. _builtinfuncargs:
|
Builtin resources / function arguments
|
||||||
|
|
||||||
Builtin function arguments
|
|
||||||
-----------------------------------------------------
|
-----------------------------------------------------
|
||||||
|
|
||||||
You can ask for available builtin or project-custom
|
You can ask for available builtin or project-custom
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@
|
||||||
#
|
#
|
||||||
# The full version, including alpha/beta/rc tags.
|
# The full version, including alpha/beta/rc tags.
|
||||||
# The short X.Y version.
|
# The short X.Y version.
|
||||||
version = release = "2.3.0.dev3"
|
version = release = "2.3.0.dev6"
|
||||||
|
|
||||||
import sys, os
|
import sys, os
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,289 +0,0 @@
|
||||||
|
|
||||||
Scoping and parametrizing Funcarg factories
|
|
||||||
---------------------------------------------------
|
|
||||||
|
|
||||||
.. regendoc:wipe
|
|
||||||
|
|
||||||
.. versionadded:: 2.3
|
|
||||||
|
|
||||||
The ``@pytest.mark.funcarg`` marker allows
|
|
||||||
|
|
||||||
* to mark a function without a ``pytest_funcarg__`` as a factory
|
|
||||||
* to cause parametrization and run all tests multiple times
|
|
||||||
with the multiple created resources
|
|
||||||
* to set a scope which determines the level of caching
|
|
||||||
|
|
||||||
Here is a simple example for defining a SMTPServer server
|
|
||||||
object with a session scope::
|
|
||||||
|
|
||||||
# content of conftest.py
|
|
||||||
import pytest
|
|
||||||
import smtplib
|
|
||||||
|
|
||||||
@pytest.mark.funcarg(scope="session")
|
|
||||||
def smtp(request):
|
|
||||||
smtp = smtplib.SMTP("merlinux.eu")
|
|
||||||
request.addfinalizer(smtp.close)
|
|
||||||
return smtp
|
|
||||||
|
|
||||||
You can now use this server connection from your tests::
|
|
||||||
|
|
||||||
# content of test_module.py
|
|
||||||
def test_ehlo(smtp):
|
|
||||||
response = smtp.ehlo()
|
|
||||||
assert response[0] == 250
|
|
||||||
assert "merlinux" in response[1]
|
|
||||||
assert 0 # for demo purposes
|
|
||||||
|
|
||||||
def test_noop(smtp):
|
|
||||||
response = smtp.noop()
|
|
||||||
assert response[0] == 250
|
|
||||||
assert 0 # for demo purposes
|
|
||||||
|
|
||||||
If you run the tests::
|
|
||||||
|
|
||||||
$ py.test -q
|
|
||||||
collecting ... collected 2 items
|
|
||||||
FF
|
|
||||||
================================= FAILURES =================================
|
|
||||||
________________________________ test_ehlo _________________________________
|
|
||||||
|
|
||||||
smtp = <smtplib.SMTP instance at 0x2b8ebd8>
|
|
||||||
|
|
||||||
def test_ehlo(smtp):
|
|
||||||
response = smtp.ehlo()
|
|
||||||
assert response[0] == 250
|
|
||||||
assert "merlinux" in response[1]
|
|
||||||
> assert 0 # for demo purposes
|
|
||||||
E assert 0
|
|
||||||
|
|
||||||
test_module.py:5: AssertionError
|
|
||||||
________________________________ test_noop _________________________________
|
|
||||||
|
|
||||||
smtp = <smtplib.SMTP instance at 0x2b8ebd8>
|
|
||||||
|
|
||||||
def test_noop(smtp):
|
|
||||||
response = smtp.noop()
|
|
||||||
assert response[0] == 250
|
|
||||||
> assert 0 # for demo purposes
|
|
||||||
E assert 0
|
|
||||||
|
|
||||||
test_module.py:10: AssertionError
|
|
||||||
2 failed in 0.26 seconds
|
|
||||||
|
|
||||||
you will see the two ``assert 0`` failing and can see that
|
|
||||||
the same (session-scoped) object was passed into the two test functions.
|
|
||||||
|
|
||||||
If you now want to test multiple servers you can simply parametrize
|
|
||||||
the ``smtp`` factory::
|
|
||||||
|
|
||||||
# content of conftest.py
|
|
||||||
import pytest
|
|
||||||
import smtplib
|
|
||||||
|
|
||||||
@pytest.mark.funcarg(scope="session",
|
|
||||||
params=["merlinux.eu", "mail.python.org"])
|
|
||||||
def smtp(request):
|
|
||||||
smtp = smtplib.SMTP(request.param)
|
|
||||||
def fin():
|
|
||||||
print "closing", smtp
|
|
||||||
smtp.close()
|
|
||||||
request.addfinalizer(fin)
|
|
||||||
return smtp
|
|
||||||
|
|
||||||
Only two lines changed and no test code needs to change. Let's do
|
|
||||||
another run::
|
|
||||||
|
|
||||||
$ py.test -q
|
|
||||||
collecting ... collected 4 items
|
|
||||||
FFFF
|
|
||||||
================================= FAILURES =================================
|
|
||||||
__________________________ test_ehlo[merlinux.eu] __________________________
|
|
||||||
|
|
||||||
smtp = <smtplib.SMTP instance at 0x2ee5200>
|
|
||||||
|
|
||||||
def test_ehlo(smtp):
|
|
||||||
response = smtp.ehlo()
|
|
||||||
assert response[0] == 250
|
|
||||||
assert "merlinux" in response[1]
|
|
||||||
> assert 0 # for demo purposes
|
|
||||||
E assert 0
|
|
||||||
|
|
||||||
test_module.py:5: AssertionError
|
|
||||||
__________________________ test_noop[merlinux.eu] __________________________
|
|
||||||
|
|
||||||
smtp = <smtplib.SMTP instance at 0x2ee5200>
|
|
||||||
|
|
||||||
def test_noop(smtp):
|
|
||||||
response = smtp.noop()
|
|
||||||
assert response[0] == 250
|
|
||||||
> assert 0 # for demo purposes
|
|
||||||
E assert 0
|
|
||||||
|
|
||||||
test_module.py:10: AssertionError
|
|
||||||
________________________ test_ehlo[mail.python.org] ________________________
|
|
||||||
|
|
||||||
smtp = <smtplib.SMTP instance at 0x2eee5a8>
|
|
||||||
|
|
||||||
def test_ehlo(smtp):
|
|
||||||
response = smtp.ehlo()
|
|
||||||
assert response[0] == 250
|
|
||||||
> assert "merlinux" in response[1]
|
|
||||||
E assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
|
|
||||||
|
|
||||||
test_module.py:4: AssertionError
|
|
||||||
________________________ test_noop[mail.python.org] ________________________
|
|
||||||
|
|
||||||
smtp = <smtplib.SMTP instance at 0x2eee5a8>
|
|
||||||
|
|
||||||
def test_noop(smtp):
|
|
||||||
response = smtp.noop()
|
|
||||||
assert response[0] == 250
|
|
||||||
> assert 0 # for demo purposes
|
|
||||||
E assert 0
|
|
||||||
|
|
||||||
test_module.py:10: AssertionError
|
|
||||||
4 failed in 6.94 seconds
|
|
||||||
|
|
||||||
We get four failures because we are running the two tests twice with
|
|
||||||
different ``smtp`` instantiations as defined on the factory.
|
|
||||||
Note that with the ``mail.python.org`` connection the second tests
|
|
||||||
fails already in ``test_ehlo`` because it wrongly expects a specific
|
|
||||||
server string.
|
|
||||||
|
|
||||||
You can look at what tests pytest collects without running them::
|
|
||||||
|
|
||||||
$ py.test --collectonly
|
|
||||||
=========================== test session starts ============================
|
|
||||||
platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev5
|
|
||||||
plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
|
|
||||||
collecting ... collected 4 items
|
|
||||||
<Module 'test_module.py'>
|
|
||||||
<Function 'test_ehlo[merlinux.eu]'>
|
|
||||||
<Function 'test_noop[merlinux.eu]'>
|
|
||||||
<Function 'test_ehlo[mail.python.org]'>
|
|
||||||
<Function 'test_noop[mail.python.org]'>
|
|
||||||
|
|
||||||
============================= in 0.02 seconds =============================
|
|
||||||
|
|
||||||
And you can run without output capturing and minimized failure reporting to check that the ``smtp`` objects are finalized at session end::
|
|
||||||
|
|
||||||
$ py.test --tb=line -q -s
|
|
||||||
collecting ... collected 4 items
|
|
||||||
FFFF
|
|
||||||
================================= FAILURES =================================
|
|
||||||
/home/hpk/tmp/doc-exec-389/test_module.py:5: assert 0
|
|
||||||
/home/hpk/tmp/doc-exec-389/test_module.py:10: assert 0
|
|
||||||
/home/hpk/tmp/doc-exec-389/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
|
|
||||||
/home/hpk/tmp/doc-exec-389/test_module.py:10: assert 0
|
|
||||||
4 failed in 9.99 seconds
|
|
||||||
closing <smtplib.SMTP instance at 0x2a61560>
|
|
||||||
closing <smtplib.SMTP instance at 0x2a6b248>
|
|
||||||
|
|
||||||
.. _`new_setup`:
|
|
||||||
|
|
||||||
``@pytest.mark.setup``: xUnit on steroids
|
|
||||||
--------------------------------------------------------------------
|
|
||||||
|
|
||||||
.. regendoc:wipe
|
|
||||||
|
|
||||||
.. versionadded:: 2.3
|
|
||||||
|
|
||||||
The ``@pytest.mark.setup`` marker allows
|
|
||||||
|
|
||||||
* to define setup-functions close to test code or in conftest.py files
|
|
||||||
or plugins.
|
|
||||||
* to mark a function as a setup/fixture method; the function can itself
|
|
||||||
receive funcargs and will execute multiple times if the funcargs
|
|
||||||
are parametrized
|
|
||||||
* to set a scope which determines the level of caching and how often
|
|
||||||
the setup function is going to be called.
|
|
||||||
|
|
||||||
Here is a simple example which configures a global funcarg without
|
|
||||||
the test needing to have it in its signature::
|
|
||||||
|
|
||||||
# content of conftest.py
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
@pytest.mark.funcarg(scope="module")
|
|
||||||
def resource(request, tmpdir):
|
|
||||||
def fin():
|
|
||||||
print "finalize", tmpdir
|
|
||||||
request.addfinalizer(fin)
|
|
||||||
print "created resource", tmpdir
|
|
||||||
return tmpdir
|
|
||||||
|
|
||||||
And the test file contains a setup function using this resource::
|
|
||||||
|
|
||||||
# content of test_module.py
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
@pytest.mark.setup(scope="module")
|
|
||||||
def setresource(resource):
|
|
||||||
print "setupresource", resource
|
|
||||||
global myresource
|
|
||||||
myresource = resource
|
|
||||||
|
|
||||||
def test_1():
|
|
||||||
assert myresource
|
|
||||||
print "using myresource", myresource
|
|
||||||
|
|
||||||
def test_2():
|
|
||||||
assert myresource
|
|
||||||
print "using myresource", myresource
|
|
||||||
|
|
||||||
Let's run this module::
|
|
||||||
|
|
||||||
$ py.test -qs
|
|
||||||
collecting ... collected 2 items
|
|
||||||
..
|
|
||||||
2 passed in 0.62 seconds
|
|
||||||
created resource /home/hpk/tmp/pytest-4224/test_10
|
|
||||||
setupresource /home/hpk/tmp/pytest-4224/test_10
|
|
||||||
using myresource /home/hpk/tmp/pytest-4224/test_10
|
|
||||||
using myresource /home/hpk/tmp/pytest-4224/test_10
|
|
||||||
finalize /home/hpk/tmp/pytest-4224/test_10
|
|
||||||
|
|
||||||
The two test functions will see the same resource instance because it has
|
|
||||||
a module life cycle or scope.
|
|
||||||
|
|
||||||
The resource funcarg can later add parametrization without any test
|
|
||||||
or setup code needing to change::
|
|
||||||
|
|
||||||
# content of conftest.py
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
@pytest.mark.funcarg(scope="module", params=["aaa", "bbb"])
|
|
||||||
def resource(request, tmpdir):
|
|
||||||
newtmp = tmpdir.join(request.param)
|
|
||||||
def fin():
|
|
||||||
print "finalize", newtmp
|
|
||||||
request.addfinalizer(fin)
|
|
||||||
print "created resource", newtmp
|
|
||||||
return newtmp
|
|
||||||
|
|
||||||
Running this will run four tests::
|
|
||||||
|
|
||||||
$ py.test -qs
|
|
||||||
collecting ... collected 4 items
|
|
||||||
....
|
|
||||||
4 passed in 0.25 seconds
|
|
||||||
created resource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa
|
|
||||||
setupresource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa
|
|
||||||
using myresource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa
|
|
||||||
using myresource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa
|
|
||||||
finalize /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa
|
|
||||||
created resource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb
|
|
||||||
setupresource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb
|
|
||||||
using myresource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb
|
|
||||||
using myresource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb
|
|
||||||
finalize /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb
|
|
||||||
|
|
||||||
Each parameter causes the creation of a respective resource and the
|
|
||||||
unchanged test module uses it in its ``@setup`` decorated method.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
Parametrized Resources will be grouped together during test execution.
|
|
||||||
Moreover, any added finalizers will be run before the next parametrized
|
|
||||||
resource is being setup.
|
|
||||||
|
|
@ -329,87 +329,3 @@ Running it results in some skips if we don't have all the python interpreters in
|
||||||
========================= short test summary info ==========================
|
========================= short test summary info ==========================
|
||||||
SKIP [27] /home/hpk/p/pytest/doc/en/example/multipython.py:36: 'python2.8' not found
|
SKIP [27] /home/hpk/p/pytest/doc/en/example/multipython.py:36: 'python2.8' not found
|
||||||
48 passed, 27 skipped in 1.70 seconds
|
48 passed, 27 skipped in 1.70 seconds
|
||||||
|
|
||||||
.. regendoc:wipe
|
|
||||||
|
|
||||||
Grouping test execution by parameter
|
|
||||||
-----------------------------------------
|
|
||||||
|
|
||||||
By default pytest will execute test functions by executing all its parametrized invocations. If you rather want to group execution by parameter, you can
|
|
||||||
use something like the following ``conftest.py`` example. It uses
|
|
||||||
a parametrized "resource" object::
|
|
||||||
|
|
||||||
# content of conftest.py
|
|
||||||
def pytest_collection_modifyitems(items):
|
|
||||||
def cmp(item1, item2):
|
|
||||||
param1 = item1.callspec.getparam("resource")
|
|
||||||
param2 = item2.callspec.getparam("resource")
|
|
||||||
if param1 < param2:
|
|
||||||
return -1
|
|
||||||
elif param1 > param2:
|
|
||||||
return 1
|
|
||||||
return 0
|
|
||||||
items.sort(cmp=cmp)
|
|
||||||
|
|
||||||
def pytest_generate_tests(metafunc):
|
|
||||||
if "resource" in metafunc.funcargnames:
|
|
||||||
metafunc.parametrize("resource", [1,2], indirect=True)
|
|
||||||
|
|
||||||
class Resource:
|
|
||||||
def __init__(self, num):
|
|
||||||
self.num = num
|
|
||||||
def finalize(self):
|
|
||||||
print "finalize", self
|
|
||||||
|
|
||||||
def pytest_funcarg__resource(request):
|
|
||||||
return request.cached_setup(lambda: Resource(request.param),
|
|
||||||
teardown=lambda res: res.finalize(),
|
|
||||||
extrakey=request.param)
|
|
||||||
|
|
||||||
|
|
||||||
If you have a test file like this::
|
|
||||||
|
|
||||||
# content of test_resource.py
|
|
||||||
def test_hello(resource):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def test_world(resource):
|
|
||||||
pass
|
|
||||||
|
|
||||||
class TestClass:
|
|
||||||
def test_method1(self, resource):
|
|
||||||
pass
|
|
||||||
def test_method2(self, resource):
|
|
||||||
pass
|
|
||||||
|
|
||||||
then a subsequent execution will order the running of tests by
|
|
||||||
parameter value::
|
|
||||||
|
|
||||||
$ py.test -v -s
|
|
||||||
=========================== test session starts ============================
|
|
||||||
platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev3 -- /home/hpk/venv/1/bin/python
|
|
||||||
cachedir: /home/hpk/tmp/doc-exec-340/.cache
|
|
||||||
plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
|
|
||||||
collecting ... collected 8 items
|
|
||||||
|
|
||||||
test_resource.py:1: test_hello[1] PASSED
|
|
||||||
test_resource.py:4: test_world[1] PASSED
|
|
||||||
test_resource.py:8: TestClass.test_method1[1] PASSED
|
|
||||||
test_resource.py:10: TestClass.test_method2[1] PASSED
|
|
||||||
test_resource.py:1: test_hello[2] PASSED
|
|
||||||
test_resource.py:4: test_world[2] PASSED
|
|
||||||
test_resource.py:8: TestClass.test_method1[2] PASSED
|
|
||||||
test_resource.py:10: TestClass.test_method2[2] PASSED
|
|
||||||
|
|
||||||
========================= 8 passed in 0.03 seconds =========================
|
|
||||||
finalize <conftest.Resource instance at 0x2e7f878>
|
|
||||||
finalize <conftest.Resource instance at 0x2e7ddd0>
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
Despite the per-session ordering the finalize() of the session-scoped
|
|
||||||
resource executes at the end of the whole test session. The life
|
|
||||||
cycle of the two parametrized instantiated resources will thus overlap.
|
|
||||||
One possible workaround is to make the resource instantiations be
|
|
||||||
aware of each other and teardown the other one before returning a new
|
|
||||||
resource. There are plans for future releases of pytest to offer an
|
|
||||||
out-of-the-box way to support session-ordering.
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,228 @@
|
||||||
|
|
||||||
|
V5: changes to new resource/setup facilities
|
||||||
|
=============================================================
|
||||||
|
|
||||||
|
**Target audience**: Reading this document requires basic knowledge of
|
||||||
|
python testing, xUnit setup methods and the basic pytest funcarg mechanism,
|
||||||
|
see http://pytest.org/latest/funcargs.html
|
||||||
|
|
||||||
|
|
||||||
|
**Changes**: This V5 draft is based on incorporating and thinking about
|
||||||
|
feedback on previous versions provided by Floris Bruynooghe, Carl Meyer,
|
||||||
|
Ronny Pfannschmidt and Samuele Pedroni. I have also now implemented it
|
||||||
|
which triggered a number of refinements as well. The main changes are:
|
||||||
|
|
||||||
|
* Collapse funcarg factory decorators into a single "@resource" one.
|
||||||
|
You can specify scopes and params with it. When using the decorator
|
||||||
|
the "pytest_funcarg__" prefix is not allowed and the old-style
|
||||||
|
``request`` object cannot be received.
|
||||||
|
|
||||||
|
* funcarg resource factories can now use funcargs themselves
|
||||||
|
|
||||||
|
* Drop setup/directory scope from this draft
|
||||||
|
|
||||||
|
* introduce a new @setup decorator similar to the @funcarg one
|
||||||
|
except that setup-markers cannot define parametriation themselves.
|
||||||
|
Instead they can easily depend on a parametrized funcarg (which
|
||||||
|
must not be visible at test function signatures).
|
||||||
|
|
||||||
|
* drop consideration of setup_X support for funcargs because
|
||||||
|
it is less flexible and probably causes more implementation
|
||||||
|
troubles than the current @setup approach which can share
|
||||||
|
a lot of logic with the @funcarg one.
|
||||||
|
|
||||||
|
* tests are grouped by parametrized funcargs and according to scope
|
||||||
|
(sounds like a small thing but is a big deal)
|
||||||
|
|
||||||
|
* make the new-style funcargs/setup use a "testcontext" object
|
||||||
|
which offers test context info and addfinalizer() methods but no
|
||||||
|
getfuncargvalue()/cached_setup()/applymarker anymore. Reason
|
||||||
|
being that getfuncargvalue()/cached_setup breaks other features
|
||||||
|
such as sorting by resource-scope and parametrization
|
||||||
|
|
||||||
|
|
||||||
|
.. currentmodule:: _pytest
|
||||||
|
|
||||||
|
Shortcomings of the previous pytest_funcarg__ mechanism
|
||||||
|
---------------------------------------------------------
|
||||||
|
|
||||||
|
The previous funcarg mechanism calls a factory each time a
|
||||||
|
funcarg for a test function is testcontexted. If a factory wants
|
||||||
|
t re-use a resource across different scopes, it often used
|
||||||
|
the ``testcontext.cached_setup()`` helper to manage caching of
|
||||||
|
resources. Here is a basic example how we could implement
|
||||||
|
a per-session Database object::
|
||||||
|
|
||||||
|
# content of conftest.py
|
||||||
|
class Database:
|
||||||
|
def __init__(self):
|
||||||
|
print ("database instance created")
|
||||||
|
def destroy(self):
|
||||||
|
print ("database instance destroyed")
|
||||||
|
|
||||||
|
def pytest_funcarg__db(request):
|
||||||
|
return request.cached_setup(setup=DataBase,
|
||||||
|
teardown=lambda db: db.destroy,
|
||||||
|
scope="session")
|
||||||
|
|
||||||
|
There are several limitations and difficulties with this approach:
|
||||||
|
|
||||||
|
1. Scoping funcarg resource creation is not straight forward, instead one must
|
||||||
|
understand the intricate cached_setup() method mechanics.
|
||||||
|
|
||||||
|
2. parametrizing the "db" resource is not straight forward:
|
||||||
|
you need to apply a "parametrize" decorator or implement a
|
||||||
|
:py:func:`~hookspec.pytest_generate_tests` hook
|
||||||
|
calling :py:func:`~python.Metafunc.parametrize` which
|
||||||
|
performs parametrization at the places where the resource
|
||||||
|
is used. Moreover, you need to modify the factory to use an
|
||||||
|
``extrakey`` parameter containing ``request.param`` to the
|
||||||
|
:py:func:`~python.Request.cached_setup` call.
|
||||||
|
|
||||||
|
3. Multiple parametrized session-scoped resources will be active
|
||||||
|
at the same time, making it hard for them to affect global state
|
||||||
|
of the application under test.
|
||||||
|
|
||||||
|
4. there is no way how you can make use of funcarg factories
|
||||||
|
in xUnit setup methods.
|
||||||
|
|
||||||
|
5. A non-parametrized funcarg factory cannot use a parametrized
|
||||||
|
funcarg resource if it isn't stated in the test function signature.
|
||||||
|
|
||||||
|
All of these limitations are addressed with pytest-2.3 and its
|
||||||
|
new facilities.
|
||||||
|
|
||||||
|
Direct scoping of funcarg factories
|
||||||
|
--------------------------------------------------------
|
||||||
|
|
||||||
|
Instead of calling cached_setup(), you can decorate your factory
|
||||||
|
to state its scope::
|
||||||
|
|
||||||
|
@pytest.mark.resource(scope="session")
|
||||||
|
def db(testcontext):
|
||||||
|
# factory will only be invoked once per session -
|
||||||
|
db = DataBase()
|
||||||
|
testcontext.addfinalizer(db.destroy) # destroy when session is finished
|
||||||
|
return db
|
||||||
|
|
||||||
|
This factory implementation does not need to call ``cached_setup()`` anymore
|
||||||
|
because it will only be invoked once per session. Moreover, the
|
||||||
|
``testcontext.addfinalizer()`` registers a finalizer according to the specified
|
||||||
|
resource scope on which the factory function is operating. With this new
|
||||||
|
scoping, the still existing ``cached_setup()`` should be much less used
|
||||||
|
but will remain for compatibility reasons and for the case where you
|
||||||
|
still want to have your factory get called on a per-item basis.
|
||||||
|
|
||||||
|
|
||||||
|
Direct parametrization of funcarg resource factories
|
||||||
|
----------------------------------------------------------
|
||||||
|
|
||||||
|
.. note:: Implemented
|
||||||
|
|
||||||
|
Previously, funcarg factories could not directly cause parametrization.
|
||||||
|
You needed to specify a ``@parametrize`` or implement a ``pytest_generate_tests`` hook to perform parametrization, i.e. calling a test multiple times
|
||||||
|
with different value sets. pytest-2.X introduces a decorator for use
|
||||||
|
on the factory itself::
|
||||||
|
|
||||||
|
@pytest.mark.resource(params=["mysql", "pg"])
|
||||||
|
def pytest_funcarg__db(testcontext):
|
||||||
|
...
|
||||||
|
|
||||||
|
Here the factory will be invoked twice (with the respective "mysql"
|
||||||
|
and "pg" values set as ``testcontext.param`` attributes) and and all of
|
||||||
|
the tests requiring "db" will run twice as well. The "mysql" and
|
||||||
|
"pg" values will also be used for reporting the test-invocation variants.
|
||||||
|
|
||||||
|
This new way of parametrizing funcarg factories should in many cases
|
||||||
|
allow to re-use already written factories because effectively
|
||||||
|
``testcontext.param`` are already the parametrization attribute for test
|
||||||
|
functions/classes were parametrized via
|
||||||
|
:py:func:`~_pytest.python.Metafunc.parametrize(indirect=True)` calls.
|
||||||
|
|
||||||
|
Of course it's perfectly fine to combine parametrization and scoping::
|
||||||
|
|
||||||
|
@pytest.mark.resource(scope="session", params=["mysql", "pg"])
|
||||||
|
def pytest_funcarg__db(testcontext):
|
||||||
|
if testcontext.param == "mysql":
|
||||||
|
db = MySQL()
|
||||||
|
elif testcontext.param == "pg":
|
||||||
|
db = PG()
|
||||||
|
testcontext.addfinalizer(db.destroy) # destroy when session is finished
|
||||||
|
return db
|
||||||
|
|
||||||
|
This would execute all tests requiring the per-session "db" resource twice,
|
||||||
|
receiving the values created by the two respective invocations to the
|
||||||
|
factory function.
|
||||||
|
|
||||||
|
|
||||||
|
No ``pytest_funcarg__`` prefix when using @resource decorator
|
||||||
|
-------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
.. note:: Implemented
|
||||||
|
|
||||||
|
When using the ``@funcarg`` decorator the name of the function
|
||||||
|
does not need to (and in fact cannot) use the ``pytest_funcarg__``
|
||||||
|
naming::
|
||||||
|
|
||||||
|
@pytest.mark.resource
|
||||||
|
def db(testcontext):
|
||||||
|
...
|
||||||
|
|
||||||
|
The name under which the funcarg resource can be requested is ``db``.
|
||||||
|
|
||||||
|
You can also use the "old" non-decorator way of specifying funcarg factories
|
||||||
|
aka::
|
||||||
|
|
||||||
|
def pytest_funcarg__db(testcontext):
|
||||||
|
...
|
||||||
|
|
||||||
|
It is recommended to use the resource decorator, however.
|
||||||
|
|
||||||
|
|
||||||
|
solving per-session setup / the new @setup marker
|
||||||
|
--------------------------------------------------------------
|
||||||
|
|
||||||
|
.. note:: Implemented, at least working for basic situations.
|
||||||
|
|
||||||
|
pytest for a long time offered a pytest_configure and a pytest_sessionstart
|
||||||
|
hook which are often used to setup global resources. This suffers from
|
||||||
|
several problems:
|
||||||
|
|
||||||
|
1. in distributed testing the master process would setup test resources
|
||||||
|
that are never needed because it only co-ordinates the test run
|
||||||
|
activities of the slave processes.
|
||||||
|
|
||||||
|
2. if you only perform a collection (with "--collectonly")
|
||||||
|
resource-setup will still be executed.
|
||||||
|
|
||||||
|
3. If a pytest_sessionstart is contained in some subdirectories
|
||||||
|
conftest.py file, it will not be called. This stems from the
|
||||||
|
fact that this hook is actually used for reporting, in particular
|
||||||
|
the test-header with platform/custom information.
|
||||||
|
|
||||||
|
Moreover, it is today not easy to define a scoped setup from plugins or
|
||||||
|
conftest files other than to implement a ``pytest_runtest_setup()`` hook
|
||||||
|
and caring for scoping/caching yourself. And it's virtually impossible
|
||||||
|
to do this with parametrization as ``pytest_runtest_setup()`` is called
|
||||||
|
during test execution and parametrization happens at collection time.
|
||||||
|
|
||||||
|
It follows that pytest_configure/session/runtest_setup are often not
|
||||||
|
appropriate for implementing common fixture needs. Therefore,
|
||||||
|
pytest-2.X introduces a new "@pytest.mark.setup" marker which takes
|
||||||
|
an optional "scope" parameter.
|
||||||
|
|
||||||
|
See :ref:`new_setup` for examples.
|
||||||
|
|
||||||
|
funcarg and setup discovery now happens at collection time
|
||||||
|
---------------------------------------------------------------------
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
Partially implemented - collectonly shows no extra information however.
|
||||||
|
|
||||||
|
pytest-2.X takes care to discover funcarg factories and @setup methods
|
||||||
|
at collection time. This is more efficient especially for large test suites.
|
||||||
|
Moreover, a call to "py.test --collectonly" should be able to show
|
||||||
|
a lot of setup-information and thus presents a nice method to get an
|
||||||
|
overview of resource management in your project.
|
||||||
|
|
||||||
|
|
@ -11,6 +11,13 @@ Injecting objects into test functions (funcargs)
|
||||||
Dependency injection through function arguments
|
Dependency injection through function arguments
|
||||||
=================================================
|
=================================================
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
|
||||||
|
This section describes the pytest mechanisms prior
|
||||||
|
to the pytest-2.3 release. If you haven't used these
|
||||||
|
features yet, it makes more sense to stop here and read
|
||||||
|
:ref:`resources` instead.
|
||||||
|
|
||||||
py.test lets you inject objects into test invocations and precisely
|
py.test lets you inject objects into test invocations and precisely
|
||||||
control their life cycle in relation to the overall test execution. Moreover,
|
control their life cycle in relation to the overall test execution. Moreover,
|
||||||
you can run a test function multiple times injecting different objects.
|
you can run a test function multiple times injecting different objects.
|
||||||
|
|
@ -39,7 +46,7 @@ very useful if you want to test e.g. against different database backends
|
||||||
or with multiple numerical arguments sets and want to reuse the same set
|
or with multiple numerical arguments sets and want to reuse the same set
|
||||||
of test functions.
|
of test functions.
|
||||||
|
|
||||||
py.test comes with some :ref:`builtinfuncargs` and there are some refined usages in the examples section.
|
py.test comes with some :ref:`builtinresources` and there are some refined usages in the examples section.
|
||||||
|
|
||||||
.. _funcarg:
|
.. _funcarg:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -25,11 +25,11 @@ Welcome to pytest!
|
||||||
|
|
||||||
- **supports functional testing and complex test setups**
|
- **supports functional testing and complex test setups**
|
||||||
|
|
||||||
|
- (new in 2.3) :ref:`easy test resource management and generalized xUnit setup <resources>`
|
||||||
- (new in 2.2) :ref:`durations`
|
- (new in 2.2) :ref:`durations`
|
||||||
- (much improved in 2.2) :ref:`marking and test selection <mark>`
|
- (much improved in 2.2) :ref:`marking and test selection <mark>`
|
||||||
- (improved in 2.2) :ref:`parametrized test functions <parametrized test functions>`
|
- (improved in 2.2) :ref:`parametrized test functions <parametrized test functions>`
|
||||||
- advanced :ref:`skip and xfail`
|
- advanced :ref:`skip and xfail`
|
||||||
- unique :ref:`dependency injection through funcargs <funcargs>`
|
|
||||||
- can :ref:`distribute tests to multiple CPUs <xdistcpu>` through :ref:`xdist plugin <xdist>`
|
- can :ref:`distribute tests to multiple CPUs <xdistcpu>` through :ref:`xdist plugin <xdist>`
|
||||||
- can :ref:`continuously re-run failing tests <looponfailing>`
|
- can :ref:`continuously re-run failing tests <looponfailing>`
|
||||||
- many :ref:`builtin helpers <pytest helpers>`
|
- many :ref:`builtin helpers <pytest helpers>`
|
||||||
|
|
|
||||||
|
|
@ -1,271 +1,414 @@
|
||||||
|
|
||||||
V4: Creating and working with parametrized resources
|
.. _resources:
|
||||||
===============================================================
|
|
||||||
|
|
||||||
**Target audience**: Reading this document requires basic knowledge of
|
test resource management and xUnit setup (on steroids)
|
||||||
python testing, xUnit setup methods and the basic pytest funcarg mechanism,
|
=======================================================
|
||||||
see http://pytest.org/latest/funcargs.html
|
|
||||||
|
|
||||||
**Abstract**: pytest-2.X provides yet more powerful and flexible
|
.. _`Dependency injection`: http://en.wikipedia.org/wiki/Dependency_injection
|
||||||
fixture machinery by introducing:
|
|
||||||
|
|
||||||
* a new ``@pytest.mark.funcarg`` marker to define funcarg factories and their
|
.. versionadded: 2.3
|
||||||
scoping and parametrization. No special ``pytest_funcarg__`` naming there.
|
|
||||||
|
|
||||||
* a new ``@pytest.mark.setup`` marker to define setup functions and their
|
pytest offers advanced resource parametrization and injection mechanisms
|
||||||
|
including a fully integrated generalization of the popular xUnit
|
||||||
|
setup-style methods. A resource is created by a ``@pytest.mark.factory``
|
||||||
|
marked function and its name is the name of the function. A resource
|
||||||
|
is injected into test or setup functions if they use the name
|
||||||
|
in their signature. Therefore and also for historic reasons, resources are
|
||||||
|
sometimes called "funcargs" because they ultimately appear as
|
||||||
|
function arguments.
|
||||||
|
|
||||||
|
The pytest resource management and setup features are exposed through
|
||||||
|
three decorators:
|
||||||
|
|
||||||
|
* a `@pytest.mark.factory`_ marker to define resource factories,
|
||||||
|
their scoping and parametrization.
|
||||||
|
|
||||||
|
* a `@pytest.mark.setup`_ marker to define setup functions and their
|
||||||
scoping.
|
scoping.
|
||||||
|
|
||||||
* directly use funcargs through funcarg factory signatures
|
* a `@pytest.mark.parametrize`_ marker for executing test functions
|
||||||
|
multiple times with different parameter sets
|
||||||
|
|
||||||
Both funcarg factories and setup functions can be defined in test modules,
|
Generally, resource factories and setup functions:
|
||||||
classes, conftest.py files and installed plugins.
|
|
||||||
|
|
||||||
The introduction of these two markers lifts several prior limitations
|
- can be defined in test modules, test classes, conftest.py files or
|
||||||
and allows to easily define and implement complex testing scenarios.
|
in plugins.
|
||||||
|
|
||||||
Nonwithstanding these extensions, already existing test suites and plugins
|
- can themselves receive resources through their function arguments,
|
||||||
written to work for previous pytest versions shall run unmodified.
|
simplifying the setup and use of interdependent resources.
|
||||||
|
|
||||||
|
- can use the special `testcontext`_ object for access to the
|
||||||
|
context in which the factory/setup is called and for registering
|
||||||
|
finalizers.
|
||||||
|
|
||||||
|
This document showcases these features through some basic examples.
|
||||||
|
|
||||||
|
Note that pytest also comes with some :ref:`builtinresources` which
|
||||||
|
you can use without defining them yourself.
|
||||||
|
|
||||||
|
Background and terms
|
||||||
|
---------------------------
|
||||||
|
|
||||||
|
The pytest resource management mechanism is an example of `Dependency
|
||||||
|
Injection`_ which helps to de-couple test code from resource
|
||||||
|
instantiation code required for them to execute. At test writing time
|
||||||
|
you typically do not need to care for the details of how your required
|
||||||
|
resources are constructed, if they live through a function, class,
|
||||||
|
module or session scope or if the test will be called multiple times
|
||||||
|
with different resource instances.
|
||||||
|
|
||||||
|
To create a value with which to call a test function a resource factory
|
||||||
|
function is called which gets full access to the test context and can
|
||||||
|
register finalizers which are to be run after the last test in that context
|
||||||
|
finished. Resource factories can be implemented in same test class or
|
||||||
|
test module, in a per-directory ``conftest.py`` file or in an external plugin. This allows total de-coupling of test and setup code.
|
||||||
|
|
||||||
|
A test function may be invoked multiple times in which case we
|
||||||
|
speak of :ref:`parametrized testing <parametrizing-tests>`. This can be
|
||||||
|
very useful if you want to test e.g. against different database backends
|
||||||
|
or with multiple numerical arguments sets and want to reuse the same set
|
||||||
|
of test functions.
|
||||||
|
|
||||||
|
|
||||||
**Changes**: This V4 draft is based on incorporating and thinking about
|
.. _`@pytest.mark.factory`:
|
||||||
feedback on previous versions provided by Floris Bruynooghe, Carl Meyer,
|
|
||||||
Ronny Pfannschmidt and Samuele Pedroni. It remains as draft
|
|
||||||
documentation, pending further refinements and changes according to
|
|
||||||
implementation or backward compatibility issues. The main changes are:
|
|
||||||
|
|
||||||
* Collapse funcarg factory decorators into a single "@funcarg" one.
|
``@pytest.mark.factory``: Creating parametrized, scoped resources
|
||||||
You can specify scopes and params with it. When using the decorator
|
-----------------------------------------------------------------
|
||||||
the "pytest_funcarg__" prefix becomes optional.
|
|
||||||
|
|
||||||
* funcarg factories can now use funcargs themselves
|
.. regendoc:wipe
|
||||||
|
|
||||||
* Drop setup/directory scope from this draft
|
.. versionadded:: 2.3
|
||||||
|
|
||||||
* introduce a new @setup decorator similar to the @funcarg one
|
The `@pytest.mark.factory`_ marker allows to
|
||||||
except that setup-markers cannot define parametriation themselves.
|
|
||||||
Instead they can easily depend on a parametrized funcarg (which
|
|
||||||
must not be visible at test function signatures).
|
|
||||||
|
|
||||||
* drop consideration of setup_X support for funcargs because
|
* mark a function as a factory for resources used by test and setup functions
|
||||||
it is less flexible and probably causes more implementation
|
* define parametrization to run tests multiple times with different
|
||||||
troubles than the current @setup approach which can share
|
resource instances
|
||||||
a lot of logic with the @funcarg one.
|
* set a scope which determines the level of caching. valid scopes
|
||||||
|
are ``session``, ``module``, ``class`` and ``function``.
|
||||||
|
|
||||||
* tests are grouped by parametrized funcargs
|
Here is a simple example of a factory creating a shared ``smtplib.SMTP``
|
||||||
|
connection resource which test functions then may use across the whole
|
||||||
.. currentmodule:: _pytest
|
test session::
|
||||||
|
|
||||||
|
|
||||||
Shortcomings of the previous pytest_funcarg__ mechanism
|
|
||||||
---------------------------------------------------------
|
|
||||||
|
|
||||||
The previous funcarg mechanism calls a factory each time a
|
|
||||||
funcarg for a test function is requested. If a factory wants
|
|
||||||
t re-use a resource across different scopes, it often used
|
|
||||||
the ``request.cached_setup()`` helper to manage caching of
|
|
||||||
resources. Here is a basic example how we could implement
|
|
||||||
a per-session Database object::
|
|
||||||
|
|
||||||
# content of conftest.py
|
# content of conftest.py
|
||||||
class Database:
|
import pytest
|
||||||
def __init__(self):
|
import smtplib
|
||||||
print ("database instance created")
|
|
||||||
def destroy(self):
|
|
||||||
print ("database instance destroyed")
|
|
||||||
|
|
||||||
def pytest_funcarg__db(request):
|
@pytest.mark.factory(scope="session")
|
||||||
return request.cached_setup(setup=DataBase,
|
def smtp(testcontext):
|
||||||
teardown=lambda db: db.destroy,
|
smtp = smtplib.SMTP("merlinux.eu")
|
||||||
scope="session")
|
testcontext.addfinalizer(smtp.close)
|
||||||
|
return smtp
|
||||||
|
|
||||||
There are some problems with this approach:
|
The name of the resource is ``smtp`` (the factory function name)
|
||||||
|
and you can now access the ``smtp`` resource by listing it as
|
||||||
|
an input parameter in any test function below the directory where
|
||||||
|
``conftest.py`` is located::
|
||||||
|
|
||||||
1. Scoping resource creation is not straight forward, instead one must
|
# content of test_module.py
|
||||||
understand the intricate cached_setup() method mechanics.
|
def test_ehlo(smtp):
|
||||||
|
response = smtp.ehlo()
|
||||||
|
assert response[0] == 250
|
||||||
|
assert "merlinux" in response[1]
|
||||||
|
assert 0 # for demo purposes
|
||||||
|
|
||||||
2. parametrizing the "db" resource is not straight forward:
|
def test_noop(smtp):
|
||||||
you need to apply a "parametrize" decorator or implement a
|
response = smtp.noop()
|
||||||
:py:func:`~hookspec.pytest_generate_tests` hook
|
assert response[0] == 250
|
||||||
calling :py:func:`~python.Metafunc.parametrize` which
|
assert 0 # for demo purposes
|
||||||
performs parametrization at the places where the resource
|
|
||||||
is used. Moreover, you need to modify the factory to use an
|
|
||||||
``extrakey`` parameter containing ``request.param`` to the
|
|
||||||
:py:func:`~python.Request.cached_setup` call.
|
|
||||||
|
|
||||||
3. there is no way how you can make use of funcarg factories
|
If you run the tests::
|
||||||
in xUnit setup methods.
|
|
||||||
|
|
||||||
4. A non-parametrized funcarg factory cannot use a parametrized
|
$ py.test -q
|
||||||
funcarg resource if it isn't stated in the test function signature.
|
collecting ... collected 2 items
|
||||||
|
FF
|
||||||
|
================================= FAILURES =================================
|
||||||
|
________________________________ test_ehlo _________________________________
|
||||||
|
|
||||||
The following sections address the advances which solve all of these problems.
|
smtp = <smtplib.SMTP instance at 0x1c7c638>
|
||||||
|
|
||||||
|
def test_ehlo(smtp):
|
||||||
|
response = smtp.ehlo()
|
||||||
|
assert response[0] == 250
|
||||||
|
assert "merlinux" in response[1]
|
||||||
|
> assert 0 # for demo purposes
|
||||||
|
E assert 0
|
||||||
|
|
||||||
|
test_module.py:5: AssertionError
|
||||||
|
________________________________ test_noop _________________________________
|
||||||
|
|
||||||
|
smtp = <smtplib.SMTP instance at 0x1c7c638>
|
||||||
|
|
||||||
|
def test_noop(smtp):
|
||||||
|
response = smtp.noop()
|
||||||
|
assert response[0] == 250
|
||||||
|
> assert 0 # for demo purposes
|
||||||
|
E assert 0
|
||||||
|
|
||||||
|
test_module.py:10: AssertionError
|
||||||
|
2 failed in 0.18 seconds
|
||||||
|
|
||||||
|
you will see the two ``assert 0`` failing and can see that
|
||||||
|
the same (session-scoped) object was passed into the two test functions.
|
||||||
|
|
||||||
|
If you now want to test multiple servers you can simply parametrize
|
||||||
|
the ``smtp`` factory::
|
||||||
|
|
||||||
|
# content of conftest.py
|
||||||
|
import pytest
|
||||||
|
import smtplib
|
||||||
|
|
||||||
|
@pytest.mark.factory(scope="session",
|
||||||
|
params=["merlinux.eu", "mail.python.org"])
|
||||||
|
def smtp(testcontext):
|
||||||
|
smtp = smtplib.SMTP(testcontext.param)
|
||||||
|
def fin():
|
||||||
|
smtp.close()
|
||||||
|
testcontext.addfinalizer(fin)
|
||||||
|
return smtp
|
||||||
|
|
||||||
|
The main change is the definition of a ``params`` list in the
|
||||||
|
``factory``-marker and the ``testcontext.param`` access within the
|
||||||
|
factory function. No test code needs to change. So let's just do another
|
||||||
|
run::
|
||||||
|
|
||||||
|
$ py.test -q
|
||||||
|
collecting ... collected 4 items
|
||||||
|
FFFF
|
||||||
|
================================= FAILURES =================================
|
||||||
|
__________________________ test_ehlo[merlinux.eu] __________________________
|
||||||
|
|
||||||
|
smtp = <smtplib.SMTP instance at 0x1d162d8>
|
||||||
|
|
||||||
|
def test_ehlo(smtp):
|
||||||
|
response = smtp.ehlo()
|
||||||
|
assert response[0] == 250
|
||||||
|
assert "merlinux" in response[1]
|
||||||
|
> assert 0 # for demo purposes
|
||||||
|
E assert 0
|
||||||
|
|
||||||
|
test_module.py:5: AssertionError
|
||||||
|
__________________________ test_noop[merlinux.eu] __________________________
|
||||||
|
|
||||||
|
smtp = <smtplib.SMTP instance at 0x1d162d8>
|
||||||
|
|
||||||
|
def test_noop(smtp):
|
||||||
|
response = smtp.noop()
|
||||||
|
assert response[0] == 250
|
||||||
|
> assert 0 # for demo purposes
|
||||||
|
E assert 0
|
||||||
|
|
||||||
|
test_module.py:10: AssertionError
|
||||||
|
________________________ test_ehlo[mail.python.org] ________________________
|
||||||
|
|
||||||
|
smtp = <smtplib.SMTP instance at 0x1d1f098>
|
||||||
|
|
||||||
|
def test_ehlo(smtp):
|
||||||
|
response = smtp.ehlo()
|
||||||
|
assert response[0] == 250
|
||||||
|
> assert "merlinux" in response[1]
|
||||||
|
E assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN'
|
||||||
|
|
||||||
|
test_module.py:4: AssertionError
|
||||||
|
________________________ test_noop[mail.python.org] ________________________
|
||||||
|
|
||||||
|
smtp = <smtplib.SMTP instance at 0x1d1f098>
|
||||||
|
|
||||||
|
def test_noop(smtp):
|
||||||
|
response = smtp.noop()
|
||||||
|
assert response[0] == 250
|
||||||
|
> assert 0 # for demo purposes
|
||||||
|
E assert 0
|
||||||
|
|
||||||
|
test_module.py:10: AssertionError
|
||||||
|
4 failed in 6.42 seconds
|
||||||
|
|
||||||
|
We get four failures because we are running the two tests twice with
|
||||||
|
different ``smtp`` instantiations as defined on the factory.
|
||||||
|
Note that with the ``mail.python.org`` connection the second test
|
||||||
|
fails in ``test_ehlo`` because it expects a specific server string.
|
||||||
|
|
||||||
|
You can also look at what tests pytest collects without running them::
|
||||||
|
|
||||||
|
$ py.test --collectonly
|
||||||
|
=========================== test session starts ============================
|
||||||
|
platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev7
|
||||||
|
plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
|
||||||
|
collecting ... collected 4 items
|
||||||
|
<Module 'test_module.py'>
|
||||||
|
<Function 'test_ehlo[merlinux.eu]'>
|
||||||
|
<Function 'test_noop[merlinux.eu]'>
|
||||||
|
<Function 'test_ehlo[mail.python.org]'>
|
||||||
|
<Function 'test_noop[mail.python.org]'>
|
||||||
|
|
||||||
|
============================= in 0.02 seconds =============================
|
||||||
|
|
||||||
|
Note that pytest orders your test run by resource usage, minimizing
|
||||||
|
the number of active resources at any given time.
|
||||||
|
|
||||||
|
|
||||||
Direct scoping of funcarg factories
|
Accessing resources from a factory function
|
||||||
--------------------------------------------------------
|
|
||||||
|
|
||||||
.. note:: Implemented
|
|
||||||
|
|
||||||
Instead of calling cached_setup(), you can decorate your factory
|
|
||||||
to state its scope::
|
|
||||||
|
|
||||||
@pytest.mark.funcarg(scope="session")
|
|
||||||
def pytest_funcarg__db(request):
|
|
||||||
# factory will only be invoked once per session -
|
|
||||||
db = DataBase()
|
|
||||||
request.addfinalizer(db.destroy) # destroy when session is finished
|
|
||||||
return db
|
|
||||||
|
|
||||||
This factory implementation does not need to call ``cached_setup()`` anymore
|
|
||||||
because it will only be invoked once per session. Moreover, the
|
|
||||||
``request.addfinalizer()`` registers a finalizer according to the specified
|
|
||||||
resource scope on which the factory function is operating. With this new
|
|
||||||
scoping, the still existing ``cached_setup()`` should be much less used
|
|
||||||
but will remain for compatibility reasons and for the case where you
|
|
||||||
still want to have your factory get called on a per-item basis.
|
|
||||||
|
|
||||||
|
|
||||||
Direct parametrization of funcarg resource factories
|
|
||||||
----------------------------------------------------------
|
----------------------------------------------------------
|
||||||
|
|
||||||
.. note:: Implemented
|
You can directly use resources as funcargs in resource factories.
|
||||||
|
Extending the previous example we can instantiate an application
|
||||||
|
object and stick the live ``smtp`` resource into it::
|
||||||
|
|
||||||
Previously, funcarg factories could not directly cause parametrization.
|
# content of test_appsetup.py
|
||||||
You needed to specify a ``@parametrize`` or implement a ``pytest_generate_tests`` hook to perform parametrization, i.e. calling a test multiple times
|
|
||||||
with different value sets. pytest-2.X introduces a decorator for use
|
|
||||||
on the factory itself::
|
|
||||||
|
|
||||||
@pytest.mark.funcarg(params=["mysql", "pg"])
|
import pytest
|
||||||
def pytest_funcarg__db(request):
|
|
||||||
...
|
|
||||||
|
|
||||||
Here the factory will be invoked twice (with the respective "mysql"
|
class App:
|
||||||
and "pg" values set as ``request.param`` attributes) and and all of
|
def __init__(self, smtp):
|
||||||
the tests requiring "db" will run twice as well. The "mysql" and
|
self.smtp = smtp
|
||||||
"pg" values will also be used for reporting the test-invocation variants.
|
|
||||||
|
|
||||||
This new way of parametrizing funcarg factories should in many cases
|
@pytest.mark.factory(scope="module")
|
||||||
allow to re-use already written factories because effectively
|
def app(smtp):
|
||||||
``request.param`` are already the parametrization attribute for test
|
return App(smtp)
|
||||||
functions/classes were parametrized via
|
|
||||||
:py:func:`~_pytest.python.Metafunc.parametrize(indirect=True)` calls.
|
|
||||||
|
|
||||||
Of course it's perfectly fine to combine parametrization and scoping::
|
def test_exists(app):
|
||||||
|
assert app.smtp
|
||||||
|
|
||||||
@pytest.mark.funcarg(scope="session", params=["mysql", "pg"])
|
Let's run this::
|
||||||
def pytest_funcarg__db(request):
|
|
||||||
if request.param == "mysql":
|
|
||||||
db = MySQL()
|
|
||||||
elif request.param == "pg":
|
|
||||||
db = PG()
|
|
||||||
request.addfinalizer(db.destroy) # destroy when session is finished
|
|
||||||
return db
|
|
||||||
|
|
||||||
This would execute all tests requiring the per-session "db" resource twice,
|
$ py.test -v test_appsetup.py
|
||||||
receiving the values created by the two respective invocations to the
|
=========================== test session starts ============================
|
||||||
factory function.
|
platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev7 -- /home/hpk/venv/1/bin/python
|
||||||
|
cachedir: /home/hpk/tmp/doc-exec-398/.cache
|
||||||
|
plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
|
||||||
|
collecting ... collected 2 items
|
||||||
|
|
||||||
Direct usage of funcargs with funcargs factories
|
test_appsetup.py:12: test_exists[merlinux.eu] PASSED
|
||||||
----------------------------------------------------------
|
test_appsetup.py:12: test_exists[mail.python.org] PASSED
|
||||||
|
|
||||||
.. note:: Implemented.
|
========================= 2 passed in 5.96 seconds =========================
|
||||||
|
|
||||||
You can now directly use funcargs in funcarg factories. Example::
|
Due to the parametrization of ``smtp`` the test will
|
||||||
|
run twice with two different ``App`` instances and respective smtp servers.
|
||||||
@pytest.mark.funcarg(scope="session")
|
There is no need for the ``app`` factory to be aware of the parametrization.
|
||||||
def db(request, tmpdir):
|
|
||||||
# tmpdir is a session-specific tempdir
|
|
||||||
|
|
||||||
Apart from convenience it also solves an issue when your factory
|
|
||||||
depends on a parametrized funcarg. Previously, a call to
|
|
||||||
``request.getfuncargvalue()`` happens at test execution time and
|
|
||||||
thus pytest would not know at collection time about the fact that
|
|
||||||
a required resource is parametrized.
|
|
||||||
|
|
||||||
No ``pytest_funcarg__`` prefix when using @funcarg decorator
|
|
||||||
-------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
.. note:: Implemented
|
.. _`new_setup`:
|
||||||
|
.. _`@pytest.mark.setup`:
|
||||||
|
|
||||||
When using the ``@funcarg`` decorator the name of the function
|
``@pytest.mark.setup``: xUnit setup methods on steroids
|
||||||
does not need to (and in fact cannot) use the ``pytest_funcarg__``
|
-----------------------------------------------------------------
|
||||||
naming::
|
|
||||||
|
|
||||||
@pytest.mark.funcarg
|
.. regendoc:wipe
|
||||||
def db(request):
|
|
||||||
...
|
|
||||||
|
|
||||||
The name under which the funcarg resource can be requested is ``db``.
|
.. versionadded:: 2.3
|
||||||
|
|
||||||
You can also use the "old" non-decorator way of specifying funcarg factories
|
The ``@pytest.mark.setup`` marker allows
|
||||||
aka::
|
|
||||||
|
|
||||||
def pytest_funcarg__db(request):
|
* to define setup-functions close to test code or in conftest.py files
|
||||||
...
|
or plugins.
|
||||||
|
* to mark a function as a setup method; the function can itself
|
||||||
|
receive funcargs and will execute multiple times if the funcargs
|
||||||
|
are parametrized
|
||||||
|
* to set a scope which influences when the setup function going to be
|
||||||
|
called. valid scopes are ``session``, ``module``, ``class`` and ``function``.
|
||||||
|
|
||||||
It is recommended to use the funcarg-decorator, however.
|
Here is a simple example. First we define a global ``globdir`` resource::
|
||||||
|
|
||||||
|
# content of conftest.py
|
||||||
|
import pytest
|
||||||
|
|
||||||
solving per-session setup / the new @setup marker
|
@pytest.mark.factory(scope="module")
|
||||||
--------------------------------------------------------------
|
def globdir(testcontext, tmpdir):
|
||||||
|
def fin():
|
||||||
|
print "finalize", tmpdir
|
||||||
|
testcontext.addfinalizer(fin)
|
||||||
|
print "created resource", tmpdir
|
||||||
|
return tmpdir
|
||||||
|
|
||||||
.. note:: Implemented, at least working for basic situations.
|
And then we write a test file containing a setup-marked function
|
||||||
|
taking this resource and setting it as a module global::
|
||||||
|
|
||||||
pytest for a long time offered a pytest_configure and a pytest_sessionstart
|
# content of test_module.py
|
||||||
hook which are often used to setup global resources. This suffers from
|
import pytest
|
||||||
several problems:
|
|
||||||
|
|
||||||
1. in distributed testing the master process would setup test resources
|
@pytest.mark.setup(scope="module")
|
||||||
that are never needed because it only co-ordinates the test run
|
def setresource(testcontext, globdir):
|
||||||
activities of the slave processes.
|
print "setupresource", globdir
|
||||||
|
testcontext.module.myresource = globdir
|
||||||
|
|
||||||
2. if you only perform a collection (with "--collectonly")
|
def test_1():
|
||||||
resource-setup will still be executed.
|
assert myresource
|
||||||
|
print "using myresource", myresource
|
||||||
|
|
||||||
3. If a pytest_sessionstart is contained in some subdirectories
|
def test_2():
|
||||||
conftest.py file, it will not be called. This stems from the
|
assert myresource
|
||||||
fact that this hook is actually used for reporting, in particular
|
print "using myresource", myresource
|
||||||
the test-header with platform/custom information.
|
|
||||||
|
|
||||||
Moreover, it is today not easy to define a scoped setup from plugins or
|
Let's run this module::
|
||||||
conftest files other than to implement a ``pytest_runtest_setup()`` hook
|
|
||||||
and caring for scoping/caching yourself. And it's virtually impossible
|
|
||||||
to do this with parametrization as ``pytest_runtest_setup()`` is called
|
|
||||||
during test execution and parametrization happens at collection time.
|
|
||||||
|
|
||||||
It follows that pytest_configure/session/runtest_setup are often not
|
$ py.test -qs
|
||||||
appropriate for implementing common fixture needs. Therefore,
|
collecting ... collected 2 items
|
||||||
pytest-2.X introduces a new "@pytest.mark.setup" marker which takes
|
..
|
||||||
an optional "scope" parameter.
|
2 passed in 0.26 seconds
|
||||||
|
created resource /home/hpk/tmp/pytest-4427/test_10
|
||||||
|
setupresource /home/hpk/tmp/pytest-4427/test_10
|
||||||
|
using myresource /home/hpk/tmp/pytest-4427/test_10
|
||||||
|
using myresource /home/hpk/tmp/pytest-4427/test_10
|
||||||
|
finalize /home/hpk/tmp/pytest-4427/test_10
|
||||||
|
|
||||||
See :ref:`new_setup` for examples.
|
The two test functions in the module use the same global ``myresource``
|
||||||
|
object because the ``setresource`` set it as a module attribute.
|
||||||
|
|
||||||
funcarg and setup discovery now happens at collection time
|
The ``globdir`` factory can now become parametrized without any test
|
||||||
---------------------------------------------------------------------
|
or setup code needing to change::
|
||||||
|
|
||||||
|
# content of conftest.py
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
@pytest.mark.factory(scope="module", params=["aaa", "bbb"])
|
||||||
|
def globdir(testcontext, tmpdir):
|
||||||
|
newtmp = tmpdir.join(testcontext.param)
|
||||||
|
def fin():
|
||||||
|
print "finalize", newtmp
|
||||||
|
testcontext.addfinalizer(fin)
|
||||||
|
print "created resource", newtmp
|
||||||
|
return newtmp
|
||||||
|
|
||||||
|
Running the unchanged previous test files now runs four tests::
|
||||||
|
|
||||||
|
$ py.test -qs
|
||||||
|
collecting ... collected 4 items
|
||||||
|
....
|
||||||
|
4 passed in 0.26 seconds
|
||||||
|
created resource /home/hpk/tmp/pytest-4428/test_1_aaa_0/aaa
|
||||||
|
setupresource /home/hpk/tmp/pytest-4428/test_1_aaa_0/aaa
|
||||||
|
using myresource /home/hpk/tmp/pytest-4428/test_1_aaa_0/aaa
|
||||||
|
using myresource /home/hpk/tmp/pytest-4428/test_1_aaa_0/aaa
|
||||||
|
finalize /home/hpk/tmp/pytest-4428/test_1_aaa_0/aaa
|
||||||
|
created resource /home/hpk/tmp/pytest-4428/test_1_bbb_0/bbb
|
||||||
|
setupresource /home/hpk/tmp/pytest-4428/test_1_bbb_0/bbb
|
||||||
|
using myresource /home/hpk/tmp/pytest-4428/test_1_bbb_0/bbb
|
||||||
|
using myresource /home/hpk/tmp/pytest-4428/test_1_bbb_0/bbb
|
||||||
|
finalize /home/hpk/tmp/pytest-4428/test_1_bbb_0/bbb
|
||||||
|
|
||||||
|
Each parameter causes the creation of a respective resource and the
|
||||||
|
unchanged test module uses it in its ``@setup`` decorated method.
|
||||||
|
|
||||||
.. note::
|
.. note::
|
||||||
Partially implemented - collectonly shows no extra information however.
|
|
||||||
|
|
||||||
pytest-2.X takes care to discover funcarg factories and @setup methods
|
|
||||||
at collection time. This is more efficient especially for large test suites.
|
|
||||||
Moreover, a call to "py.test --collectonly" should be able to show
|
|
||||||
a lot of setup-information and thus presents a nice method to get an
|
|
||||||
overview of resource management in your project.
|
|
||||||
|
|
||||||
|
Tests using a particular parametrized resource instance will
|
||||||
|
executed next to each other. Any finalizers will be run before the
|
||||||
|
next parametrized resource instance is being setup and tests
|
||||||
|
are rerun.
|
||||||
|
|
||||||
Grouping tests by resource parameters
|
Grouping tests by resource parameters
|
||||||
----------------------------------------------------------
|
----------------------------------------------------------
|
||||||
|
|
||||||
.. note:: Implemented.
|
.. regendoc: wipe
|
||||||
|
|
||||||
pytest used to always sort test items by their source location.
|
pytest minimizes the number of active resources during test runs.
|
||||||
With pytest-2.X tests are first grouped by funcarg parameters.
|
If you have a parametrized resource, then all the tests using one
|
||||||
If you have a parametrized funcarg, then all the tests using it
|
resource instance will execute one after another. Then any finalizers
|
||||||
will first execute with it. Then any finalizers are called and then
|
are called for that resource instance and then the next parametrized
|
||||||
the next parametrized resource instance is created and its tests are run.
|
resource instance is created and its tests are run. Among other things,
|
||||||
Among other things, this eases testing of applications which create
|
this eases testing of applications which create and use global state.
|
||||||
and use global state.
|
|
||||||
|
|
||||||
The following example uses two parametrized funcargs, one of which is
|
The following example uses two parametrized funcargs, one of which is
|
||||||
scoped on a per-module basis::
|
scoped on a per-module basis::
|
||||||
|
|
@ -273,18 +416,18 @@ scoped on a per-module basis::
|
||||||
# content of test_module.py
|
# content of test_module.py
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
@pytest.mark.funcarg(scope="module", params=["mod1", "mod2"])
|
@pytest.mark.factory(scope="module", params=["mod1", "mod2"])
|
||||||
def modarg(request):
|
def modarg(testcontext):
|
||||||
param = request.param
|
param = testcontext.param
|
||||||
print "create", param
|
print "create", param
|
||||||
def fin():
|
def fin():
|
||||||
print "fin", param
|
print "fin", param
|
||||||
request.addfinalizer(fin)
|
testcontext.addfinalizer(fin)
|
||||||
return param
|
return param
|
||||||
|
|
||||||
@pytest.mark.funcarg(scope="function", params=[1,2])
|
@pytest.mark.factory(scope="function", params=[1,2])
|
||||||
def otherarg(request):
|
def otherarg(testcontext):
|
||||||
return request.param
|
return testcontext.param
|
||||||
|
|
||||||
def test_0(otherarg):
|
def test_0(otherarg):
|
||||||
print " test0", otherarg
|
print " test0", otherarg
|
||||||
|
|
@ -297,8 +440,8 @@ Let's run the tests in verbose mode and with looking at the print-output::
|
||||||
|
|
||||||
$ py.test -v -s
|
$ py.test -v -s
|
||||||
=========================== test session starts ============================
|
=========================== test session starts ============================
|
||||||
platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev5 -- /home/hpk/venv/1/bin/python
|
platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev7 -- /home/hpk/venv/1/bin/python
|
||||||
cachedir: /home/hpk/tmp/doc-exec-388/.cache
|
cachedir: /home/hpk/tmp/doc-exec-398/.cache
|
||||||
plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
|
plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
|
||||||
collecting ... collected 8 items
|
collecting ... collected 8 items
|
||||||
|
|
||||||
|
|
@ -325,10 +468,132 @@ Let's run the tests in verbose mode and with looking at the print-output::
|
||||||
test2 2 mod2
|
test2 2 mod2
|
||||||
fin mod2
|
fin mod2
|
||||||
|
|
||||||
You can see that that the parametrized ``modarg`` resource lead to
|
You can see that the parametrized module-scoped ``modarg`` resource caused
|
||||||
a re-ordering of test execution. The finalizer for the "mod1" parametrized
|
a re-ordering of test execution. The finalizer for the ``mod1`` parametrized
|
||||||
resource was executed before the "mod2" resource was setup.
|
resource was executed before the ``mod2`` resource was setup.
|
||||||
|
|
||||||
.. note::
|
.. currentmodule:: _pytest.python
|
||||||
|
.. _`testcontext`:
|
||||||
|
|
||||||
The current implementation is experimental.
|
``testcontext``: interacting with test context
|
||||||
|
---------------------------------------------------
|
||||||
|
|
||||||
|
The ``testcontext`` object may be received by `@pytest.mark.factory`_ or
|
||||||
|
`@pytest.mark.setup`_ marked functions. It contains information relating
|
||||||
|
to the test context within which the function executes. Moreover, you
|
||||||
|
can call ``testcontext.addfinalizer(myfinalizer)`` in order to trigger
|
||||||
|
a call to ``myfinalizer`` after the last test in the test context has executed.
|
||||||
|
If passed to a parametrized factory ``testcontext.param`` will contain
|
||||||
|
a parameter (one value out of the ``params`` list specified with the
|
||||||
|
`@pytest.mark.factory`_ marker).
|
||||||
|
|
||||||
|
.. autoclass:: _pytest.python.TestContext()
|
||||||
|
:members:
|
||||||
|
|
||||||
|
.. _`@pytest.mark.parametrize`:
|
||||||
|
|
||||||
|
``@pytest.mark.parametrize``: directly parametrizing test functions
|
||||||
|
----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
.. versionadded:: 2.2
|
||||||
|
|
||||||
|
The builtin ``pytest.mark.parametrize`` decorator enables
|
||||||
|
parametrization of arguments for a test function. Here is an example
|
||||||
|
of a test function that wants check for expected output given a certain input::
|
||||||
|
|
||||||
|
# content of test_expectation.py
|
||||||
|
import pytest
|
||||||
|
@pytest.mark.parametrize(("input", "expected"), [
|
||||||
|
("3+5", 8),
|
||||||
|
("2+4", 6),
|
||||||
|
("6*9", 42),
|
||||||
|
])
|
||||||
|
def test_eval(input, expected):
|
||||||
|
assert eval(input) == expected
|
||||||
|
|
||||||
|
we parametrize two arguments of the test function so that the test
|
||||||
|
function is called three times. Let's run it::
|
||||||
|
|
||||||
|
$ py.test -q
|
||||||
|
collecting ... collected 11 items
|
||||||
|
..F........
|
||||||
|
================================= FAILURES =================================
|
||||||
|
____________________________ test_eval[6*9-42] _____________________________
|
||||||
|
|
||||||
|
input = '6*9', expected = 42
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("input", "expected"), [
|
||||||
|
("3+5", 8),
|
||||||
|
("2+4", 6),
|
||||||
|
("6*9", 42),
|
||||||
|
])
|
||||||
|
def test_eval(input, expected):
|
||||||
|
> assert eval(input) == expected
|
||||||
|
E assert 54 == 42
|
||||||
|
E + where 54 = eval('6*9')
|
||||||
|
|
||||||
|
test_expectation.py:8: AssertionError
|
||||||
|
1 failed, 10 passed in 0.04 seconds
|
||||||
|
|
||||||
|
As expected only one pair of input/output values fails the simple test function.
|
||||||
|
|
||||||
|
Note that there are various ways how you can mark groups of functions,
|
||||||
|
see :ref:`mark`.
|
||||||
|
|
||||||
|
Generating parameters combinations, depending on command line
|
||||||
|
----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
.. regendoc:wipe
|
||||||
|
|
||||||
|
Let's say we want to execute a test with different computation
|
||||||
|
parameters and the parameter range shall be determined by a command
|
||||||
|
line argument. Let's first write a simple (do-nothing) computation test::
|
||||||
|
|
||||||
|
# content of test_compute.py
|
||||||
|
|
||||||
|
def test_compute(param1):
|
||||||
|
assert param1 < 4
|
||||||
|
|
||||||
|
Now we add a test configuration like this::
|
||||||
|
|
||||||
|
# content of conftest.py
|
||||||
|
|
||||||
|
def pytest_addoption(parser):
|
||||||
|
parser.addoption("--all", action="store_true",
|
||||||
|
help="run all combinations")
|
||||||
|
|
||||||
|
def pytest_generate_tests(metafunc):
|
||||||
|
if 'param1' in metafunc.funcargnames:
|
||||||
|
if metafunc.config.option.all:
|
||||||
|
end = 5
|
||||||
|
else:
|
||||||
|
end = 2
|
||||||
|
metafunc.parametrize("param1", range(end))
|
||||||
|
|
||||||
|
This means that we only run 2 tests if we do not pass ``--all``::
|
||||||
|
|
||||||
|
$ py.test -q test_compute.py
|
||||||
|
collecting ... collected 2 items
|
||||||
|
..
|
||||||
|
2 passed in 0.03 seconds
|
||||||
|
|
||||||
|
We run only two computations, so we see two dots.
|
||||||
|
let's run the full monty::
|
||||||
|
|
||||||
|
$ py.test -q --all
|
||||||
|
collecting ... collected 5 items
|
||||||
|
....F
|
||||||
|
================================= FAILURES =================================
|
||||||
|
_____________________________ test_compute[4] ______________________________
|
||||||
|
|
||||||
|
param1 = 4
|
||||||
|
|
||||||
|
def test_compute(param1):
|
||||||
|
> assert param1 < 4
|
||||||
|
E assert 4 < 4
|
||||||
|
|
||||||
|
test_compute.py:3: AssertionError
|
||||||
|
1 failed, 4 passed in 0.03 seconds
|
||||||
|
|
||||||
|
As expected when running the full range of ``param1`` values
|
||||||
|
we'll get an error on the last one.
|
||||||
|
|
|
||||||
2
setup.py
2
setup.py
|
|
@ -24,7 +24,7 @@ def main():
|
||||||
name='pytest',
|
name='pytest',
|
||||||
description='py.test: simple powerful testing with Python',
|
description='py.test: simple powerful testing with Python',
|
||||||
long_description = long_description,
|
long_description = long_description,
|
||||||
version='2.3.0.dev6',
|
version='2.3.0.dev7',
|
||||||
url='http://pytest.org',
|
url='http://pytest.org',
|
||||||
license='MIT license',
|
license='MIT license',
|
||||||
platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'],
|
platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'],
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue