diff --git a/_pytest/__init__.py b/_pytest/__init__.py index 7b3589798..98b4fa5e0 100644 --- a/_pytest/__init__.py +++ b/_pytest/__init__.py @@ -1,2 +1,2 @@ # -__version__ = '2.3.0.dev6' +__version__ = '2.3.0.dev7' diff --git a/_pytest/python.py b/_pytest/python.py index 7b022d016..eb89e713b 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1355,7 +1355,8 @@ scope2props["function"] = scope2props["class"] + ("function", "keywords") def scopeprop(attr, name=None, doc=None): if doc is None: - doc = "%s of underlying test context" % (attr,) + doc = ("%s of underlying test context, may not exist " + "if the testcontext has a higher scope" % (attr,)) name = name or attr def get(self): if name in scope2props[self.scope]: @@ -1370,6 +1371,7 @@ def rprop(attr, doc=None): return property(lambda x: getattr(x._request, attr), doc=doc) class TestContext(object): + """ Basic objects of the current testing context. """ def __init__(self, request, scope): self._request = request self.scope = scope @@ -1379,7 +1381,6 @@ class TestContext(object): config = rprop("config", "pytest config object.") session = rprop("session", "pytest session object.") - param = rprop("param") function = scopeprop("function") module = scopeprop("module") @@ -1400,6 +1401,8 @@ class TestContextSetup(TestContext): self._setupcall.addfinalizer(finalizer) class TestContextResource(TestContext): + param = rprop("param") + def __init__(self, request): super(TestContextResource, self).__init__(request, request.scope) diff --git a/doc/en/builtin.txt b/doc/en/builtin.txt index 5a0cd2a88..5614c364e 100644 --- a/doc/en/builtin.txt +++ b/doc/en/builtin.txt @@ -17,10 +17,9 @@ to get an overview on the globally available helpers. .. automodule:: pytest :members: +.. _builtinresources: -.. _builtinfuncargs: - -Builtin function arguments +Builtin resources / function arguments ----------------------------------------------------- You can ask for available builtin or project-custom diff --git a/doc/en/conf.py b/doc/en/conf.py index 46235ca5d..f53a1757f 100644 --- a/doc/en/conf.py +++ b/doc/en/conf.py @@ -17,7 +17,7 @@ # # The full version, including alpha/beta/rc tags. # The short X.Y version. -version = release = "2.3.0.dev3" +version = release = "2.3.0.dev6" import sys, os diff --git a/doc/en/example/newexamples.txt b/doc/en/example/newexamples.txt deleted file mode 100644 index 9a4902b65..000000000 --- a/doc/en/example/newexamples.txt +++ /dev/null @@ -1,289 +0,0 @@ - -Scoping and parametrizing Funcarg factories ---------------------------------------------------- - -.. regendoc:wipe - -.. versionadded:: 2.3 - -The ``@pytest.mark.funcarg`` marker allows - -* to mark a function without a ``pytest_funcarg__`` as a factory -* to cause parametrization and run all tests multiple times - with the multiple created resources -* to set a scope which determines the level of caching - -Here is a simple example for defining a SMTPServer server -object with a session scope:: - - # content of conftest.py - import pytest - import smtplib - - @pytest.mark.funcarg(scope="session") - def smtp(request): - smtp = smtplib.SMTP("merlinux.eu") - request.addfinalizer(smtp.close) - return smtp - -You can now use this server connection from your tests:: - - # content of test_module.py - def test_ehlo(smtp): - response = smtp.ehlo() - assert response[0] == 250 - assert "merlinux" in response[1] - assert 0 # for demo purposes - - def test_noop(smtp): - response = smtp.noop() - assert response[0] == 250 - assert 0 # for demo purposes - -If you run the tests:: - - $ py.test -q - collecting ... collected 2 items - FF - ================================= FAILURES ================================= - ________________________________ test_ehlo _________________________________ - - smtp = - - def test_ehlo(smtp): - response = smtp.ehlo() - assert response[0] == 250 - assert "merlinux" in response[1] - > assert 0 # for demo purposes - E assert 0 - - test_module.py:5: AssertionError - ________________________________ test_noop _________________________________ - - smtp = - - def test_noop(smtp): - response = smtp.noop() - assert response[0] == 250 - > assert 0 # for demo purposes - E assert 0 - - test_module.py:10: AssertionError - 2 failed in 0.26 seconds - -you will see the two ``assert 0`` failing and can see that -the same (session-scoped) object was passed into the two test functions. - -If you now want to test multiple servers you can simply parametrize -the ``smtp`` factory:: - - # content of conftest.py - import pytest - import smtplib - - @pytest.mark.funcarg(scope="session", - params=["merlinux.eu", "mail.python.org"]) - def smtp(request): - smtp = smtplib.SMTP(request.param) - def fin(): - print "closing", smtp - smtp.close() - request.addfinalizer(fin) - return smtp - -Only two lines changed and no test code needs to change. Let's do -another run:: - - $ py.test -q - collecting ... collected 4 items - FFFF - ================================= FAILURES ================================= - __________________________ test_ehlo[merlinux.eu] __________________________ - - smtp = - - def test_ehlo(smtp): - response = smtp.ehlo() - assert response[0] == 250 - assert "merlinux" in response[1] - > assert 0 # for demo purposes - E assert 0 - - test_module.py:5: AssertionError - __________________________ test_noop[merlinux.eu] __________________________ - - smtp = - - def test_noop(smtp): - response = smtp.noop() - assert response[0] == 250 - > assert 0 # for demo purposes - E assert 0 - - test_module.py:10: AssertionError - ________________________ test_ehlo[mail.python.org] ________________________ - - smtp = - - def test_ehlo(smtp): - response = smtp.ehlo() - assert response[0] == 250 - > assert "merlinux" in response[1] - E assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN' - - test_module.py:4: AssertionError - ________________________ test_noop[mail.python.org] ________________________ - - smtp = - - def test_noop(smtp): - response = smtp.noop() - assert response[0] == 250 - > assert 0 # for demo purposes - E assert 0 - - test_module.py:10: AssertionError - 4 failed in 6.94 seconds - -We get four failures because we are running the two tests twice with -different ``smtp`` instantiations as defined on the factory. -Note that with the ``mail.python.org`` connection the second tests -fails already in ``test_ehlo`` because it wrongly expects a specific -server string. - -You can look at what tests pytest collects without running them:: - - $ py.test --collectonly - =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev5 - plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov - collecting ... collected 4 items - - - - - - - ============================= in 0.02 seconds ============================= - -And you can run without output capturing and minimized failure reporting to check that the ``smtp`` objects are finalized at session end:: - - $ py.test --tb=line -q -s - collecting ... collected 4 items - FFFF - ================================= FAILURES ================================= - /home/hpk/tmp/doc-exec-389/test_module.py:5: assert 0 - /home/hpk/tmp/doc-exec-389/test_module.py:10: assert 0 - /home/hpk/tmp/doc-exec-389/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN' - /home/hpk/tmp/doc-exec-389/test_module.py:10: assert 0 - 4 failed in 9.99 seconds - closing - closing - -.. _`new_setup`: - -``@pytest.mark.setup``: xUnit on steroids --------------------------------------------------------------------- - -.. regendoc:wipe - -.. versionadded:: 2.3 - -The ``@pytest.mark.setup`` marker allows - -* to define setup-functions close to test code or in conftest.py files - or plugins. -* to mark a function as a setup/fixture method; the function can itself - receive funcargs and will execute multiple times if the funcargs - are parametrized -* to set a scope which determines the level of caching and how often - the setup function is going to be called. - -Here is a simple example which configures a global funcarg without -the test needing to have it in its signature:: - - # content of conftest.py - import pytest - - @pytest.mark.funcarg(scope="module") - def resource(request, tmpdir): - def fin(): - print "finalize", tmpdir - request.addfinalizer(fin) - print "created resource", tmpdir - return tmpdir - -And the test file contains a setup function using this resource:: - - # content of test_module.py - import pytest - - @pytest.mark.setup(scope="module") - def setresource(resource): - print "setupresource", resource - global myresource - myresource = resource - - def test_1(): - assert myresource - print "using myresource", myresource - - def test_2(): - assert myresource - print "using myresource", myresource - -Let's run this module:: - - $ py.test -qs - collecting ... collected 2 items - .. - 2 passed in 0.62 seconds - created resource /home/hpk/tmp/pytest-4224/test_10 - setupresource /home/hpk/tmp/pytest-4224/test_10 - using myresource /home/hpk/tmp/pytest-4224/test_10 - using myresource /home/hpk/tmp/pytest-4224/test_10 - finalize /home/hpk/tmp/pytest-4224/test_10 - -The two test functions will see the same resource instance because it has -a module life cycle or scope. - -The resource funcarg can later add parametrization without any test -or setup code needing to change:: - - # content of conftest.py - import pytest - - @pytest.mark.funcarg(scope="module", params=["aaa", "bbb"]) - def resource(request, tmpdir): - newtmp = tmpdir.join(request.param) - def fin(): - print "finalize", newtmp - request.addfinalizer(fin) - print "created resource", newtmp - return newtmp - -Running this will run four tests:: - - $ py.test -qs - collecting ... collected 4 items - .... - 4 passed in 0.25 seconds - created resource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa - setupresource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa - using myresource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa - using myresource /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa - finalize /home/hpk/tmp/pytest-4225/test_1_aaa_0/aaa - created resource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb - setupresource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb - using myresource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb - using myresource /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb - finalize /home/hpk/tmp/pytest-4225/test_1_bbb_0/bbb - -Each parameter causes the creation of a respective resource and the -unchanged test module uses it in its ``@setup`` decorated method. - -.. note:: - - Parametrized Resources will be grouped together during test execution. - Moreover, any added finalizers will be run before the next parametrized - resource is being setup. diff --git a/doc/en/example/parametrize.txt b/doc/en/example/parametrize.txt index f4f8584f6..6512d8b70 100644 --- a/doc/en/example/parametrize.txt +++ b/doc/en/example/parametrize.txt @@ -329,87 +329,3 @@ Running it results in some skips if we don't have all the python interpreters in ========================= short test summary info ========================== SKIP [27] /home/hpk/p/pytest/doc/en/example/multipython.py:36: 'python2.8' not found 48 passed, 27 skipped in 1.70 seconds - -.. regendoc:wipe - -Grouping test execution by parameter ------------------------------------------ - -By default pytest will execute test functions by executing all its parametrized invocations. If you rather want to group execution by parameter, you can -use something like the following ``conftest.py`` example. It uses -a parametrized "resource" object:: - - # content of conftest.py - def pytest_collection_modifyitems(items): - def cmp(item1, item2): - param1 = item1.callspec.getparam("resource") - param2 = item2.callspec.getparam("resource") - if param1 < param2: - return -1 - elif param1 > param2: - return 1 - return 0 - items.sort(cmp=cmp) - - def pytest_generate_tests(metafunc): - if "resource" in metafunc.funcargnames: - metafunc.parametrize("resource", [1,2], indirect=True) - - class Resource: - def __init__(self, num): - self.num = num - def finalize(self): - print "finalize", self - - def pytest_funcarg__resource(request): - return request.cached_setup(lambda: Resource(request.param), - teardown=lambda res: res.finalize(), - extrakey=request.param) - - -If you have a test file like this:: - - # content of test_resource.py - def test_hello(resource): - pass - - def test_world(resource): - pass - - class TestClass: - def test_method1(self, resource): - pass - def test_method2(self, resource): - pass - -then a subsequent execution will order the running of tests by -parameter value:: - - $ py.test -v -s - =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev3 -- /home/hpk/venv/1/bin/python - cachedir: /home/hpk/tmp/doc-exec-340/.cache - plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov - collecting ... collected 8 items - - test_resource.py:1: test_hello[1] PASSED - test_resource.py:4: test_world[1] PASSED - test_resource.py:8: TestClass.test_method1[1] PASSED - test_resource.py:10: TestClass.test_method2[1] PASSED - test_resource.py:1: test_hello[2] PASSED - test_resource.py:4: test_world[2] PASSED - test_resource.py:8: TestClass.test_method1[2] PASSED - test_resource.py:10: TestClass.test_method2[2] PASSED - - ========================= 8 passed in 0.03 seconds ========================= - finalize - finalize - -.. note:: - Despite the per-session ordering the finalize() of the session-scoped - resource executes at the end of the whole test session. The life - cycle of the two parametrized instantiated resources will thus overlap. - One possible workaround is to make the resource instantiations be - aware of each other and teardown the other one before returning a new - resource. There are plans for future releases of pytest to offer an - out-of-the-box way to support session-ordering. diff --git a/doc/en/funcarg_compare.txt b/doc/en/funcarg_compare.txt new file mode 100644 index 000000000..e28248724 --- /dev/null +++ b/doc/en/funcarg_compare.txt @@ -0,0 +1,228 @@ + +V5: changes to new resource/setup facilities +============================================================= + +**Target audience**: Reading this document requires basic knowledge of +python testing, xUnit setup methods and the basic pytest funcarg mechanism, +see http://pytest.org/latest/funcargs.html + + +**Changes**: This V5 draft is based on incorporating and thinking about +feedback on previous versions provided by Floris Bruynooghe, Carl Meyer, +Ronny Pfannschmidt and Samuele Pedroni. I have also now implemented it +which triggered a number of refinements as well. The main changes are: + +* Collapse funcarg factory decorators into a single "@resource" one. + You can specify scopes and params with it. When using the decorator + the "pytest_funcarg__" prefix is not allowed and the old-style + ``request`` object cannot be received. + +* funcarg resource factories can now use funcargs themselves + +* Drop setup/directory scope from this draft + +* introduce a new @setup decorator similar to the @funcarg one + except that setup-markers cannot define parametriation themselves. + Instead they can easily depend on a parametrized funcarg (which + must not be visible at test function signatures). + +* drop consideration of setup_X support for funcargs because + it is less flexible and probably causes more implementation + troubles than the current @setup approach which can share + a lot of logic with the @funcarg one. + +* tests are grouped by parametrized funcargs and according to scope + (sounds like a small thing but is a big deal) + +* make the new-style funcargs/setup use a "testcontext" object + which offers test context info and addfinalizer() methods but no + getfuncargvalue()/cached_setup()/applymarker anymore. Reason + being that getfuncargvalue()/cached_setup breaks other features + such as sorting by resource-scope and parametrization + + +.. currentmodule:: _pytest + +Shortcomings of the previous pytest_funcarg__ mechanism +--------------------------------------------------------- + +The previous funcarg mechanism calls a factory each time a +funcarg for a test function is testcontexted. If a factory wants +t re-use a resource across different scopes, it often used +the ``testcontext.cached_setup()`` helper to manage caching of +resources. Here is a basic example how we could implement +a per-session Database object:: + + # content of conftest.py + class Database: + def __init__(self): + print ("database instance created") + def destroy(self): + print ("database instance destroyed") + + def pytest_funcarg__db(request): + return request.cached_setup(setup=DataBase, + teardown=lambda db: db.destroy, + scope="session") + +There are several limitations and difficulties with this approach: + +1. Scoping funcarg resource creation is not straight forward, instead one must + understand the intricate cached_setup() method mechanics. + +2. parametrizing the "db" resource is not straight forward: + you need to apply a "parametrize" decorator or implement a + :py:func:`~hookspec.pytest_generate_tests` hook + calling :py:func:`~python.Metafunc.parametrize` which + performs parametrization at the places where the resource + is used. Moreover, you need to modify the factory to use an + ``extrakey`` parameter containing ``request.param`` to the + :py:func:`~python.Request.cached_setup` call. + +3. Multiple parametrized session-scoped resources will be active + at the same time, making it hard for them to affect global state + of the application under test. + +4. there is no way how you can make use of funcarg factories + in xUnit setup methods. + +5. A non-parametrized funcarg factory cannot use a parametrized + funcarg resource if it isn't stated in the test function signature. + +All of these limitations are addressed with pytest-2.3 and its +new facilities. + +Direct scoping of funcarg factories +-------------------------------------------------------- + +Instead of calling cached_setup(), you can decorate your factory +to state its scope:: + + @pytest.mark.resource(scope="session") + def db(testcontext): + # factory will only be invoked once per session - + db = DataBase() + testcontext.addfinalizer(db.destroy) # destroy when session is finished + return db + +This factory implementation does not need to call ``cached_setup()`` anymore +because it will only be invoked once per session. Moreover, the +``testcontext.addfinalizer()`` registers a finalizer according to the specified +resource scope on which the factory function is operating. With this new +scoping, the still existing ``cached_setup()`` should be much less used +but will remain for compatibility reasons and for the case where you +still want to have your factory get called on a per-item basis. + + +Direct parametrization of funcarg resource factories +---------------------------------------------------------- + +.. note:: Implemented + +Previously, funcarg factories could not directly cause parametrization. +You needed to specify a ``@parametrize`` or implement a ``pytest_generate_tests`` hook to perform parametrization, i.e. calling a test multiple times +with different value sets. pytest-2.X introduces a decorator for use +on the factory itself:: + + @pytest.mark.resource(params=["mysql", "pg"]) + def pytest_funcarg__db(testcontext): + ... + +Here the factory will be invoked twice (with the respective "mysql" +and "pg" values set as ``testcontext.param`` attributes) and and all of +the tests requiring "db" will run twice as well. The "mysql" and +"pg" values will also be used for reporting the test-invocation variants. + +This new way of parametrizing funcarg factories should in many cases +allow to re-use already written factories because effectively +``testcontext.param`` are already the parametrization attribute for test +functions/classes were parametrized via +:py:func:`~_pytest.python.Metafunc.parametrize(indirect=True)` calls. + +Of course it's perfectly fine to combine parametrization and scoping:: + + @pytest.mark.resource(scope="session", params=["mysql", "pg"]) + def pytest_funcarg__db(testcontext): + if testcontext.param == "mysql": + db = MySQL() + elif testcontext.param == "pg": + db = PG() + testcontext.addfinalizer(db.destroy) # destroy when session is finished + return db + +This would execute all tests requiring the per-session "db" resource twice, +receiving the values created by the two respective invocations to the +factory function. + + +No ``pytest_funcarg__`` prefix when using @resource decorator +------------------------------------------------------------------- + + +.. note:: Implemented + +When using the ``@funcarg`` decorator the name of the function +does not need to (and in fact cannot) use the ``pytest_funcarg__`` +naming:: + + @pytest.mark.resource + def db(testcontext): + ... + +The name under which the funcarg resource can be requested is ``db``. + +You can also use the "old" non-decorator way of specifying funcarg factories +aka:: + + def pytest_funcarg__db(testcontext): + ... + +It is recommended to use the resource decorator, however. + + +solving per-session setup / the new @setup marker +-------------------------------------------------------------- + +.. note:: Implemented, at least working for basic situations. + +pytest for a long time offered a pytest_configure and a pytest_sessionstart +hook which are often used to setup global resources. This suffers from +several problems: + +1. in distributed testing the master process would setup test resources + that are never needed because it only co-ordinates the test run + activities of the slave processes. + +2. if you only perform a collection (with "--collectonly") + resource-setup will still be executed. + +3. If a pytest_sessionstart is contained in some subdirectories + conftest.py file, it will not be called. This stems from the + fact that this hook is actually used for reporting, in particular + the test-header with platform/custom information. + +Moreover, it is today not easy to define a scoped setup from plugins or +conftest files other than to implement a ``pytest_runtest_setup()`` hook +and caring for scoping/caching yourself. And it's virtually impossible +to do this with parametrization as ``pytest_runtest_setup()`` is called +during test execution and parametrization happens at collection time. + +It follows that pytest_configure/session/runtest_setup are often not +appropriate for implementing common fixture needs. Therefore, +pytest-2.X introduces a new "@pytest.mark.setup" marker which takes +an optional "scope" parameter. + +See :ref:`new_setup` for examples. + +funcarg and setup discovery now happens at collection time +--------------------------------------------------------------------- + +.. note:: + Partially implemented - collectonly shows no extra information however. + +pytest-2.X takes care to discover funcarg factories and @setup methods +at collection time. This is more efficient especially for large test suites. +Moreover, a call to "py.test --collectonly" should be able to show +a lot of setup-information and thus presents a nice method to get an +overview of resource management in your project. + diff --git a/doc/en/funcargs.txt b/doc/en/funcargs.txt index e53c49098..5999cf6d1 100644 --- a/doc/en/funcargs.txt +++ b/doc/en/funcargs.txt @@ -11,6 +11,13 @@ Injecting objects into test functions (funcargs) Dependency injection through function arguments ================================================= +.. note:: + + This section describes the pytest mechanisms prior + to the pytest-2.3 release. If you haven't used these + features yet, it makes more sense to stop here and read + :ref:`resources` instead. + py.test lets you inject objects into test invocations and precisely control their life cycle in relation to the overall test execution. Moreover, you can run a test function multiple times injecting different objects. @@ -39,7 +46,7 @@ very useful if you want to test e.g. against different database backends or with multiple numerical arguments sets and want to reuse the same set of test functions. -py.test comes with some :ref:`builtinfuncargs` and there are some refined usages in the examples section. +py.test comes with some :ref:`builtinresources` and there are some refined usages in the examples section. .. _funcarg: diff --git a/doc/en/index.txt b/doc/en/index.txt index 944e50c2f..dd4eb44de 100644 --- a/doc/en/index.txt +++ b/doc/en/index.txt @@ -25,11 +25,11 @@ Welcome to pytest! - **supports functional testing and complex test setups** + - (new in 2.3) :ref:`easy test resource management and generalized xUnit setup ` - (new in 2.2) :ref:`durations` - (much improved in 2.2) :ref:`marking and test selection ` - (improved in 2.2) :ref:`parametrized test functions ` - advanced :ref:`skip and xfail` - - unique :ref:`dependency injection through funcargs ` - can :ref:`distribute tests to multiple CPUs ` through :ref:`xdist plugin ` - can :ref:`continuously re-run failing tests ` - many :ref:`builtin helpers ` diff --git a/doc/en/resources.txt b/doc/en/resources.txt index 4c2c301a1..2c410b900 100644 --- a/doc/en/resources.txt +++ b/doc/en/resources.txt @@ -1,271 +1,414 @@ -V4: Creating and working with parametrized resources -=============================================================== +.. _resources: -**Target audience**: Reading this document requires basic knowledge of -python testing, xUnit setup methods and the basic pytest funcarg mechanism, -see http://pytest.org/latest/funcargs.html +test resource management and xUnit setup (on steroids) +======================================================= -**Abstract**: pytest-2.X provides yet more powerful and flexible -fixture machinery by introducing: +.. _`Dependency injection`: http://en.wikipedia.org/wiki/Dependency_injection -* a new ``@pytest.mark.funcarg`` marker to define funcarg factories and their - scoping and parametrization. No special ``pytest_funcarg__`` naming there. +.. versionadded: 2.3 -* a new ``@pytest.mark.setup`` marker to define setup functions and their +pytest offers advanced resource parametrization and injection mechanisms +including a fully integrated generalization of the popular xUnit +setup-style methods. A resource is created by a ``@pytest.mark.factory`` +marked function and its name is the name of the function. A resource +is injected into test or setup functions if they use the name +in their signature. Therefore and also for historic reasons, resources are +sometimes called "funcargs" because they ultimately appear as +function arguments. + +The pytest resource management and setup features are exposed through +three decorators: + +* a `@pytest.mark.factory`_ marker to define resource factories, + their scoping and parametrization. + +* a `@pytest.mark.setup`_ marker to define setup functions and their scoping. -* directly use funcargs through funcarg factory signatures +* a `@pytest.mark.parametrize`_ marker for executing test functions + multiple times with different parameter sets -Both funcarg factories and setup functions can be defined in test modules, -classes, conftest.py files and installed plugins. +Generally, resource factories and setup functions: -The introduction of these two markers lifts several prior limitations -and allows to easily define and implement complex testing scenarios. +- can be defined in test modules, test classes, conftest.py files or + in plugins. -Nonwithstanding these extensions, already existing test suites and plugins -written to work for previous pytest versions shall run unmodified. +- can themselves receive resources through their function arguments, + simplifying the setup and use of interdependent resources. + +- can use the special `testcontext`_ object for access to the + context in which the factory/setup is called and for registering + finalizers. + +This document showcases these features through some basic examples. + +Note that pytest also comes with some :ref:`builtinresources` which +you can use without defining them yourself. + +Background and terms +--------------------------- + +The pytest resource management mechanism is an example of `Dependency +Injection`_ which helps to de-couple test code from resource +instantiation code required for them to execute. At test writing time +you typically do not need to care for the details of how your required +resources are constructed, if they live through a function, class, +module or session scope or if the test will be called multiple times +with different resource instances. + +To create a value with which to call a test function a resource factory +function is called which gets full access to the test context and can +register finalizers which are to be run after the last test in that context +finished. Resource factories can be implemented in same test class or +test module, in a per-directory ``conftest.py`` file or in an external plugin. This allows total de-coupling of test and setup code. + +A test function may be invoked multiple times in which case we +speak of :ref:`parametrized testing `. This can be +very useful if you want to test e.g. against different database backends +or with multiple numerical arguments sets and want to reuse the same set +of test functions. -**Changes**: This V4 draft is based on incorporating and thinking about -feedback on previous versions provided by Floris Bruynooghe, Carl Meyer, -Ronny Pfannschmidt and Samuele Pedroni. It remains as draft -documentation, pending further refinements and changes according to -implementation or backward compatibility issues. The main changes are: +.. _`@pytest.mark.factory`: -* Collapse funcarg factory decorators into a single "@funcarg" one. - You can specify scopes and params with it. When using the decorator - the "pytest_funcarg__" prefix becomes optional. +``@pytest.mark.factory``: Creating parametrized, scoped resources +----------------------------------------------------------------- -* funcarg factories can now use funcargs themselves +.. regendoc:wipe -* Drop setup/directory scope from this draft +.. versionadded:: 2.3 -* introduce a new @setup decorator similar to the @funcarg one - except that setup-markers cannot define parametriation themselves. - Instead they can easily depend on a parametrized funcarg (which - must not be visible at test function signatures). +The `@pytest.mark.factory`_ marker allows to -* drop consideration of setup_X support for funcargs because - it is less flexible and probably causes more implementation - troubles than the current @setup approach which can share - a lot of logic with the @funcarg one. +* mark a function as a factory for resources used by test and setup functions +* define parametrization to run tests multiple times with different + resource instances +* set a scope which determines the level of caching. valid scopes + are ``session``, ``module``, ``class`` and ``function``. -* tests are grouped by parametrized funcargs +Here is a simple example of a factory creating a shared ``smtplib.SMTP`` +connection resource which test functions then may use across the whole +test session:: -.. currentmodule:: _pytest + # content of conftest.py + import pytest + import smtplib + + @pytest.mark.factory(scope="session") + def smtp(testcontext): + smtp = smtplib.SMTP("merlinux.eu") + testcontext.addfinalizer(smtp.close) + return smtp + +The name of the resource is ``smtp`` (the factory function name) +and you can now access the ``smtp`` resource by listing it as +an input parameter in any test function below the directory where +``conftest.py`` is located:: + + # content of test_module.py + def test_ehlo(smtp): + response = smtp.ehlo() + assert response[0] == 250 + assert "merlinux" in response[1] + assert 0 # for demo purposes + + def test_noop(smtp): + response = smtp.noop() + assert response[0] == 250 + assert 0 # for demo purposes + +If you run the tests:: + + $ py.test -q + collecting ... collected 2 items + FF + ================================= FAILURES ================================= + ________________________________ test_ehlo _________________________________ + + smtp = + + def test_ehlo(smtp): + response = smtp.ehlo() + assert response[0] == 250 + assert "merlinux" in response[1] + > assert 0 # for demo purposes + E assert 0 + + test_module.py:5: AssertionError + ________________________________ test_noop _________________________________ + + smtp = + + def test_noop(smtp): + response = smtp.noop() + assert response[0] == 250 + > assert 0 # for demo purposes + E assert 0 + + test_module.py:10: AssertionError + 2 failed in 0.18 seconds + +you will see the two ``assert 0`` failing and can see that +the same (session-scoped) object was passed into the two test functions. + +If you now want to test multiple servers you can simply parametrize +the ``smtp`` factory:: + + # content of conftest.py + import pytest + import smtplib + + @pytest.mark.factory(scope="session", + params=["merlinux.eu", "mail.python.org"]) + def smtp(testcontext): + smtp = smtplib.SMTP(testcontext.param) + def fin(): + smtp.close() + testcontext.addfinalizer(fin) + return smtp + +The main change is the definition of a ``params`` list in the +``factory``-marker and the ``testcontext.param`` access within the +factory function. No test code needs to change. So let's just do another +run:: + + $ py.test -q + collecting ... collected 4 items + FFFF + ================================= FAILURES ================================= + __________________________ test_ehlo[merlinux.eu] __________________________ + + smtp = + + def test_ehlo(smtp): + response = smtp.ehlo() + assert response[0] == 250 + assert "merlinux" in response[1] + > assert 0 # for demo purposes + E assert 0 + + test_module.py:5: AssertionError + __________________________ test_noop[merlinux.eu] __________________________ + + smtp = + + def test_noop(smtp): + response = smtp.noop() + assert response[0] == 250 + > assert 0 # for demo purposes + E assert 0 + + test_module.py:10: AssertionError + ________________________ test_ehlo[mail.python.org] ________________________ + + smtp = + + def test_ehlo(smtp): + response = smtp.ehlo() + assert response[0] == 250 + > assert "merlinux" in response[1] + E assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN' + + test_module.py:4: AssertionError + ________________________ test_noop[mail.python.org] ________________________ + + smtp = + + def test_noop(smtp): + response = smtp.noop() + assert response[0] == 250 + > assert 0 # for demo purposes + E assert 0 + + test_module.py:10: AssertionError + 4 failed in 6.42 seconds + +We get four failures because we are running the two tests twice with +different ``smtp`` instantiations as defined on the factory. +Note that with the ``mail.python.org`` connection the second test +fails in ``test_ehlo`` because it expects a specific server string. + +You can also look at what tests pytest collects without running them:: + + $ py.test --collectonly + =========================== test session starts ============================ + platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev7 + plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov + collecting ... collected 4 items + + + + + + + ============================= in 0.02 seconds ============================= + +Note that pytest orders your test run by resource usage, minimizing +the number of active resources at any given time. -Shortcomings of the previous pytest_funcarg__ mechanism ---------------------------------------------------------- - -The previous funcarg mechanism calls a factory each time a -funcarg for a test function is requested. If a factory wants -t re-use a resource across different scopes, it often used -the ``request.cached_setup()`` helper to manage caching of -resources. Here is a basic example how we could implement -a per-session Database object:: - - # content of conftest.py - class Database: - def __init__(self): - print ("database instance created") - def destroy(self): - print ("database instance destroyed") - - def pytest_funcarg__db(request): - return request.cached_setup(setup=DataBase, - teardown=lambda db: db.destroy, - scope="session") - -There are some problems with this approach: - -1. Scoping resource creation is not straight forward, instead one must - understand the intricate cached_setup() method mechanics. - -2. parametrizing the "db" resource is not straight forward: - you need to apply a "parametrize" decorator or implement a - :py:func:`~hookspec.pytest_generate_tests` hook - calling :py:func:`~python.Metafunc.parametrize` which - performs parametrization at the places where the resource - is used. Moreover, you need to modify the factory to use an - ``extrakey`` parameter containing ``request.param`` to the - :py:func:`~python.Request.cached_setup` call. - -3. there is no way how you can make use of funcarg factories - in xUnit setup methods. - -4. A non-parametrized funcarg factory cannot use a parametrized - funcarg resource if it isn't stated in the test function signature. - -The following sections address the advances which solve all of these problems. - - -Direct scoping of funcarg factories --------------------------------------------------------- - -.. note:: Implemented - -Instead of calling cached_setup(), you can decorate your factory -to state its scope:: - - @pytest.mark.funcarg(scope="session") - def pytest_funcarg__db(request): - # factory will only be invoked once per session - - db = DataBase() - request.addfinalizer(db.destroy) # destroy when session is finished - return db - -This factory implementation does not need to call ``cached_setup()`` anymore -because it will only be invoked once per session. Moreover, the -``request.addfinalizer()`` registers a finalizer according to the specified -resource scope on which the factory function is operating. With this new -scoping, the still existing ``cached_setup()`` should be much less used -but will remain for compatibility reasons and for the case where you -still want to have your factory get called on a per-item basis. - - -Direct parametrization of funcarg resource factories +Accessing resources from a factory function ---------------------------------------------------------- -.. note:: Implemented +You can directly use resources as funcargs in resource factories. +Extending the previous example we can instantiate an application +object and stick the live ``smtp`` resource into it:: -Previously, funcarg factories could not directly cause parametrization. -You needed to specify a ``@parametrize`` or implement a ``pytest_generate_tests`` hook to perform parametrization, i.e. calling a test multiple times -with different value sets. pytest-2.X introduces a decorator for use -on the factory itself:: + # content of test_appsetup.py + + import pytest - @pytest.mark.funcarg(params=["mysql", "pg"]) - def pytest_funcarg__db(request): - ... + class App: + def __init__(self, smtp): + self.smtp = smtp -Here the factory will be invoked twice (with the respective "mysql" -and "pg" values set as ``request.param`` attributes) and and all of -the tests requiring "db" will run twice as well. The "mysql" and -"pg" values will also be used for reporting the test-invocation variants. + @pytest.mark.factory(scope="module") + def app(smtp): + return App(smtp) -This new way of parametrizing funcarg factories should in many cases -allow to re-use already written factories because effectively -``request.param`` are already the parametrization attribute for test -functions/classes were parametrized via -:py:func:`~_pytest.python.Metafunc.parametrize(indirect=True)` calls. + def test_exists(app): + assert app.smtp -Of course it's perfectly fine to combine parametrization and scoping:: +Let's run this:: - @pytest.mark.funcarg(scope="session", params=["mysql", "pg"]) - def pytest_funcarg__db(request): - if request.param == "mysql": - db = MySQL() - elif request.param == "pg": - db = PG() - request.addfinalizer(db.destroy) # destroy when session is finished - return db + $ py.test -v test_appsetup.py + =========================== test session starts ============================ + platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev7 -- /home/hpk/venv/1/bin/python + cachedir: /home/hpk/tmp/doc-exec-398/.cache + plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov + collecting ... collected 2 items + + test_appsetup.py:12: test_exists[merlinux.eu] PASSED + test_appsetup.py:12: test_exists[mail.python.org] PASSED + + ========================= 2 passed in 5.96 seconds ========================= -This would execute all tests requiring the per-session "db" resource twice, -receiving the values created by the two respective invocations to the -factory function. - -Direct usage of funcargs with funcargs factories ----------------------------------------------------------- - -.. note:: Implemented. - -You can now directly use funcargs in funcarg factories. Example:: - - @pytest.mark.funcarg(scope="session") - def db(request, tmpdir): - # tmpdir is a session-specific tempdir - -Apart from convenience it also solves an issue when your factory -depends on a parametrized funcarg. Previously, a call to -``request.getfuncargvalue()`` happens at test execution time and -thus pytest would not know at collection time about the fact that -a required resource is parametrized. - -No ``pytest_funcarg__`` prefix when using @funcarg decorator -------------------------------------------------------------------- +Due to the parametrization of ``smtp`` the test will +run twice with two different ``App`` instances and respective smtp servers. +There is no need for the ``app`` factory to be aware of the parametrization. -.. note:: Implemented +.. _`new_setup`: +.. _`@pytest.mark.setup`: -When using the ``@funcarg`` decorator the name of the function -does not need to (and in fact cannot) use the ``pytest_funcarg__`` -naming:: +``@pytest.mark.setup``: xUnit setup methods on steroids +----------------------------------------------------------------- - @pytest.mark.funcarg - def db(request): - ... +.. regendoc:wipe -The name under which the funcarg resource can be requested is ``db``. +.. versionadded:: 2.3 -You can also use the "old" non-decorator way of specifying funcarg factories -aka:: +The ``@pytest.mark.setup`` marker allows - def pytest_funcarg__db(request): - ... +* to define setup-functions close to test code or in conftest.py files + or plugins. +* to mark a function as a setup method; the function can itself + receive funcargs and will execute multiple times if the funcargs + are parametrized +* to set a scope which influences when the setup function going to be + called. valid scopes are ``session``, ``module``, ``class`` and ``function``. -It is recommended to use the funcarg-decorator, however. +Here is a simple example. First we define a global ``globdir`` resource:: + # content of conftest.py + import pytest -solving per-session setup / the new @setup marker --------------------------------------------------------------- + @pytest.mark.factory(scope="module") + def globdir(testcontext, tmpdir): + def fin(): + print "finalize", tmpdir + testcontext.addfinalizer(fin) + print "created resource", tmpdir + return tmpdir -.. note:: Implemented, at least working for basic situations. +And then we write a test file containing a setup-marked function +taking this resource and setting it as a module global:: -pytest for a long time offered a pytest_configure and a pytest_sessionstart -hook which are often used to setup global resources. This suffers from -several problems: + # content of test_module.py + import pytest -1. in distributed testing the master process would setup test resources - that are never needed because it only co-ordinates the test run - activities of the slave processes. + @pytest.mark.setup(scope="module") + def setresource(testcontext, globdir): + print "setupresource", globdir + testcontext.module.myresource = globdir -2. if you only perform a collection (with "--collectonly") - resource-setup will still be executed. + def test_1(): + assert myresource + print "using myresource", myresource -3. If a pytest_sessionstart is contained in some subdirectories - conftest.py file, it will not be called. This stems from the - fact that this hook is actually used for reporting, in particular - the test-header with platform/custom information. + def test_2(): + assert myresource + print "using myresource", myresource -Moreover, it is today not easy to define a scoped setup from plugins or -conftest files other than to implement a ``pytest_runtest_setup()`` hook -and caring for scoping/caching yourself. And it's virtually impossible -to do this with parametrization as ``pytest_runtest_setup()`` is called -during test execution and parametrization happens at collection time. +Let's run this module:: -It follows that pytest_configure/session/runtest_setup are often not -appropriate for implementing common fixture needs. Therefore, -pytest-2.X introduces a new "@pytest.mark.setup" marker which takes -an optional "scope" parameter. + $ py.test -qs + collecting ... collected 2 items + .. + 2 passed in 0.26 seconds + created resource /home/hpk/tmp/pytest-4427/test_10 + setupresource /home/hpk/tmp/pytest-4427/test_10 + using myresource /home/hpk/tmp/pytest-4427/test_10 + using myresource /home/hpk/tmp/pytest-4427/test_10 + finalize /home/hpk/tmp/pytest-4427/test_10 -See :ref:`new_setup` for examples. +The two test functions in the module use the same global ``myresource`` +object because the ``setresource`` set it as a module attribute. -funcarg and setup discovery now happens at collection time ---------------------------------------------------------------------- +The ``globdir`` factory can now become parametrized without any test +or setup code needing to change:: -.. note:: - Partially implemented - collectonly shows no extra information however. + # content of conftest.py + import pytest -pytest-2.X takes care to discover funcarg factories and @setup methods -at collection time. This is more efficient especially for large test suites. -Moreover, a call to "py.test --collectonly" should be able to show -a lot of setup-information and thus presents a nice method to get an -overview of resource management in your project. + @pytest.mark.factory(scope="module", params=["aaa", "bbb"]) + def globdir(testcontext, tmpdir): + newtmp = tmpdir.join(testcontext.param) + def fin(): + print "finalize", newtmp + testcontext.addfinalizer(fin) + print "created resource", newtmp + return newtmp +Running the unchanged previous test files now runs four tests:: + + $ py.test -qs + collecting ... collected 4 items + .... + 4 passed in 0.26 seconds + created resource /home/hpk/tmp/pytest-4428/test_1_aaa_0/aaa + setupresource /home/hpk/tmp/pytest-4428/test_1_aaa_0/aaa + using myresource /home/hpk/tmp/pytest-4428/test_1_aaa_0/aaa + using myresource /home/hpk/tmp/pytest-4428/test_1_aaa_0/aaa + finalize /home/hpk/tmp/pytest-4428/test_1_aaa_0/aaa + created resource /home/hpk/tmp/pytest-4428/test_1_bbb_0/bbb + setupresource /home/hpk/tmp/pytest-4428/test_1_bbb_0/bbb + using myresource /home/hpk/tmp/pytest-4428/test_1_bbb_0/bbb + using myresource /home/hpk/tmp/pytest-4428/test_1_bbb_0/bbb + finalize /home/hpk/tmp/pytest-4428/test_1_bbb_0/bbb + +Each parameter causes the creation of a respective resource and the +unchanged test module uses it in its ``@setup`` decorated method. + +.. note:: + + Tests using a particular parametrized resource instance will + executed next to each other. Any finalizers will be run before the + next parametrized resource instance is being setup and tests + are rerun. Grouping tests by resource parameters ---------------------------------------------------------- -.. note:: Implemented. +.. regendoc: wipe -pytest used to always sort test items by their source location. -With pytest-2.X tests are first grouped by funcarg parameters. -If you have a parametrized funcarg, then all the tests using it -will first execute with it. Then any finalizers are called and then -the next parametrized resource instance is created and its tests are run. -Among other things, this eases testing of applications which create -and use global state. +pytest minimizes the number of active resources during test runs. +If you have a parametrized resource, then all the tests using one +resource instance will execute one after another. Then any finalizers +are called for that resource instance and then the next parametrized +resource instance is created and its tests are run. Among other things, +this eases testing of applications which create and use global state. The following example uses two parametrized funcargs, one of which is scoped on a per-module basis:: @@ -273,18 +416,18 @@ scoped on a per-module basis:: # content of test_module.py import pytest - @pytest.mark.funcarg(scope="module", params=["mod1", "mod2"]) - def modarg(request): - param = request.param + @pytest.mark.factory(scope="module", params=["mod1", "mod2"]) + def modarg(testcontext): + param = testcontext.param print "create", param def fin(): print "fin", param - request.addfinalizer(fin) + testcontext.addfinalizer(fin) return param - @pytest.mark.funcarg(scope="function", params=[1,2]) - def otherarg(request): - return request.param + @pytest.mark.factory(scope="function", params=[1,2]) + def otherarg(testcontext): + return testcontext.param def test_0(otherarg): print " test0", otherarg @@ -297,8 +440,8 @@ Let's run the tests in verbose mode and with looking at the print-output:: $ py.test -v -s =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev5 -- /home/hpk/venv/1/bin/python - cachedir: /home/hpk/tmp/doc-exec-388/.cache + platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev7 -- /home/hpk/venv/1/bin/python + cachedir: /home/hpk/tmp/doc-exec-398/.cache plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov collecting ... collected 8 items @@ -325,10 +468,132 @@ Let's run the tests in verbose mode and with looking at the print-output:: test2 2 mod2 fin mod2 -You can see that that the parametrized ``modarg`` resource lead to -a re-ordering of test execution. The finalizer for the "mod1" parametrized -resource was executed before the "mod2" resource was setup. +You can see that the parametrized module-scoped ``modarg`` resource caused +a re-ordering of test execution. The finalizer for the ``mod1`` parametrized +resource was executed before the ``mod2`` resource was setup. -.. note:: +.. currentmodule:: _pytest.python +.. _`testcontext`: - The current implementation is experimental. +``testcontext``: interacting with test context +--------------------------------------------------- + +The ``testcontext`` object may be received by `@pytest.mark.factory`_ or +`@pytest.mark.setup`_ marked functions. It contains information relating +to the test context within which the function executes. Moreover, you +can call ``testcontext.addfinalizer(myfinalizer)`` in order to trigger +a call to ``myfinalizer`` after the last test in the test context has executed. +If passed to a parametrized factory ``testcontext.param`` will contain +a parameter (one value out of the ``params`` list specified with the +`@pytest.mark.factory`_ marker). + +.. autoclass:: _pytest.python.TestContext() + :members: + +.. _`@pytest.mark.parametrize`: + +``@pytest.mark.parametrize``: directly parametrizing test functions +---------------------------------------------------------------------------- + +.. versionadded:: 2.2 + +The builtin ``pytest.mark.parametrize`` decorator enables +parametrization of arguments for a test function. Here is an example +of a test function that wants check for expected output given a certain input:: + + # content of test_expectation.py + import pytest + @pytest.mark.parametrize(("input", "expected"), [ + ("3+5", 8), + ("2+4", 6), + ("6*9", 42), + ]) + def test_eval(input, expected): + assert eval(input) == expected + +we parametrize two arguments of the test function so that the test +function is called three times. Let's run it:: + + $ py.test -q + collecting ... collected 11 items + ..F........ + ================================= FAILURES ================================= + ____________________________ test_eval[6*9-42] _____________________________ + + input = '6*9', expected = 42 + + @pytest.mark.parametrize(("input", "expected"), [ + ("3+5", 8), + ("2+4", 6), + ("6*9", 42), + ]) + def test_eval(input, expected): + > assert eval(input) == expected + E assert 54 == 42 + E + where 54 = eval('6*9') + + test_expectation.py:8: AssertionError + 1 failed, 10 passed in 0.04 seconds + +As expected only one pair of input/output values fails the simple test function. + +Note that there are various ways how you can mark groups of functions, +see :ref:`mark`. + +Generating parameters combinations, depending on command line +---------------------------------------------------------------------------- + +.. regendoc:wipe + +Let's say we want to execute a test with different computation +parameters and the parameter range shall be determined by a command +line argument. Let's first write a simple (do-nothing) computation test:: + + # content of test_compute.py + + def test_compute(param1): + assert param1 < 4 + +Now we add a test configuration like this:: + + # content of conftest.py + + def pytest_addoption(parser): + parser.addoption("--all", action="store_true", + help="run all combinations") + + def pytest_generate_tests(metafunc): + if 'param1' in metafunc.funcargnames: + if metafunc.config.option.all: + end = 5 + else: + end = 2 + metafunc.parametrize("param1", range(end)) + +This means that we only run 2 tests if we do not pass ``--all``:: + + $ py.test -q test_compute.py + collecting ... collected 2 items + .. + 2 passed in 0.03 seconds + +We run only two computations, so we see two dots. +let's run the full monty:: + + $ py.test -q --all + collecting ... collected 5 items + ....F + ================================= FAILURES ================================= + _____________________________ test_compute[4] ______________________________ + + param1 = 4 + + def test_compute(param1): + > assert param1 < 4 + E assert 4 < 4 + + test_compute.py:3: AssertionError + 1 failed, 4 passed in 0.03 seconds + +As expected when running the full range of ``param1`` values +we'll get an error on the last one. diff --git a/setup.py b/setup.py index 78d9826ab..b6f2ff1cc 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def main(): name='pytest', description='py.test: simple powerful testing with Python', long_description = long_description, - version='2.3.0.dev6', + version='2.3.0.dev7', url='http://pytest.org', license='MIT license', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'],