From 4766497515169d40b9308825de0001b2e5766ca8 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 16 Jul 2012 11:11:26 +0200 Subject: [PATCH] V3 draft of resource api --- _pytest/main.py | 2 +- doc/en/conf.py | 1 + doc/en/contents.txt | 3 +- doc/en/example/resources.txt | 369 ------------------------------ doc/en/funcargs.txt | 3 +- doc/en/plugins.txt | 4 +- doc/en/resources.txt | 426 +++++++++++++++++++++++++++++++++++ 7 files changed, 435 insertions(+), 373 deletions(-) delete mode 100644 doc/en/example/resources.txt create mode 100644 doc/en/resources.txt diff --git a/_pytest/main.py b/_pytest/main.py index 6ecb12dbd..5e9e38bc8 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -610,6 +610,7 @@ class Session(FSCollector): yield x node.ihook.pytest_collectreport(report=rep) + # XXX not used yet def register_resource_factory(self, name, factoryfunc, matchscope=None, cachescope=None): @@ -634,4 +635,3 @@ class Session(FSCollector): - diff --git a/doc/en/conf.py b/doc/en/conf.py index 5a0f623b1..59fc8089e 100644 --- a/doc/en/conf.py +++ b/doc/en/conf.py @@ -27,6 +27,7 @@ import sys, os #sys.path.insert(0, os.path.abspath('.')) autodoc_member_order = "bysource" +todo_include_todos = 1 # -- General configuration ----------------------------------------------------- diff --git a/doc/en/contents.txt b/doc/en/contents.txt index c5dad99b3..871be88fb 100644 --- a/doc/en/contents.txt +++ b/doc/en/contents.txt @@ -23,5 +23,6 @@ Full pytest documentation :hidden: changelog.txt - examples/resources.txt + resources + example/resources_attic diff --git a/doc/en/example/resources.txt b/doc/en/example/resources.txt deleted file mode 100644 index 3021a7bb3..000000000 --- a/doc/en/example/resources.txt +++ /dev/null @@ -1,369 +0,0 @@ - -V2: Creating and working with parametrized test resources -=============================================================== - -pytest-2.X provides generalized resource parametrization, unifying -and extending all existing funcarg and parametrization features of -previous pytest versions. Existing test suites and plugins written -for previous pytest versions shall run unmodified. - -This V2 draft focuses on incorporating feedback provided by Floris Bruynooghe, -Carl Meyer and Ronny Pfannschmidt. It remains as draft documentation, pending -further refinements and changes according to implementation or backward -compatibility issues. The main changes to V1 are: - -* changed API names (atnode -> scopenode) -* register_factory now happens at Node.collect_init() or pytest_collection_init - time. It will raise an Error if called during the runtestloop - (which performs setup/call/teardown for each collected test). -* new examples and notes related to @parametrize and metafunc.parametrize() -* use 2.X as the version for introduction - not sure if 2.3 or 2.4 will - actually bring it. -* examples/uses which were previously not possible to implement easily - are marked with "NEW" in the title. - -(NEW) the init_collection and init_runtestloop hooks ------------------------------------------------------- - -pytest for a long time offers a pytest_configure and a pytest_sessionstart -hook which are often used to setup global resources. This suffers from -several problems: - -1. in distributed testing the master process would setup test resources - that are never needed because it only co-ordinates the test run - activities of the slave processes. - -2. In large test suites resources are created which might not be needed - for the concrete test run. - -3. Thirdly, even if you only perform a collection (with "--collectonly") - resource-setup will be executed. - -4. there is no place way to allow global parametrized collection and setup - -The existing hooks are not a good place regarding these issues. pytest-2.X -solves all these issues through the introduction of two specific hooks -(and the new register_factory/getresource API):: - - def pytest_init_collection(session): - # called ahead of pytest_collection, which implements the - # collection process - - def pytest_init_runtestloop(session): - # called ahead of pytest_runtestloop() which executes the - # setup and calling of tests - -The pytest_init_collection hook can be used for registering resources, -see `global resource management`_ and `parametrizing global resources`_. - -The init_runtests can be used to setup and/or interact with global -resources. If you just use a global resource, you may explicitely -use it in a function argument or through a `class resource attribute`_. - -.. _`global resource management`: - -managing a global database resource ---------------------------------------------------------------- - -If you have one database object which you want to use in tests -you can write the following into a conftest.py file:: - - # contest of conftest.py - - class Database: - def __init__(self): - print ("database instance created") - def destroy(self): - print ("database instance destroyed") - - def factory_db(name, node): - db = Database() - node.addfinalizer(db.destroy) - return db - - def pytest_init_collection(session): - session.register_factory("db", factory_db) - -You can then access the constructed resource in a test by specifying -the pre-registered name in your function definition:: - - def test_something(db): - ... - -The "db" function argument will lead to a lookup and call of the respective -factory function and its result will be passed to the function body. -As the factory is registered on the session, it will by default only -get called once per session and its value will thus be re-used across -the whole test session. - -Previously, factories would need to call the ``request.cached_setup()`` -method to manage caching. Here is how we could implement the above -with traditional funcargs:: - - # content of conftest.py - class DataBase: - ... as above - - def pytest_funcarg__db(request): - return request.cached_setup(setup=DataBase, - teardown=lambda db: db.destroy, - scope="session") - -As the funcarg factory is automatically registered by detecting its -name and because it is called each time "db" is requested, it needs -to care for caching itself, here by calling the cached_setup() method -to manage it. As it encodes the caching scope in the factory code body, -py.test has no way to report this via e. g. "py.test --funcargs". -More seriously, it's not exactly trivial to provide parametrization: -we would need to add a "parametrize" decorator where the resource is -used or implement a pytest_generate_tests(metafunc) hook to -call metafunc.parametrize() with the "db" argument, and then the -factory would need to care to pass the appropriate "extrakey" into -cached_setup(). By contrast, the new way just requires a modified -call to register factories:: - - def pytest_init_collection(session): - session.register_factory("db", [factory_mysql, factory_pg]) - -and no other code needs to change or get decorated. - -(NEW) instantiating one database for each test module ---------------------------------------------------------------- - -If you want one database instance per test module you can restrict -caching by modifying the "scopenode" parameter of the registration -call above: - - def pytest_init_collection(session): - session.register_factory("db", factory_db, scopenode=pytest.Module) - -Neither the tests nor the factory function will need to change. -This means that you can decide the scoping of resources at runtime - -e.g. based on a command line option: for developer settings you might -want per-session and for Continous Integration runs you might prefer -per-module or even per-function scope like this:: - - def pytest_init_collection(session): - session.register_factory("db", factory_db, - scopenode=pytest.Function) - -Using a resource from another resource factory ----------------------------------------------- - -You can use the database resource from a another resource factory through -the ``node.getresource()`` method. Let's add a resource factory for -a "db_users" table at module-level, extending the previous db-example:: - - def pytest_init_collection(session): - ... - # this factory will be using a scopenode=pytest.Module because - # it is defined in a test module. - session.register_factory("db_users", createusers) - - def createusers(name, node): - db = node.getresource("db") - table = db.create_table("users", ...) - node.addfinalizer(lambda: db.destroy_table("users") - - def test_user_creation(db_users): - ... - -The create-users will be called for each module. After the tests in -that module finish execution, the table will be destroyed according -to registered finalizer. Note that calling getresource() for a resource -which has a tighter scope will raise a LookupError because the -is not available at a more general scope. Concretely, if you -table is defined as a per-session resource and the database object as a -per-module one, the table creation cannot work on a per-session basis. - -amending/decorating a resource / funcarg__ compatibility ----------------------------------------------------------------------- - -If you want to decorate a session-registered resource with -a test-module one, you can do the following:: - - # content of conftest.py - def pytest_init_collection(session): - session.register_factory("db_users", createusers) - -This will register a db_users method on a per-session basis. -If you want to create a dummy user such that all test -methods in a test module can work with it:: - - # content of test_user_admin.py - def setup_class(cls, db_users): - - def pytest_init_collection(session): - session.register_factory("db_users", createcreate_users, - scopenode=pytest.Module) - - def create_users(name, node): - # get the session-managed resource - db_users = node.getresource(name) - # add a user and define a remove_user undo function - ... - node.addfinalizer(remove_user) - return db_users - - def test_user_fields(db_users): - # work with db_users with a pre-created entry - ... - -Using the pytest_funcarg__ mechanism, you can do the equivalent:: - - # content of test_user_admin.py - - def pytest_funcarg__db_users(request): - def create_user(): - db_users = request.getfuncargvalue("db_users") - # add a user - return db_users - def remove_user(db_users): - ... - return request.cached_setup(create_user, remove_user, scope="module") - -As the funcarg mechanism is implemented in terms of the new API -it's also possible to mix - use register_factory/getresource at plugin-level -and pytest_funcarg__ factories at test module level. - -As discussed previously with `global resource management`_, the funcarg-factory -does not easily extend to provide parametrization. - - -.. _`class resource attributes`: - -(NEW) Setting resources as class attributes -------------------------------------------- - -If you want to make an attribute available on a test class, you can -use a new mark:: - - @pytest.mark.class_resource("db") - class TestClass: - def test_something(self): - #use self.db - -Note that this way of using resources work with unittest.TestCase-style -tests as well. If you have defined "db" as a parametrized resource, -the functions of the Test class will be run multiple times with different -values found in "self.db". - -Previously, pytest could not offer its resource management features -since those were tied to passing function arguments ("funcargs") and -this cannot be easily integrated with the unittest framework and its -common per-project customizations. - - -.. _`parametrizing global resources`: - -(NEW) parametrizing global resources ----------------------------------------------------- - -If you want to rerun tests with different resource values you can specify -a list of factories instead of just one:: - - def pytest_init_collection(session): - session.register_factory("db", [factory1, factory2]) - -In this case all tests that require the "db" resource will be run twice -using the respective values obtained from the two factory functions. - -For reporting purposes you might want to also define identifiers -for the db values:: - - def pytest_init_collection(session): - session.register_factory("db", [factory1, factory2], - ids=["mysql", "pg"]) - -This will make pytest use the respective id values when reporting -nodeids. - - -(New) Declaring resource usage / implicit parametrization ----------------------------------------------------------- - -Sometimes you may have a resource that can work in multiple variants, -like using different database backends. As another use-case, -pytest's own test suite uses a "testdir" funcarg which helps to setup -example scenarios, perform a subprocess-pytest run and check the output. -However, there are many features that should also work with the pytest-xdist -mode, distributing tests to multiple CPUs or hosts. The invocation -variants are not visible in the function signature and cannot be easily -addressed through a "parametrize" decorator or call. Nevertheless we want -to have both invocation variants to be collected and executed. - -The solution is to tell pytest that you are using a resource implicitely:: - - @pytest.mark.uses_resource("invocation-option") - class TestClass: - def test_method(self, testdir): - ... - -When the testdir factory gets the parametrized "invocation-option" -resource, it will see different values, depending on what the respective -factories provide. To register the invocation-mode factory you would write:: - - # content of conftest.py - def pytest_init_collection(session): - session.register_factory("invocation-option", - [lambda **kw: "", lambda **kw: "-n1"]) - -The testdir factory can then access it easily:: - - option = node.getresource("invocation-option", "") - ... - -.. note:: - - apart from the "uses_resource" decoration none of the already - written test functions needs to be modified for the new API. - - The implicit "testdir" parametrization only happens for the tests - which declare use of the invocation-option resource. All other - tests will get the default value passed as the second parameter - to node.getresource() above. You can thus restrict - running the variants to particular tests or test sets. - -To conclude, these three code fragments work together to allow efficient -cross-session resource parametrization. - - -Implementation and compatibility notes -============================================================ - -The new API is designed to support all existing resource parametrization -and funcarg usages. This chapter discusses implementation aspects. -Feel free to choose ignorance and only consider the above usage-level. - -Implementing the funcarg mechanism in terms of the new API -------------------------------------------------------------- - -Prior to pytest-2.X, pytest mainly advertised the "funcarg" mechanism -for resource management. It provides automatic registration of -factories through discovery of ``pytest_funcarg__NAME`` factory methods -on plugins, test modules, classes and functions. Those factories are be -called *each time* a resource (funcarg) is required, hence the support -for a ``request.cached_setup" method which helps to cache resources -across calls. Request objects internally keep a (item, requested_name, -remaining-factories) state. The "reamaining-factories" state is -used for implementing decorating factories; a factory for a given -name can call ``getfuncargvalue(name)`` to invoke the next-matching -factory factories and then amend the return value. - -In order to implement the existing funcarg mechanism through -the new API, the new API needs to internally keep around similar -state. XXX - -As an example let's consider the Module.setup_collect() method:: - - class Module(PyCollector): - def setup_collect(self): - for name, func in self.obj.__dict__.items(): - if name.startswith("pytest_funcarg__"): - resourcename = name[len("pytest_funcarg__"):] - self.register_factory(resourcename, - RequestAdapter(self, name, func)) - -The request adapater takes care to provide the pre-2.X API for funcarg -factories, i.e. request.cached_setup/addfinalizer/getfuncargvalue -methods and some attributes. diff --git a/doc/en/funcargs.txt b/doc/en/funcargs.txt index 3e82a4050..1dfa1a838 100644 --- a/doc/en/funcargs.txt +++ b/doc/en/funcargs.txt @@ -110,12 +110,13 @@ with a list of available function arguments. The request object passed to factories ----------------------------------------- -Each funcarg factory receives a :py:class:`~_pytest.main.Request` object which +Each funcarg factory receives a :py:class:`~_pytest.python.Request` object which provides methods to manage caching and finalization in the context of the test invocation as well as several attributes of the the underlying test item. In fact, as of version pytest-2.3, the request API is implemented on all Item objects and therefore the request object has general :py:class:`Node attributes and methods <_pytest.main.Node>` attributes. This is a backward compatible change so no changes are neccessary for pre-2.3 funcarg factories. + .. _`parametrizing tests, generalized`: http://tetamap.wordpress.com/2009/05/13/parametrizing-python-tests-generalized/ .. _`blog post about the monkeypatch funcarg`: http://tetamap.wordpress.com/2009/03/03/monkeypatching-in-unit-tests-done-right/ diff --git a/doc/en/plugins.txt b/doc/en/plugins.txt index 719404192..531221e47 100644 --- a/doc/en/plugins.txt +++ b/doc/en/plugins.txt @@ -296,6 +296,7 @@ into interactive debugging when a test failure occurs. The :py:mod:`_pytest.terminal` reported specifically uses the reporting hook to print information about a test run. + Collection hooks ------------------------------ @@ -309,6 +310,7 @@ For influencing the collection of objects in Python modules you can use the following hook: .. autofunction:: pytest_pycollect_makeitem +.. autofunction:: pytest_generate_tests Reporting hooks @@ -329,7 +331,7 @@ test execution: Reference of objects involved in hooks =========================================================== -.. autoclass:: _pytest.main.Request() +.. autoclass:: _pytest.python.Request() :members: .. autoclass:: _pytest.config.Config() diff --git a/doc/en/resources.txt b/doc/en/resources.txt new file mode 100644 index 000000000..cd0554997 --- /dev/null +++ b/doc/en/resources.txt @@ -0,0 +1,426 @@ + +V3: Creating and working with parametrized test resources +=============================================================== + +**Target audience**: Reading this document requires basic knowledge of +python testing, xUnit setup methods and the basic pytest funcarg mechanism, +see http://pytest.org/latest/funcargs.html + +**Abstract**: pytest-2.X provides more powerful and more flexible funcarg +and setup machinery. It does so by introducing a new @funcarg and a +new @setup marker which allows to define scoping and parametrization +parameters. If using ``@funcarg``, following the ``pytest_funcarg__`` +naming pattern becomes optional. Functions decorated with ``@setup`` +are called independenlty from the definition of funcargs but can +access funcarg values if needed. This allows for ultimate flexibility +in designing your test fixtures and their parametrization. Also, +you can now use ``py.test --collectonly`` to inspect your fixture +setup. Nonwithstanding these extensions, pre-existing test suites +and plugins written to work for previous pytest versions shall run unmodified. + + +**Changes**: This V3 draft is based on incorporating and thinking about +feedback provided by Floris Bruynooghe, Carl Meyer and Samuele Pedroni. +It remains as draft documentation, pending further refinements and +changes according to implementation or backward compatibility issues. +The main changes to V2 are: + +* Collapse funcarg factory decorator into a single "@funcarg" one. + You can specify scopes and params with it. Moreover, if you supply + a "name" you do not need to follow the "pytest_funcarg__NAME" naming + pattern. Keeping with "funcarg" naming arguable now makes more + sense since the main interface using these resources are test and + setup functions. Keeping it probably causes the least semantic friction. + +* Drop setup_directory/setup_session and introduce a new @setup + decorator similar to the @funcarg one but accepting funcargs. + +* cosnider the extended setup_X funcargs for dropping because + the new @setup decorator probably is more flexible and introduces + less implementation complexity. + +.. currentmodule:: _pytest + + +Shortcomings of the previous pytest_funcarg__ mechanism +--------------------------------------------------------- + +The previous funcarg mechanism calls a factory each time a +funcarg for a test function is requested. If a factory wants +t re-use a resource across different scopes, it often used +the ``request.cached_setup()`` helper to manage caching of +resources. Here is a basic example how we could implement +a per-session Database object:: + + # content of conftest.py + class Database: + def __init__(self): + print ("database instance created") + def destroy(self): + print ("database instance destroyed") + + def pytest_funcarg__db(request): + return request.cached_setup(setup=DataBase, + teardown=lambda db: db.destroy, + scope="session") + +There are some problems with this approach: + +1. Scoping resource creation is not straight forward, instead one must + understand the intricate cached_setup() method mechanics. + +2. parametrizing the "db" resource is not straight forward: + you need to apply a "parametrize" decorator or implement a + :py:func:`~hookspec.pytest_generate_tests` hook + calling :py:func:`~python.Metafunc.parametrize` which + performs parametrization at the places where the resource + is used. Moreover, you need to modify the factory to use an + ``extrakey`` parameter containing ``request.param`` to the + :py:func:`~python.Request.cached_setup` call. + +3. the current implementation is inefficient: it performs factory discovery + each time a "db" argument is required. This discovery wrongly happens at + setup-time. + +4. there is no way how you can use funcarg factories, let alone + parametrization, when your tests use the xUnit setup_X approach. + +5. there is no way to specify a per-directory scope for caching. + +In the following sections, API extensions are presented to solve +each of these problems. + + +Direct scoping of funcarg factories +-------------------------------------------------------- + +Instead of calling cached_setup(), you can decorate your factory +to state its scope:: + + @pytest.mark.funcarg(scope="session") + def pytest_funcarg__db(request): + # factory will only be invoked once per session - + db = DataBase() + request.addfinalizer(db.destroy) # destroy when session is finished + return db + +This factory implementation does not need to call ``cached_setup()`` anymore +because it will only be invoked once per session. Moreover, the +``request.addfinalizer()`` registers a finalizer according to the specified +resource scope on which the factory function is operating. With this new +scoping, the still existing ``cached_setup()`` should be much less used +but will remain for compatibility reasons and for the case where you +still want to have your factory get called on a per-item basis. + + +Direct parametrization of funcarg resource factories +---------------------------------------------------------- + +Previously, funcarg factories could not directly cause parametrization. +You needed to specify a ``@parametrize`` or implement a ``pytest_generate_tests`` hook to perform parametrization, i.e. calling a test multiple times +with different value sets. pytest-2.X introduces a decorator for use +on the factory itself:: + + @pytest.mark.funcarg(params=["mysql", "pg"]) + def pytest_funcarg__db(request): + ... + +Here the factory will be invoked twice (with the respective "mysql" +and "pg" values set as ``request.param`` attributes) and and all of +the tests requiring "db" will run twice as well. The "mysql" and +"pg" values will also be used for reporting the test-invocation variants. + +This new way of parametrizing funcarg factories should in many cases +allow to re-use already written factories because effectively +``request.param`` are already the parametrization attribute for test +functions/classes were parametrized via +:py:func:`~_pytest.python.Metafunc.parametrize(indirect=True)` calls. + +Of course it's perfectly fine to combine parametrization and scoping:: + + @pytest.mark.funcarg(scope="session", params=["mysql", "pg"]) + def pytest_funcarg__db(request): + if request.param == "mysql": + db = MySQL() + elif request.param == "pg": + db = PG() + request.addfinalizer(db.destroy) # destroy when session is finished + return db + +This would execute all tests requiring the per-session "db" resource twice, +receiving the values created by the two respective invocations to the +factory function. + +Direct usage of funcargs with funcargs factories +---------------------------------------------------------- + +You can now directly use funcargs in funcarg factories. Example:: + + @pytest.mark.funcarg(scope="session") + def db(request, tmpdir): + # tmpdir is a session-specific tempdir + +Apart from convenience it also solves an issue when your factory +depends on a parametrized funcarg. Previously, a call to +``request.getfuncargvalue()`` would not allow pytest to know +at collection time about the fact that a required resource is +actually parametrized. + +The "pytest_funcarg__" prefix becomes optional +----------------------------------------------------- + +When using the ``@funcarg`` decorator you do not need to use +the ``pytest_funcarg__`` prefix any more:: + + @pytest.mark.funcarg + def db(request): + ... + +The name under which the funcarg resource can be requested is ``db``. +Any ``pytest_funcarg__`` prefix will be stripped. Note that a an +unqualified funcarg-marker implies a scope of "function" meaning +that the funcarg factory will be called for each test function invocation. + + + +support for a new @setup marker +------------------------------------------------------ + +pytest for a long time offered a pytest_configure and a pytest_sessionstart +hook which are often used to setup global resources. This suffers from +several problems: + +1. in distributed testing the master process would setup test resources + that are never needed because it only co-ordinates the test run + activities of the slave processes. + +2. if you only perform a collection (with "--collectonly") + resource-setup will still be executed. + +3. If a pytest_sessionstart is contained in some subdirectories + conftest.py file, it will not be called. This stems from the + fact that this hook is actually used for reporting, in particular + the test-header with platform/custom information. + +4. there is no direct way how you can restrict setup to a directory scope. + +Moreover, it is today not easy to define scoped setup from plugins or +conftest files other than to implement a ``pytest_runtest_setup()`` hook +and caring for scoping/caching yourself. And it's virtually impossible +to do this with parametrization as ``pytest_runtest_setup()`` is called +during test execution and parametrization happens at collection time. + +It follows that pytest_configure/session/runtest_setup are often not +appropriate for implementing common fixture needs. Therefore, +pytest-2.X introduces a new "@pytest.mark.setup" marker, accepting +the same parameters as the @funcargs decorator. The difference is +that the decorated function can accept function arguments itself +Example:: + + # content of conftest.py + import pytest + @pytest.mark.setup(scope="session") + def mysetup(db): + ... + +This ``mysetup`` function is going to be executed when the first +test in the directory tree executes. It is going to be executed once +per-session and it receives the ``db`` funcarg which must be of same +of higher scope; you e. g. generally cannot use a per-module or per-function +scoped resource in a session-scoped setup function. + +You can also use ``@setup`` inside a test module or class:: + + # content of test_module.py + import pytest + + @pytest.mark.setup(scope="module", params=[1,2,3]) + def modes(tmpdir, request): + # ... + +This would execute the ``modes`` function once for each parameter. +In addition to normal funcargs you can also receive the "request" +funcarg which represents a takes on each of the values in the +``params=[1,2,3]`` decorator argument. + +.. note:: + + For each scope, the funcargs will be setup and then the setup functions + will be called. This allows @setup-decorated functions to depend + on already setup funcarg values by accessing ``request.funcargs``. + +Using funcarg resources in xUnit setup methods +------------------------------------------------------------ + +XXX Consider this feature in contrast to the @setup feature - probably +introducing one of them is better and the @setup decorator is more flexible. + +For a long time, pytest has recommended the usage of funcarg +factories as a primary means for managing resources in your test run. +It is a better approach than the jUnit-based approach in many cases, even +more with the new pytest-2.X features, because the funcarg resource factory +provides a single place to determine scoping and parametrization. Your tests +do not need to encode setup/teardown details in every test file's +setup_module/class/method. + +However, the jUnit methods originally introduced by pytest to Python, +remain popoular with nose and unittest-based test suites. Without question, +there are large existing test suites using this paradigm. pytest-2.X +recognizes this fact and now offers direct integration with funcarg resources. Here is a basic example for getting a per-module tmpdir:: + + def setup_module(mod, tmpdir): + mod.tmpdir = tmpdir + +This will trigger pytest's funcarg mechanism to create a value of +"tmpdir" which can then be used throughout the module as a global. + +The new extension to setup_X methods also works in case a resource is +parametrized. For example, let's consider an setup_class example using +our "db" resource:: + + class TestClass: + def setup_class(cls, db): + cls.db = db + # perform some extra things on db + # so that test methods can work with it + +With pytest-2.X the setup* methods will be discovered at collection-time, +allowing to seemlessly integrate this approach with parametrization, +allowing the factory specification to determine all details. The +setup_class itself does not itself need to be aware of the fact that +"db" might be a mysql/PG database. +Note that if the specified resource is provided only as a per-testfunction +resource, collection would early on report a ScopingMismatch error. + + +the "directory" caching scope +-------------------------------------------- + +All API accepting a scope (:py:func:`cached_setup()` and +the new funcarg/setup decorators) now also accept a "directory" +specification. This allows to restrict/cache resource values on a +per-directory level. + +funcarg and setup discovery now happens at collection time +--------------------------------------------------------------------- + +pytest-2.X takes care to discover funcarg factories and setup_X methods +at collection time. This is more efficient especially for large test suites. +Moreover, a call to "py.test --collectonly" should be able to show +a lot of setup-information and thus presents a nice method to get an +overview of resource management in your project. + +Implementation level +=================================================================== + +To implement the above new features, pytest-2.X grows some new hooks and +methods. At the time of writing V2 and without actually implementing +it, it is not clear how much of this new internal API will also be +exposed and advertised e. g. for plugin writers. + +The main effort, however, will lie in revising what is done at +collection and what at test setup time. All funcarg factories and +xUnit setup methods need to be discovered at collection time +for the above mechanism to work. Additionally all test function +signatures need to be parsed in order to know which resources are +used. On the plus side, all previously collected fixtures and +test functions only need to be called, no discovery is neccessary +is required anymore. + +the "request" object incorporates scope-specific behaviour +------------------------------------------------------------------ + +funcarg factories receive a request object to help with implementing +finalization and inspection of the requesting-context. If there is +no scoping is in effect, nothing much will change of the API behaviour. +However, with scoping the request object represents the according context. +Let's consider this example:: + + @pytest.mark.factory_scope("class") + def pytest_funcarg__db(request): + # ... + request.getfuncargvalue(...) + # + request.addfinalizer(db) + +Due to the class-scope, the request object will: + +- provide a ``None`` value for the ``request.function`` attribute. +- default to per-class finalization with the addfinalizer() call. +- raise a ScopeMismatchError if a more broadly scoped factory + wants to use a more tighly scoped factory (e.g. per-function) + +In fact, the request object is likely going to provide a "node" +attribute, denoting the current collection node on which it internally +operates. (Prior to pytest-2.3 there already was an internal +_pyfuncitem). + +As these are rather intuitive extensions, not much friction is expected +for test/plugin writers using the new scoping and parametrization mechanism. +It's, however, a serious internal effort to reorganize the pytest +implementation. + + +node.register_factory/getresource() methods +-------------------------------------------------------- + +In order to implement factory- and setup-method discovery at +collection time, a new node API will be introduced to allow +for factory registration and a getresource() call to obtain +created values. The exact details of this API remain subject +to experimentation. The basic idea is to introduce two new +methods to the Session class which is already available on all nodes +through the ``node.session`` attribute:: + + class Session: + def register_resource_factory(self, name, factory_or_list, scope): + """ register a resource factory for the given name. + + :param name: Name of the resource. + :factory_or_list: a function or a list of functions creating + one or multiple resource values. + :param scope: a node instance. The factory will be only visisble + available for all descendant nodes. + specify the "session" instance for global availability + """ + + def getresource(self, name, node): + """ get a named resource for the give node. + + This method looks up a matching funcarg resource factory + and calls it. + """ + +.. todo:: + + XXX While this new API (or some variant of it) may suffices to implement + all of the described new usage-level features, it remains unclear how the + existing "@parametrize" or "metafunc.parametrize()" calls will map to it. + These parametrize-approaches tie resource parametrization to the + function/funcargs-usage rather than to the factories. + + + +ISSUES +-------------------------- + +decorating a parametrized funcarg factory: + + @pytest.mark.funcarg(scope="session", params=["mysql", "pg"]) + def db(request): + ... + class TestClass: + @pytest.mark.funcarg(scope="function") + def something(self, request): + session_db = request.getfuncargvalue("db") + ... + +Here the function-scoped "something" factory uses the session-scoped +"db" factory to perform some additional steps. The dependency, however, +is only visible at setup-time, when the factory actually gets called. + +In order to allow parametrization at collection-time I see two ways: + +- allow specifying dependencies in the funcarg-marker +- allow funcargs for factories as well +