From 4a78daf7f3013399ffbf13d3760209b1bbb9b3ec Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 15 Jun 2009 17:28:55 +0200 Subject: [PATCH] * refine collect hooks and docs, remove pytest_collect_recurse * write and extend extension docs --HG-- branch : trunk --- CHANGELOG | 14 +++- doc/test/config.txt | 52 +++++++++++- doc/test/extend.txt | 134 ++++++++++++++++++++++++------- doc/test/funcargs.txt | 49 ++++++----- doc/test/test.txt | 4 +- py/test/collect.py | 15 ++-- py/test/plugin/hookspec.py | 20 ++--- py/test/plugin/pytest_default.py | 20 ++--- py/test/testing/test_collect.py | 11 ++- 9 files changed, 225 insertions(+), 94 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 7bd7086fc..7ccacbc79 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,18 @@ $Id: CHANGELOG 64077 2009-04-14 21:04:57Z hpk $ -Changes between 0.9.2 and 1.0 (UNRELEASED) +Changes between 1.0.0b1 and 1.0.0b2 +============================================= + +* plugin classes are removed: one now defines + hooks directly in conftest.py or global pytest_*.py + files. + +* documented and refined various hooks + +* added new style of generative tests via pytest_generate_tests + hook + +Changes between 0.9.2 and 1.0.0b1 ============================================= * introduced new "funcarg" setup method, diff --git a/doc/test/config.txt b/doc/test/config.txt index 6a85b5ebb..264549f18 100644 --- a/doc/test/config.txt +++ b/doc/test/config.txt @@ -1,13 +1,58 @@ Test configuration ======================== -test options and values +available test options ----------------------------- -You can see all available command line options by running:: +You can see command line options by running:: py.test -h +This will display all available command line options +including the ones added by plugins `loaded at tool startup`_. + +.. _`loaded at tool startup`: extend.html#tool-startup + +.. _conftestpy: +.. _collectignore: + +conftest.py: project specific test configuration +-------------------------------------------------------- + +A unique feature of py.test are its powerful ``conftest.py`` files which +allow to `set option defaults`_, `implement hooks`_, `specify funcargs`_ +or set particular variables to influence the testing process: + +* ``pytest_plugins``: list of named plugins to load + +* ``collect_ignore``: list of paths to ignore during test collection (relative to the containing + ``conftest.py`` file) + +* ``rsyncdirs``: list of to-be-rsynced directories for distributed + testing + +You may put a conftest.py files in your project root directory or into +your package directory if you want to add project-specific test options. + +``py.test`` loads all ``conftest.py`` files upwards from the command +line specified test files. It will lookup configuration values +right-to-left, i.e. the closer conftest files will be checked first. +You may have a ``conftest.py`` in your very home directory to have some +global configuration values. + +There is a flag that may help you debugging your conftest.py +configuration:: + + py.test --traceconfig + +.. _`implement hooks`: extend.html#conftest.py-plugin +.. _`specify funcargs`: funcargs.html#application-setup-tutorial-example + +.. _`set option defaults`: + +setting option defaults +------------------------------- + py.test will lookup values of options in this order: * option value supplied at command line @@ -16,12 +61,11 @@ py.test will lookup values of options in this order: The name of an option usually is the one you find in the longform of the option, i.e. the name -behind the ``--`` double-dash. +behind the ``--`` double-dash that you get with ``py.test -h``. IOW, you can set default values for options per project, per home-directoray, per shell session or per test-run. - .. _`basetemp`: per-testrun temporary directories diff --git a/doc/test/extend.txt b/doc/test/extend.txt index 830b7ae4b..7cc958f78 100644 --- a/doc/test/extend.txt +++ b/doc/test/extend.txt @@ -1,55 +1,103 @@ ================================================ -Extending and customizating py.test +Extending and customizing py.test ================================================ - .. _`local plugin`: py.test implements much of its functionality by calling `well specified -hooks`_. Hook functions are defined in local or global plugins. -By default local plugins are the ``conftest.py`` modules in your project. -Global plugins are python modules or packages with a ``pytest_`` prefixed name. +hooks`_. Python modules which contain such hook functions are called +plugins. Hook functions are discovered in ``conftest.py`` files or +in **named** plugins. ``conftest.py`` files are sometimes called "anonymous" +or "local" plugins if they define hooks. Named plugins are python modules +or packages that have an all lowercase ``pytest_`` prefixed name and who +are imported during tool startup or the testing process. +.. _`tool startup`: -Loading plugins and specifying dependencies -============================================ +Plugin discovery at tool startup +-------------------------------------------- -py.test loads plugin modules at tool startup in the following ways: +py.test loads plugin modules at tool startup in the following way: * by reading the ``PYTEST_PLUGINS`` environment variable - and importing the comma-separated list of plugin names. + and importing the comma-separated list of named plugins. * by pre-scanning the command line for the ``-p name`` option and loading the specified plugin before actual command line parsing. -* by loading all plugins specified by the ``pytest_plugins`` - variable in a ``conftest.py`` file or test modules. +* by loading all `conftest.py plugin`_ files as inferred by the command line + invocation + +* by recursively loading all plugins specified by the + ``pytest_plugins`` variable in a ``conftest.py`` file Note that at tool startup only ``conftest.py`` files in the directory of the specified test modules (or the current dir if None) or any of the parent directories are found. There is no try to pre-scan all subdirectories to find ``conftest.py`` files or test -modules. Each plugins may specify its dependencies via -``pytest_plugins`` definition recursively. +modules. + +Specifying plugins in a test module or plugin +----------------------------------------------- + +You can specify plugins in a test module or a plugin like this: + +.. sourcecode:: python + + pytest_plugins = "name1", "name2", + +When the test module or plugin is loaded the specified plugins +will be loaded. If you specify plugins without the ``pytest_`` +prefix it will be automatically added. All plugin names +must be lowercase. + +.. _`conftest.py plugin`: +.. _`conftestplugin`: + +conftest.py as anonymous per-project plugins +-------------------------------------------------- + +The purpose of ``conftest.py`` files is to allow `project-specific +test configuration`_. But they also make for a good place to implement +project-specific test related features through hooks. For example you may +set the `collect_ignore`_ variable depending on a command line option +by defining the following hook in a ``conftest.py`` file: + +.. _`exclude-file-example`: + +.. sourcecode:: python + + # ./conftest.py in your root or package dir + collect_ignore = ['hello', 'test_world.py'] + def pytest_addoption(parser): + parser.addoption("--runall", action="store_true", default=False) + def pytest_configure(config): + if config.getvalue("runall"): + collect_ignore[:] = [] + +.. _`project-specific test configuration`: config.html#conftestpy +.. _`collect_ignore`: config.html#collectignore .. _`well specified hooks`: Available py.test hooks ==================================== -A py.test hook is nothing more than a python function with -a ``pytest_`` prefixed name taking a number of arguments. -When loading a plugin module which contains hooks py.test performs -strict checking on all hook functions. Function and argument names need -to match exactly the `original definition of the hook`_. This allows -for early mismatch reporting and minimizes version incompatibilites. +py.test calls hooks functions to implement its `test collection`_, running and +reporting process. Upon loading of a plugin py.test performs +strict checking on contained hook functions. Function and argument names +need to match exactly the `original definition of the hook`_. It thus +provides useful error reporting on mistyped hook or argument names +and minimizes version incompatibilites. .. _`original definition of the hook`: http://bitbucket.org/hpk42/py-trunk/src/tip/py/test/plugin/hookspec.py generic "runtest" hooks ------------------------------ -Each test item is usually executed by calling the following three hooks:: +Each test item is usually executed by calling the following three hooks: + +.. sourcecode:: python pytest_runtest_setup(item) pytest_runtest_call(item) @@ -57,7 +105,9 @@ Each test item is usually executed by calling the following three hooks:: For each of the three invocations a `call object`_ encapsulates information about the outcome of the call and is subsequently used -to make a report object:: +to make a report object: + +.. sourcecode:: python report = hook.pytest_runtest_makereport(item, call) @@ -69,11 +119,15 @@ Usually three reports will be generated for a single test item. However, if the ``pytest_runtest_setup`` fails no call or teardown hooks will be called and only one report will be created. -Each of the up to three reports is eventually fed to the logreport hook:: +Each of the up to three reports is eventually fed to the logreport hook: + +.. sourcecode:: python pytest_runtest_logreport(report) -A ``report`` object contains status and reporting information:: +A ``report`` object contains status and reporting information: + +.. sourcecode:: python report.longrepr = string/lines/object to print report.when = "setup", "call" or "teardown" @@ -85,13 +139,17 @@ A ``report`` object contains status and reporting information:: The `pytest_terminal plugin`_ uses this hook to print information about a test run. -The protocol described here is implemented via this hook:: +The protocol described here is implemented via this hook: + +.. sourcecode:: python pytest_runtest_protocol(item) -> True .. _`call object`: -The call object contains information about a performed call:: +The call object contains information about a performed call: + +.. sourcecode:: python call.excinfo = ExceptionInfo object or None call.when = "setup", "call" or "teardown" @@ -104,19 +162,38 @@ The call object contains information about a performed call:: generic collection hooks ------------------------------ -XXX +py.test calls the following two fundamental hooks for collecting files and directories: -Python module and test function hooks -------------------------------------------- +.. sourcecode:: python + + def pytest_collect_directory(path, parent): + """ return Collection node or None for the given path. """ + + def pytest_collect_file(path, parent): + """ return Collection node or None for the given path. """ + +Both return a `collection node`_ for a given path. All returned +nodes from all hook implementations will participate in the +collection and running protocol. The ``parent`` object is +the parent node and may be used to access command line +options via the ``parent.config`` object. + + +Python specific test function and module hooks +---------------------------------------------------- For influencing the collection of objects in Python modules you can use the following hook: +.. sourcecode:: python + pytest_pycollect_makeitem(collector, name, obj) This hook will be called for each Python object in a collected Python module. The return value is a custom `collection node`_. + + .. XXX or ``False`` if you want to indicate that the given item should not be collected. @@ -135,6 +212,7 @@ Additionally you can check out some more contributed plugins here .. _`collection process`: .. _`collection node`: +.. _`test collection`: Test Collection process diff --git a/doc/test/funcargs.txt b/doc/test/funcargs.txt index 949375afa..b4b915eb4 100644 --- a/doc/test/funcargs.txt +++ b/doc/test/funcargs.txt @@ -2,14 +2,15 @@ **funcargs**: test setup and parametrization ====================================================== -Since version 1.0 py.test automatically discovers and -manages test function arguments. The mechanism -naturally connects to the automatic discovery of -test files, classes and functions. Automatic test discovery -values the `Convention over Configuration`_ concept. -By discovering and calling functions ("funcarg providers") that -provide values for your actual test functions -it becomes easy to: +Since version 1.0 py.test introduces test function arguments, +in short "funcargs" for your Python test functions. The basic idea +that your unit-, functional- or acceptance test functions can name +arguments and py.test will discover a matching provider from your +test configuration. The mechanism complements the automatic +discovery of test files, classes and functions which follows +the `Convention over Configuration`_ strategy. By discovering and +calling functions ("funcarg providers") that provide values for your +actual test functions it becomes easy to: * separate test function code from test state setup/fixtures * manage test value setup and teardown depending on @@ -339,9 +340,8 @@ following code into a local ``conftest.py``: from myapp import MyApp - class ConftestPlugin: - def pytest_funcarg__mysetup(self, request): - return MySetup() + def pytest_funcarg__mysetup(request): + return MySetup() class MySetup: def myapp(self): @@ -406,13 +406,12 @@ and to offer a new mysetup method: import py from myapp import MyApp - class ConftestPlugin: - def pytest_funcarg__mysetup(self, request): - return MySetup(request) + def pytest_funcarg__mysetup(request): + return MySetup(request) - def pytest_addoption(self, parser): - parser.addoption("--ssh", action="store", default=None, - help="specify ssh host to run tests with") + def pytest_addoption(parser): + parser.addoption("--ssh", action="store", default=None, + help="specify ssh host to run tests with") class MySetup: @@ -465,14 +464,13 @@ example: specifying and selecting acceptance tests .. sourcecode:: python # ./conftest.py - class ConftestPlugin: - def pytest_option(self, parser): - group = parser.getgroup("myproject") - group.addoption("-A", dest="acceptance", action="store_true", - help="run (slow) acceptance tests") + def pytest_option(parser): + group = parser.getgroup("myproject") + group.addoption("-A", dest="acceptance", action="store_true", + help="run (slow) acceptance tests") - def pytest_funcarg__accept(self, request): - return AcceptFuncarg(request) + def pytest_funcarg__accept(request): + return AcceptFuncarg(request) class AcceptFuncarg: def __init__(self, request): @@ -527,13 +525,14 @@ Our module level provider will be invoked first and it can ask its request object to call the next provider and then decorate its result. This mechanism allows us to stay ignorant of how/where the function argument is provided - -in our example from a ConftestPlugin but could be any plugin. +in our example from a `conftest plugin`_. sidenote: the temporary directory used here are instances of the `py.path.local`_ class which provides many of the os.path methods in a convenient way. .. _`py.path.local`: ../path.html#local +.. _`conftest plugin`: extend.html#conftestplugin Questions and Answers diff --git a/doc/test/test.txt b/doc/test/test.txt index e9e9ecdd2..78183d200 100644 --- a/doc/test/test.txt +++ b/doc/test/test.txt @@ -15,13 +15,15 @@ funcargs_: powerful parametrized test function setup `distributed testing`_: distribute test runs to other machines and platforms. -extend_: easily write per-project hooks or global plugins +extend_: intro to extend and customize py.test runs +config_: ``conftest.py`` files and general configuration .. _quickstart: quickstart.html .. _features: features.html .. _funcargs: funcargs.html .. _extend: extend.html +.. _config: config.html .. _`distributed testing`: dist.html diff --git a/py/test/collect.py b/py/test/collect.py index a8e35a8f4..33611274d 100644 --- a/py/test/collect.py +++ b/py/test/collect.py @@ -384,7 +384,13 @@ class Directory(FSCollector): l.append(res) return l + def _ignore(self, path): + ignore_paths = self.config.getconftest_pathlist("collect_ignore", path=path) + return ignore_paths and path in ignore_paths + def consider(self, path): + if self._ignore(path): + return if path.check(file=1): res = self.consider_file(path) elif path.check(dir=1): @@ -392,10 +398,11 @@ class Directory(FSCollector): else: res = None if isinstance(res, list): - # throw out identical modules + # throw out identical results l = [] for x in res: if x not in l: + assert x.parent == self, "wrong collection tree construction" l.append(x) res = l return res @@ -406,10 +413,8 @@ class Directory(FSCollector): def consider_dir(self, path, usefilters=None): if usefilters is not None: py.log._apiwarn("0.99", "usefilters argument not needed") - res = self.config.hook.pytest_collect_recurse(path=path, parent=self) - if res is None or res: - return self.config.hook.pytest_collect_directory( - path=path, parent=self) + return self.config.hook.pytest_collect_directory( + path=path, parent=self) class Item(Node): """ a basic test item. """ diff --git a/py/test/plugin/hookspec.py b/py/test/plugin/hookspec.py index 5b398dcda..04b0849fb 100644 --- a/py/test/plugin/hookspec.py +++ b/py/test/plugin/hookspec.py @@ -15,8 +15,7 @@ def pytest_configure(config): """ def pytest_unconfigure(config): - """ called before test process is exited. - """ + """ called before test process is exited. """ # ------------------------------------------------------------------------------ # test Session related hooks @@ -29,24 +28,17 @@ def pytest_sessionfinish(session, exitstatus, excrepr=None): """ whole test run finishes. """ def pytest_deselected(items): - """ collected items that were deselected (by keyword). """ + """ repeatedly called for test items deselected by keyword. """ # ------------------------------------------------------------------------------ # collection hooks # ------------------------------------------------------------------------------ +def pytest_collect_directory(path, parent): + """ return Collection node or None for the given path. """ def pytest_collect_file(path, parent): - """ return Collection node or None. """ - -def pytest_collect_recurse(path, parent): - """ return True/False to cause/prevent recursion into given directory. - return None if you do not want to make the decision. - """ -pytest_collect_recurse.firstresult = True - -def pytest_collect_directory(path, parent): - """ return Collection node or None. """ + """ return Collection node or None for the given path. """ def pytest_collectstart(collector): """ collector starts collecting. """ @@ -71,7 +63,7 @@ def pytest_pycollect_makeitem(collector, name, obj): pytest_pycollect_makeitem.firstresult = True def pytest_pyfunc_call(pyfuncitem): - """ perform function call with the given function arguments. """ + """ perform function call to the with the given function arguments. """ pytest_pyfunc_call.firstresult = True def pytest_generate_tests(metafunc): diff --git a/py/test/plugin/pytest_default.py b/py/test/plugin/pytest_default.py index d3b872c0a..d13c7077a 100644 --- a/py/test/plugin/pytest_default.py +++ b/py/test/plugin/pytest_default.py @@ -19,24 +19,18 @@ def pytest_collect_file(path, parent): if ext == ".py": return parent.Module(path, parent=parent) -def pytest_collect_recurse(path, parent): - #excludelist = parent._config.getvalue_pathlist('dir_exclude', path) - #if excludelist and path in excludelist: - # return - if not parent.recfilter(path): - # check if cmdline specified this dir or a subdir directly - for arg in parent.config.args: - if path == arg or arg.relto(path): - break - else: - return False - return True - def pytest_collect_directory(path, parent): # XXX reconsider the following comment # not use parent.Directory here as we generally # want dir/conftest.py to be able to # define Directory(dir) already + if not parent.recfilter(path): # by default special ".cvs", ... + # check if cmdline specified this dir or a subdir directly + for arg in parent.config.args: + if path == arg or arg.relto(path): + break + else: + return Directory = parent.config.getvalue('Directory', path) return Directory(path, parent=parent) diff --git a/py/test/testing/test_collect.py b/py/test/testing/test_collect.py index d7c1b0e65..a55e7ab61 100644 --- a/py/test/testing/test_collect.py +++ b/py/test/testing/test_collect.py @@ -205,17 +205,22 @@ class TestCustomConftests: assert item.name == "hello.xxx" assert item.__class__.__name__ == "CustomItem" - def test_avoid_directory_on_option(self, testdir): + def test_collectignore_exclude_on_option(self, testdir): testdir.makeconftest(""" + collect_ignore = ['hello', 'test_world.py'] def pytest_addoption(parser): parser.addoption("--XX", action="store_true", default=False) - def pytest_collect_recurse(path, parent): - return parent.config.getvalue("XX") + def pytest_configure(config): + if config.getvalue("XX"): + collect_ignore[:] = [] """) testdir.mkdir("hello") + testdir.makepyfile(test_world="#") reprec = testdir.inline_run(testdir.tmpdir) names = [rep.collector.name for rep in reprec.getreports("pytest_collectreport")] assert 'hello' not in names + assert 'test_world.py' not in names reprec = testdir.inline_run(testdir.tmpdir, "--XX") names = [rep.collector.name for rep in reprec.getreports("pytest_collectreport")] assert 'hello' in names + assert 'test_world.py' in names