From 36189a7aa7bcd61e1caebf557b3087f17c2257e5 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Tue, 18 Aug 2009 19:04:57 +0200 Subject: [PATCH] [mq]: 101doc --HG-- branch : 1.0.x --- CHANGELOG | 2 + MANIFEST | 10 +- doc/bin.txt | 3 +- doc/confrest.py | 33 +- doc/conftest.py | 2 + doc/contact.txt | 6 + doc/download.txt | 110 ++--- doc/faq.txt | 123 ++++++ doc/index.txt | 2 +- doc/test/attic.txt | 43 ++ doc/test/config.html | 18 + doc/test/config.txt | 96 ----- doc/test/{extend.txt => customize.txt} | 145 +++++-- doc/test/extend.html | 18 + doc/test/features.txt | 309 ++++++-------- doc/test/funcargs.txt | 539 ++++++++++++------------- doc/test/index.txt | 29 ++ doc/test/mission.txt | 13 + doc/test/plugin/capture.txt | 4 +- doc/test/plugin/doctest.txt | 4 +- doc/test/plugin/figleaf.txt | 4 +- doc/test/plugin/hooklog.txt | 4 +- doc/test/plugin/hookspec.txt | 6 +- doc/test/plugin/index.txt | 6 +- doc/test/plugin/keyword.txt | 4 +- doc/test/plugin/links.txt | 38 +- doc/test/plugin/monkeypatch.txt | 17 +- doc/test/plugin/nose.txt | 4 +- doc/test/plugin/pastebin.txt | 6 +- doc/test/plugin/pdb.txt | 4 +- doc/test/plugin/recwarn.txt | 4 +- doc/test/plugin/restdoc.txt | 4 +- doc/test/plugin/resultlog.txt | 4 +- doc/test/plugin/terminal.txt | 4 +- doc/test/plugin/unittest.txt | 4 +- doc/test/plugin/xfail.txt | 4 +- doc/test/quickstart.txt | 54 ++- doc/test/talks.txt | 16 +- doc/test/test.html | 18 + doc/test/test.txt | 38 -- doc/test/xunit_setup.txt | 8 +- doc/xml.txt | 2 +- example/assertion/failure_demo.py | 12 +- makepluginlist.py | 14 +- py/__init__.py | 2 +- py/test/plugin/pytest_pastebin.py | 2 +- py/test/plugin/pytest_terminal.py | 4 +- setup.py | 2 +- 48 files changed, 992 insertions(+), 806 deletions(-) create mode 100644 doc/faq.txt create mode 100644 doc/test/config.html delete mode 100644 doc/test/config.txt rename doc/test/{extend.txt => customize.txt} (72%) create mode 100644 doc/test/extend.html create mode 100644 doc/test/index.txt create mode 100644 doc/test/mission.txt create mode 100644 doc/test/test.html delete mode 100644 doc/test/test.txt diff --git a/CHANGELOG b/CHANGELOG index 46506e19c..0c4794fd1 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -10,6 +10,8 @@ Changes between 1.0.0 and 1.0.1 they are encoded as "utf8" by default, also terminalwriting was adapted and somewhat unified between windows and linux +* improved documentation layout and content a lot + * fix issue #27: better reporting on non-collectable items given on commandline (e.g. pyc files) diff --git a/MANIFEST b/MANIFEST index ae13ef50a..8cf609ae9 100644 --- a/MANIFEST +++ b/MANIFEST @@ -14,6 +14,7 @@ doc/conftest.py doc/contact.txt doc/download.txt doc/execnet.txt +doc/faq.txt doc/img/pylib.png doc/index.txt doc/io.txt @@ -22,12 +23,15 @@ doc/misc.txt doc/path.txt doc/style.css doc/test/attic.txt -doc/test/config.txt +doc/test/config.html +doc/test/customize.txt doc/test/dist.txt doc/test/examples.txt -doc/test/extend.txt +doc/test/extend.html doc/test/features.txt doc/test/funcargs.txt +doc/test/index.txt +doc/test/mission.txt doc/test/plugin/capture.txt doc/test/plugin/doctest.txt doc/test/plugin/figleaf.txt @@ -49,7 +53,7 @@ doc/test/plugin/unittest.txt doc/test/plugin/xfail.txt doc/test/quickstart.txt doc/test/talks.txt -doc/test/test.txt +doc/test/test.html doc/test/xunit_setup.txt doc/xml.txt example/assertion/failure_demo.py diff --git a/doc/bin.txt b/doc/bin.txt index e62766ef9..2d12c47eb 100644 --- a/doc/bin.txt +++ b/doc/bin.txt @@ -15,7 +15,7 @@ available on your command prompt. The ``py.test`` executable is the main entry point into the py-lib testing tool, see the `py.test documentation`_. -.. _`py.test documentation`: test/test.html +.. _`py.test documentation`: test/index.html ``py.cleanup`` ============== @@ -51,6 +51,7 @@ prepended. Usage: ``py.rest [PATHS] [options]`` +[deprecated in 1.0, will likely be separated] Loot recursively for .txt files starting from ``PATHS`` and convert them to html using docutils or to pdf files, if the ``--pdf`` option is used. For conversion to PDF you will need several command line tools, on Ubuntu Linux diff --git a/doc/confrest.py b/doc/confrest.py index c47621a74..66b3c32cb 100644 --- a/doc/confrest.py +++ b/doc/confrest.py @@ -45,9 +45,9 @@ pageTracker._trackPageview(); def a_docref(self, name, relhtmlpath): docpath = self.project.docpath - return html.a(name, class_="menu", + return html.div(html.a(name, class_="menu", href=relpath(self.targetpath.strpath, - docpath.join(relhtmlpath).strpath)) + docpath.join(relhtmlpath).strpath))) def a_apigenref(self, name, relhtmlpath): apipath = self.project.apigenpath @@ -57,19 +57,30 @@ pageTracker._trackPageview(); def fill_menubar(self): items = [ - self.a_docref("pylib index", "index.html"), - self.a_docref("test doc-index", "test/test.html"), - self.a_docref("test quickstart", "test/quickstart.html"), - self.a_docref("test features", "test/features.html"), - self.a_docref("test plugins", "test/plugin/index.html"), - self.a_docref("py.execnet", "execnet.html"), + self.a_docref("install", "download.html"), + self.a_docref("contact", "contact.html"), + self.a_docref("faq", "faq.html"), + html.div( + html.h3("py.test:"), + self.a_docref("doc index", "test/index.html"), + self.a_docref("features", "test/features.html"), + self.a_docref("quickstart", "test/quickstart.html"), + self.a_docref("tutorials", "test/talks.html"), + self.a_docref("plugins", "test/plugin/index.html"), + self.a_docref("funcargs", "test/funcargs.html"), + self.a_docref("customize", "test/customize.html"), + ), + html.div( + html.h3("supporting APIs:"), + self.a_docref("pylib index", "index.html"), + self.a_docref("py.execnet", "execnet.html"), + self.a_docref("py.path", "path.html"), + self.a_docref("py.code", "code.html"), + ) #self.a_docref("py.code", "code.html"), #self.a_apigenref("api", "api/index.html"), #self.a_apigenref("source", "source/index.html"), #self.a_href("source", "http://bitbucket.org/hpk42/py-trunk/src/"), - self.a_href("issues", "http://bitbucket.org/hpk42/py-trunk/issues/"), - self.a_docref("contact", "contact.html"), - self.a_docref("install", "download.html"), ] self.menubar = html.div(id=css.menubar, *[ html.div(item) for item in items]) diff --git a/doc/conftest.py b/doc/conftest.py index d5a37dd5d..50b4f51c1 100644 --- a/doc/conftest.py +++ b/doc/conftest.py @@ -3,3 +3,5 @@ import py #py.test.importorskip("pygments") pytest_plugins = ['pytest_restdoc'] rsyncdirs = ['.'] + +collect_ignore = ['test/attic.txt'] diff --git a/doc/contact.txt b/doc/contact.txt index 64f142751..6e9d8d551 100644 --- a/doc/contact.txt +++ b/doc/contact.txt @@ -5,10 +5,15 @@ Contact and Communication points - #pylib on irc.freenode.net IRC channel for random questions. + - `tetamap`_: Holger Krekel's blog, often about testing and py.test related news. +- `Testing In Python`_: a mailing list for testing tools and discussion. + - `py-svn general commit mailing list`_ to follow development commits, +- http://twitter.com/pylibcommit for following commits via twitter + - `bitbucket issue tracker`_ use this bitbucket issue tracker to report bugs or request features. @@ -33,6 +38,7 @@ Contact and Communication points you are new to the python developer community please come to the IRC or the mailing list and ask questions, get involved. +.. _`Testing in Python`: http://lists.idyll.org/listinfo/testing-in-python .. _FOAF: http://en.wikipedia.org/wiki/FOAF .. _us: http://codespeak.net/mailman/listinfo/py-dev .. _codespeak: http://codespeak.net/ diff --git a/doc/download.txt b/doc/download.txt index 91283191d..4b606354b 100644 --- a/doc/download.txt +++ b/doc/download.txt @@ -7,7 +7,7 @@ Latest Release, see `PyPI project page`_ -using setuptools / easy_install +using easy_install =================================================== With a working `setuptools installation`_ you can type:: @@ -16,16 +16,11 @@ With a working `setuptools installation`_ you can type:: to get the latest release of the py lib. The ``-U`` switch will trigger an upgrade if you already have an older version installed. +On Linux systems you may need to execute the command as superuser and +on Windows you might need to write down the full path to ``easy_install``. The py lib and its tools are expected to work well on Linux, Windows and OSX, Python versions 2.3, 2.4, 2.5 and 2.6. -We provide binary eggs for Windows machines. - -On other systems you need a working C-compiler in order to -install the full py lib. If you don't have a compiler available -you can still install the py lib but without greenlets - look -below for the ``install_lib`` target. - **IMPORTANT NOTE**: if you are using Windows and have 0.8 versions of the py lib on your system, please download and execute http://codespeak.net/svn/py/build/winpathclean.py @@ -36,9 +31,11 @@ You can find out the py lib version with:: print py.version -.. _`checkout`: +.. _mercurial: http://mercurial.selenic.com/wiki/ +.. _checkout: +.. _tarball: -Installing from version control / develop mode +Working from version control or a tarball ================================================= To follow development or help with fixing things @@ -47,41 +44,59 @@ and documentation source with mercurial_:: hg clone https://bitbucket.org/hpk42/py-trunk/ -With a working `setuptools installation`_ you can then issue:: +This currrently contains a 1.0.x branch and the +default 'trunk' branch where mainline development +takes place. There also is a readonly subversion +checkout available:: + + svn co https://codespeak.net/svn/py/dist + +You can also go to the python package index and +download and unpack a TAR file:: + + http://pypi.python.org/pypi/py/ + +activating checkout with setuptools +-------------------------------------------- + +With a working `setuptools installation`_ you can issue:: python setup.py develop -in order to work with your checkout version. - -For enhancing one of the plugins you may go to -the ``py/test/plugin/`` sub directory. - -.. _mercurial: http://mercurial.selenic.com/wiki/ +in order to work with the tools and the lib of your checkout. .. _`no-setuptools`: -Working without setuptools / from source -========================================== +activating a checkout or tarball without setuptools +------------------------------------------------------------- -If you have a checkout_ or a tarball_ it is actually not neccessary to issue -``setup.py`` commands in order to use py lib and its tools. You can -simply add the root directory to ``PYTHONPATH`` and ``py/bin`` or -``py\bin\win32`` to your ``PATH`` settings. +To import the py lib the ``py`` package directory needs to +be on the ``$PYTHONPATH``. If you exexute scripts directly +from ``py/bin/`` or ``py\bin\win32`` they will find their +containing py lib automatically. -There are also helper scripts to set the environment -on windows:: +It is usually a good idea to add the parent directory of the ``py`` package +directory to your ``PYTHONPATH`` and ``py/bin`` or ``py\bin\win32`` to your +system wide ``PATH`` settings. There are helper scripts that set ``PYTHONPATH`` and ``PATH`` on your system: +on windows execute:: + + # inside autoexec.bat or shell startup c:\\path\to\checkout\py\env.cmd -and on linux/osx you can add something like this to -your shell initialization:: +on linux/OSX add this to your shell initialization:: - eval `python ~/path/to/checkout/py/env.py` + # inside .bashrc + eval `python ~/path/to/checkout/py/env.py` both of which which will get you good settings for ``PYTHONPATH`` and ``PATH``. -Note also that the command line scripts will look + +note: scripts look for "nearby" py-lib +----------------------------------------------------- + +Note that the `command line scripts`_ will look for "nearby" py libs, so if you have a layout like this:: mypkg/ @@ -90,39 +105,26 @@ for "nearby" py libs, so if you have a layout like this:: tests/ py/ -then issuing ``py.test subpkg1`` will use the py lib -from that projects root directory. +issuing ``py.test subpkg1`` will use the py lib +from that projects root directory. + +.. _`command line scripts`: bin.html Debian and RPM packages =================================== -As of July 2009 pytest/pylib 1.0 RPMs and Debian packages -are not yet available. So you will only find older -versions. +As of August 2009 pytest/pylib 1.0 RPMs and Debian packages +are not available. You will only find 0.9 versions - +on Debian systems look for ``python-codespeak-lib`` +and Dwayne Bailey has put together a Fedora `RPM`_. -On Debian systems look for ``python-codespeak-lib``. -*But this package is probably outdated - if somebody -can help with bringing this up to date, -that would be very much appreciated.* +If you can help with providing/upgrading distribution +packages please use of the contact_ channels in case +of questions or need for changes. -Dwayne Bailey has thankfully put together a Fedora `RPM`_. +.. _contact: contact.html .. _`RPM`: http://translate.sourceforge.net/releases/testing/fedora/pylib-0.9.2-1.fc9.noarch.rpm .. _`setuptools installation`: http://pypi.python.org/pypi/setuptools -.. _tarball: - -Installing from a TAR archive -=================================================== - -You need a working `setuptools installation`_. - -Go to the python package index (pypi) and download a tar file: - - http://pypi.python.org/pypi/py/ - -and unpack it to a directory, where you then type:: - - python setup.py install - diff --git a/doc/faq.txt b/doc/faq.txt new file mode 100644 index 000000000..bb324d67f --- /dev/null +++ b/doc/faq.txt @@ -0,0 +1,123 @@ +================================== +Frequently Asked Questions +================================== + +.. contents:: + :local: + :depth: 2 + +naming, nose and magic +============================ + +Why the ``py`` naming? what is it? +------------------------------------ + +Because the name was kind of available and there was the +idea to have the package evolve into a "standard" library +kind of thing that works cross-python versions and is +not tied to a particular CPython revision or its release +cycle. Clearly, this was ambitious and the naming +has maybe haunted the project rather than helping it. +There may be a project name change and possibly a +split up into different projects sometime. + +Why the ``py.test`` naming? +------------------------------------ + +the py lib contains other command line tools that +all share the ``py.`` prefix which makes it easy +to use TAB-completion on the shell. Another motivation +was to make it obvious where testing functionality +for the ``py.test`` command line tool is: in the +``py.test`` package name space. + +What's the relation to ``nosetests``? +---------------------------------------- + +py.test and nose_ share some basic philosophy when it comes +to running Python tests. In fact, +with py.test-1.0.1 it is easy to run many test suites +that currently work with ``nosetests``. nose_ was created +as a clone of ``py.test`` when it was in the ``0.8`` release +cycle so some of the newer features_ introduced with py.test-1.0 +have no counterpart in nose_. + +.. _nose: http://somethingaboutorange.com/mrl/projects/nose/0.11.1/ +.. _features: test/features.html + +What's all this "magic" with py.test? +---------------------------------------- + +"All this magic" usually boils down to two issues: + +* There is a special tweak to importing: `py/__init__.py`_ contains + contains a dictionary which maps the ``py.*`` namespaces + to objects in files. When looking at the project source code + you see imports like ``from py.__.test.session import Session``. The + the double ``__`` underscore indicates the "normal" python + filesystem/namespace coupled import, i.e. it points to + ``py/test/session.py``'s ``Session`` object. However, + normal usage happens through the clean exported `py namespaces`_ + so this distinction usually only shows up if you hack + on internal code or see internal tracebacks. + +* when an ``assert`` fails, py.test re-interprets the expression + to show intermediate values. This allows to use the plain ``assert`` + statement instead of the many methods that you otherwise need + to mimick this behaviour. This means that in case of a failing + assert, your expressions gets evaluated *twice*. If your expression + has side effects the outcome may be different. If the test suddenly + passes you will get a detailed message. It is good practise, anyway, + to not have asserts with side effects. ``py.test --nomagic`` turns + off assert re-intepretation. + +Other than that, ``py.test`` has bugs or quirks like any other computer +software although it has a *strong* focus on running robustly and has +over a thousand automated tests for its own code base. + +.. _`py namespaces`: index.html +.. _`py/__init__.py`: http://bitbucket.org/hpk42/py-trunk/src/1.0.x/py/__init__.py + + +function arguments and parametrized tests +=============================================== + +.. _`why pytest_pyfuncarg__ methods?`: + +Why the ``pytest_funcarg__*`` name for funcarg factories? +--------------------------------------------------------------- + +When experimenting with funcargs an explicit registration mechanism +was considered. But lacking a good use case for this indirection and +flexibility we decided to go for `Convention over Configuration`_ and +allow to directly specify the factory. Besides removing the need +for an indirection it allows to "grep" for ``pytest_funcarg__MYARG`` +and will safely find all factory functions for the ``MYARG`` function +argument. It helps to alleviates the de-coupling of function +argument usage and creation. + +.. _`Convention over Configuration`: http://en.wikipedia.org/wiki/Convention_over_Configuration + +Can i yield multiple values from a factory function? +----------------------------------------------------- + +There are two reasons why yielding from a factory function +is not possible: + +* Calling factories for obtaining test function arguments + is part of setting up and running a test. At that + point it is not possible to add new test calls to + the test collection anymore. + +* If multiple factories yielded values there would + be no natural place to determine the combination + policy - in real-world examples some combinations + often should not run. + +Use the `pytest_generate_tests`_ hook to solve both issues +and implement the `parametrization scheme of your choice`_. + +.. _`pytest_generate_tests`: test/funcargs.html#parametrizing-tests +.. _`parametrization scheme of your choice`: http://tetamap.wordpress.com/2009/05/13/parametrizing-python-tests-generalized/ + + diff --git a/doc/index.txt b/doc/index.txt index 2c5e576b3..77b607f46 100644 --- a/doc/index.txt +++ b/doc/index.txt @@ -38,7 +38,7 @@ For the latest Release, see `PyPI project page`_ .. _`py.io`: io.html .. _`py.path`: path.html .. _`py.code`: code.html -.. _`py.test`: test/test.html +.. _`py.test`: test/index.html .. _`py lib scripts`: bin.html .. _`py.xml`: xml.html .. _`miscellaneous features`: misc.html diff --git a/doc/test/attic.txt b/doc/test/attic.txt index 2e8c98607..25e3c5254 100644 --- a/doc/test/attic.txt +++ b/doc/test/attic.txt @@ -72,3 +72,46 @@ Customizing execution of Items and Functions .. _`py-dev mailing list`: http://codespeak.net/mailman/listinfo/py-dev +.. _`test generators`: funcargs.html#test-generators + +.. _`generative tests`: + +generative tests: yielding parametrized tests +==================================================== + +Deprecated since 1.0 in favour of `test generators`_. + +*Generative tests* are test methods that are *generator functions* which +``yield`` callables and their arguments. This is useful for running a +test function multiple times against different parameters. Example:: + + def test_generative(): + for x in (42,17,49): + yield check, x + + def check(arg): + assert arg % 7 == 0 # second generated tests fails! + +Note that ``test_generative()`` will cause three tests +to get run, notably ``check(42)``, ``check(17)`` and ``check(49)`` +of which the middle one will obviously fail. + +To make it easier to distinguish the generated tests it is possible to specify an explicit name for them, like for example:: + + def test_generative(): + for x in (42,17,49): + yield "case %d" % x, check, x + + +disabling a test class +---------------------- + +If you want to disable a complete test class you +can set the class-level attribute ``disabled``. +For example, in order to avoid running some tests on Win32:: + + class TestPosixOnly: + disabled = sys.platform == 'win32' + + def test_xxx(self): + ... diff --git a/doc/test/config.html b/doc/test/config.html new file mode 100644 index 000000000..cba5a46f9 --- /dev/null +++ b/doc/test/config.html @@ -0,0 +1,18 @@ + + + + + + + + + + + diff --git a/doc/test/config.txt b/doc/test/config.txt deleted file mode 100644 index ef8aa8e6f..000000000 --- a/doc/test/config.txt +++ /dev/null @@ -1,96 +0,0 @@ -.. contents:: - :local: - :depth: 2 - -available test options ------------------------------ - -You can see command line options by running:: - - py.test -h - -This will display all available command line options -including the ones added by plugins `loaded at tool startup`_. - -.. _`loaded at tool startup`: extend.html#tool-startup - -.. _conftestpy: -.. _collectignore: - -conftest.py: project specific test configuration --------------------------------------------------------- - -A unique feature of py.test are its powerful ``conftest.py`` files which -allow to `set option defaults`_, `implement hooks`_, `specify funcargs`_ -or set particular variables to influence the testing process: - -* ``pytest_plugins``: list of named plugins to load - -* ``collect_ignore``: list of paths to ignore during test collection (relative to the containing - ``conftest.py`` file) - -* ``rsyncdirs``: list of to-be-rsynced directories for distributed - testing - -You may put a conftest.py files in your project root directory or into -your package directory if you want to add project-specific test options. - -``py.test`` loads all ``conftest.py`` files upwards from the command -line specified test files. It will lookup configuration values -right-to-left, i.e. the closer conftest files will be checked first. -You may have a ``conftest.py`` in your very home directory to have some -global configuration values. - -There is a flag that may help you debugging your conftest.py -configuration:: - - py.test --traceconfig - -.. _`implement hooks`: extend.html#conftest.py-plugin -.. _`specify funcargs`: funcargs.html#application-setup-tutorial-example - -.. _`set option defaults`: - -setting option defaults -------------------------------- - -py.test will lookup values of options in this order: - -* option value supplied at command line -* content of environment variable ``PYTEST_OPTION_NAME=...`` -* ``name = ...`` setting in the nearest ``conftest.py`` file. - -The name of an option usually is the one you find -in the longform of the option, i.e. the name -behind the ``--`` double-dash that you get with ``py.test -h``. - -IOW, you can set default values for options per project, per -home-directoray, per shell session or per test-run. - -.. _`basetemp`: - -Temporary directories -------------------------------------------- - -``py.test`` runs provide means to create per-test session -temporary (sub) directories through the config object. -You can create directories by calling a method -on the config object: - -- ``config.mktemp(basename)``: create and returns a new tempdir - -- ``config.ensuretemp(basename)``: create or return a new tempdir - -tempdirs are created as sub directories of a per-session testdir -and will keep around the directories of the last three -test runs. You can also set the base temporary directory -with the `--basetemp`` option. When distributing -tests on the same machine, ``py.test`` takes care to -pass around the basetemp directory such that all temporary -files land below the same basetemp directory. - -The config object is available when implementing `function arguments`_ -or `extensions`_ and can otherwise be globally accessed as ``py.test.config``. - -.. _`function arguments`: funcargs.html -.. _`extensions`: extend.html diff --git a/doc/test/extend.txt b/doc/test/customize.txt similarity index 72% rename from doc/test/extend.txt rename to doc/test/customize.txt index b5c9c2a5b..4e69ad89d 100644 --- a/doc/test/extend.txt +++ b/doc/test/customize.txt @@ -1,9 +1,113 @@ ================================================ -Extending and customizing py.test +Customizing and Extending py.test ================================================ +.. contents:: + :local: + :depth: 2 + +basic test configuration +=================================== + +available command line options +--------------------------------- + +You can see command line options by running:: + + py.test -h + +This will display all available command line options +including the ones added by plugins `loaded at tool startup`_. + + +.. _`project-specific test configuration`: +.. _`collect_ignore`: + +conftest.py: project specific test configuration +-------------------------------------------------------- + +A unique feature of py.test are its ``conftest.py`` files which +allow to `set option defaults`_, `implement hooks`_, `specify funcargs`_ +or set particular variables to influence the testing process: + +* ``pytest_plugins``: list of named plugins to load + +* ``collect_ignore``: list of paths to ignore during test collection (relative to the containing + ``conftest.py`` file) + +* ``rsyncdirs``: list of to-be-rsynced directories for distributed + testing + +You may put a conftest.py files in your project root directory or into +your package directory if you want to add project-specific test options. + +``py.test`` loads all ``conftest.py`` files upwards from the command +line specified test files. It will lookup configuration values +right-to-left, i.e. the closer conftest files will be checked first. +You may have a ``conftest.py`` in your very home directory to have some +global configuration values. + +There is a flag that may help you debugging your conftest.py +configuration:: + + py.test --traceconfig + +.. _`specify funcargs`: funcargs.html#application-setup-tutorial-example + +.. _`set option defaults`: + +setting option defaults +------------------------------- + +py.test will lookup values of options in this order: + +* option value supplied at command line +* content of environment variable ``PYTEST_OPTION_NAME=...`` +* ``name = ...`` setting in the nearest ``conftest.py`` file. + +The name of an option usually is the one you find +in the longform of the option, i.e. the name +behind the ``--`` double-dash that you get with ``py.test -h``. + +IOW, you can set default values for options per project, per +home-directoray, per shell session or per test-run. + +.. _`basetemp`: + +Temporary directories +------------------------------------------- + +``py.test`` runs provide means to create per-test session +temporary (sub) directories through the config object. +You can create directories by calling a method +on the config object: + +- ``config.mktemp(basename)``: create and returns a new tempdir + +- ``config.ensuretemp(basename)``: create or return a new tempdir + +tempdirs are created as sub directories of a per-session testdir +and will keep around the directories of the last three +test runs. You can also set the base temporary directory +with the `--basetemp`` option. When distributing +tests on the same machine, ``py.test`` takes care to +pass around the basetemp directory such that all temporary +files land below the same basetemp directory. + +The config object is available when implementing `function arguments`_ +or `extensions`_ and can otherwise be globally accessed as ``py.test.config``. + +.. _`function arguments`: funcargs.html +.. _`extensions`: + +Plugin basics +========================= + .. _`local plugin`: +project specific "local" or named "global" plugins +-------------------------------------------------------------- + py.test implements much of its functionality by calling `well specified hooks`_. Python modules which contain such hook functions are called plugins. Hook functions are discovered in ``conftest.py`` files or in @@ -18,6 +122,7 @@ named plugins must be explicitely specified. .. _`named plugins`: plugin/index.html .. _`tool startup`: +.. _`loaded at tool startup`: .. _`test tool starts up`: Plugin discovery at tool startup @@ -81,17 +186,18 @@ by defining the following hook in a ``conftest.py`` file: if config.getvalue("runall"): collect_ignore[:] = [] -.. _`project-specific test configuration`: config.html#conftestpy -.. _`collect_ignore`: config.html#collectignore .. _`well specified hooks`: +.. _`implement hooks`: -Available py.test hooks +Important py.test hooks ==================================== -py.test calls hooks functions to implement its `test collection`_, running and -reporting process. When py.test loads a plugin it validates that all hook functions -conform to the `hook definition specification`_. The hook function name and its +py.test calls hooks functions to implement its `test collection`_, +running and reporting process. When py.test loads a plugin it validates +that all hook functions conform to the `hook definition specification`_. + +The hook function name and its argument names need to match exactly but it is allowed for an implementation to accept *less* parameters. You'll get useful errors on mistyped hook or argument names. Read on for some introductory information on particular @@ -170,9 +276,10 @@ For example, the `pytest_pdb plugin`_ uses this hook to activate interactive debugging on failures when ``--pdb`` is specified on the command line. -Usually three reports will be generated for a single test item. However, -if the ``pytest_runtest_setup`` fails no call or teardown hooks -will be called and only one report will be created. +Usually three reports will be generated for a single test item for each +of the three runtest hooks respectively. If ``pytest_runtest_setup`` +fails then ``pytest_runtest_teardown`` will be called but not +``pytest_runtest_call``. Each of the up to three reports is eventually fed to the logreport hook: @@ -194,7 +301,7 @@ A ``report`` object contains status and reporting information: The `pytest_terminal plugin`_ uses this hook to print information about a test run. -The protocol described here is implemented via this hook: +The whole protocol described here is implemented via this hook: .. sourcecode:: python @@ -251,19 +358,6 @@ Python module. The return value is a custom `collection node`_ or None. .. XXX or ``False`` if you want to indicate that the given item should not be collected. - -Included default plugins -============================= - -You can find the source code of all default plugins in - - http://bitbucket.org/hpk42/py-trunk/src/tip/py/test/plugin/ - -Additionally you can check out some more contributed plugins here - - http://bitbucket.org/hpk42/py-trunk/src/tip/contrib/ - - .. _`collection process`: .. _`collection node`: .. _`test collection`: @@ -272,6 +366,9 @@ Additionally you can check out some more contributed plugins here Test Collection process ====================================================== +the collection tree +--------------------------------- + The collecting process is iterative so that distribution and execution of tests can start as soon as the first test item is collected. Collection nodes with children are diff --git a/doc/test/extend.html b/doc/test/extend.html new file mode 100644 index 000000000..cba5a46f9 --- /dev/null +++ b/doc/test/extend.html @@ -0,0 +1,18 @@ + + + + + + + + + + + diff --git a/doc/test/features.txt b/doc/test/features.txt index 2de4dc178..7fbfed7aa 100644 --- a/doc/test/features.txt +++ b/doc/test/features.txt @@ -1,117 +1,77 @@ ================================================== -py.test features +py.test feature overview ================================================== -py.test is an extensible tool for running all kinds -of tests on one or more machines. It supports a variety -of testing methods including unit, functional, integration -and doc-testing. It is used in projects that run more -than 10 thousand tests regularly as well as in single-file projects. - -py.test presents a clean and powerful command line interface -and strives to generally make testing a fun no-boilerplate effort. -It works and is tested against linux, windows and osx -on CPython 2.3 - CPython 2.6. - -.. contents:: List of Contents +.. contents:: + :local: :depth: 1 +mature command line testing tool +==================================================== + +py.test is a command line tool to collect and run automated tests. It +runs well on Linux, Windows and OSX Python 2.4 through to 2.6 versions. +It can distribute a single test run to multiple machines. It is used in +many projects, ranging from running 10 thousands of tests integrated +with buildbot to a few inlined tests on a command line script. + .. _`autocollect`: automatically collects and executes tests =============================================== -py.test discovers tests automatically by inspecting specified -directories or files. By default, it collects all python -modules a leading ``test_`` or trailing ``_test`` filename. -From each test module every function with a leading ``test_`` -or class with a leading ``Test`` name is collected. - -.. _`collection process`: extend.html#collection-process - - -funcargs and xUnit style setups -=================================================== - -py.test provides powerful means for managing test -state and fixtures. Apart from the `traditional -xUnit style setup`_ for unittests it features the -simple and powerful `funcargs mechanism`_ for handling -both complex and simple test scenarious. - -.. _`funcargs mechanism`: funcargs.html -.. _`traditional xUnit style setup`: xunit_setup.html - -load-balance tests to multiple CPUs -=================================== - -For large test suites you can distribute your -tests to multiple CPUs by issuing for example:: - - py.test -n 3 - -Read more on `distributed testing`_. - -.. _`distributed testing`: dist.html - -Distribute tests across machines -=================================== - -py.test supports the sending of tests to -remote ssh-accounts or socket servers. -It can `ad-hoc run your test on multiple -platforms one a single test run`. Ad-hoc -means that there are **no installation -requirements whatsoever** on the remote side. - -.. _`ad-hoc run your test on multiple platforms one a single test run`: dist.html#atonce - -extensive debugging support -=================================== - -testing starts immediately --------------------------- - -Testing starts as soon as the first ``test item`` -is collected. The collection process is iterative -and does not need to complete before your first -test items are executed. - -support for modules containing tests --------------------------------------- - -As ``py.test`` operates as a separate cmdline -tool you can easily have a command line utility and +py.test discovers tests automatically by looking at +specified directories and its files for common +naming patterns. As ``py.test`` operates as a separate +cmdline tool you can easily have a command line utility and some tests in the same file. -debug with the ``print`` statement ----------------------------------- +supports many testing practises and methods +================================================================== -By default, ``py.test`` catches text written to stdout/stderr during -the execution of each individual test. This output will only be -displayed however if the test fails; you will not see it -otherwise. This allows you to put debugging print statements in your -code without being overwhelmed by all the output that might be -generated by tests that do not fail. +py.test supports many testing methods conventionally used in +the Python community. It runs traditional `unittest.py`_, +`doctest.py`_, supports `xUnit style setup`_ and nose_ specific +setups and test suites. It offers minimal no-boilerplate model +for configuring and deploying tests written as simple Python +functions or methods. It also integrates `coverage testing +with figleaf`_ or `Javasript unit- and functional testing`_. -Each failing test that produced output during the running of the test -function will have its output displayed in the ``recorded stdout`` section. +.. _`Javasript unit- and functional testing`: plugin/oejskit.html +.. _`coverage testing with figleaf`: plugin/figleaf.html -During Setup and Teardown ("Fixture") capturing is performed separately so -that you will only see this output if the actual fixture functions fail. +no-boilerplate test functions with Python +=================================================== -The catching of stdout/stderr output can be disabled using the -``--nocapture`` or ``-s`` option to the ``py.test`` tool. Any output will -in this case be displayed as soon as it is generated. +automatic Python test discovery +------------------------------------ -test execution order --------------------------------- +By default, all python modules with a ``test_*.py`` +filename are inspected for finding tests: -Tests usually run in the order in which they appear in the files. -However, tests should not rely on running one after another, as -this prevents more advanced usages: running tests -distributedly or selectively, or in "looponfailing" mode, -will cause them to run in random order. +* functions with a name beginning with ``test_`` +* classes with a leading ``Test`` name and ``test`` prefixed methods. +* ``unittest.TestCase`` subclasses + +test functions can run with different argument sets +----------------------------------------------------------- + +py.test offers the unique `funcargs mechanism`_ for setting up +and passing project-specific objects to Python test functions. +Test Parametrization happens by triggering a call to the same test +functions with different argument values. + +per-test capturing of output, including subprocesses +---------------------------------------------------- + +By default, ``py.test`` captures all writes to stdout/stderr. +Output from ``print`` statements as well as from subprocesses +is captured_. When a test fails, the associated captured outputs are shown. +This allows you to put debugging print statements in your code without +being overwhelmed by all the output that might be generated by tests +that do not fail. + +.. _captured: plugin/capture.html assert with the ``assert`` statement ---------------------------------------- @@ -142,61 +102,35 @@ one of two forms:: py.test.raises(Exception, func, *args, **kwargs) py.test.raises(Exception, "func(*args, **kwargs)") -both of which execute the given function with args and kwargs and +both of which execute the specified function with args and kwargs and asserts that the given ``Exception`` is raised. The reporter will provide you with helpful output in case of failures such as *no exception* or *wrong exception*. -useful tracebacks, recursion detection --------------------------------------- -A lot of care is taken to present nice tracebacks in case of test -failure. Try:: +information-rich tracebacks, PDB introspection +------------------------------------------------------- - py.test py/doc/example/pytest/failure_demo.py +.. _`example tracebacks`: http://paste.pocoo.org/show/134814/ -to see a variety of tracebacks, each representing a different -failure situation. +A lot of care is taken to present useful failure information +and in particular nice and concise Python tracebacks. This +is especially useful if you need to regularly look at failures +from nightly runs, i.e. are detached from the actual test +running session. Here are `example tracebacks`_ for a number of failing +test functions. You can modify traceback printing styles through the +command line. Using the `--pdb`` option you can automatically activate +a PDB `Python debugger`_ when a test fails. -``py.test`` uses the same order for presenting tracebacks as Python -itself: the oldest function call is shown first, and the most recent call is -shown last. A ``py.test`` reported traceback starts with your -failing test function. If the maximum recursion depth has been -exceeded during the running of a test, for instance because of -infinite recursion, ``py.test`` will indicate where in the -code the recursion was taking place. You can inhibit -traceback "cutting" magic by supplying ``--fulltrace``. - -There is also the possibility of using ``--tb=short`` to get regular CPython -tracebacks. Or you can use ``--tb=no`` to not show any tracebacks at all. - -no inheritance requirement --------------------------- - -Test classes are recognized by their leading ``Test`` name. Unlike -``unitest.py``, you don't need to inherit from some base class to make -them be found by the test runner. Besides being easier, it also allows -you to write test classes that subclass from application level -classes. - -testing for deprecated APIs ------------------------------- - -In your tests you can use ``py.test.deprecated_call(func, *args, **kwargs)`` -to test that a particular function call triggers a DeprecationWarning. -This is useful for testing phasing out of old APIs in your projects. - - -advanced test selection / skipping -========================================================= - -dynamically skipping tests +advanced skipping of tests ------------------------------- If you want to skip tests you can use ``py.test.skip`` within test or setup functions. Example:: - py.test.skip("message") + def test_hello(): + if sys.platform != "win32": + py.test.skip("only win32 supported") You can also use a helper to skip on a failing import:: @@ -206,15 +140,48 @@ or to skip if a library does not have the right version:: docutils = py.test.importorskip("docutils", minversion="0.3") -The version will be read from the module's ``__version__`` attribute. +The version will be read from the specified module's ``__version__`` attribute. + +.. _`funcargs mechanism`: funcargs.html +.. _`unittest.py`: http://docs.python.org/library/unittest.html +.. _`doctest.py`: http://docs.python.org/library/doctest.html +.. _`xUnit style setup`: xunit_setup.html +.. _`pytest_nose`: plugin/nose.html + +load-balance test runs to multiple CPUs +======================================== + +For large test suites you can distribute your +tests to multiple CPUs by issuing for example:: + + py.test -n 3 + +Read more on `distributed testing`_. + +.. _`distributed testing`: dist.html + +ad-hoc run tests cross-platform +================================================== + +py.test supports the sending of tests to +remote ssh-accounts, socket servers. +It can `ad-hoc run your test on multiple +platforms one a single test run`. Ad-hoc +means that there are **no installation +requirements whatsoever** on the remote side. + +.. _`ad-hoc run your test on multiple platforms one a single test run`: dist.html#atonce + +advanced test selection and running modes +========================================================= .. _`selection by keyword`: -selecting/unselecting tests by keyword ---------------------------------------------- +``py.test --looponfailing`` allows to run a test suite, +memorize all failures and then loop over the failing set +of tests until they all pass. It will re-start running +the tests when it detects file changes in your project. -Pytest's keyword mechanism provides a powerful way to -group and selectively run tests in your test code base. You can selectively run tests by specifiying a keyword on the command line. Examples:: @@ -246,65 +213,21 @@ plugin for more information. .. _`pytest_keyword`: plugin/keyword.html - -disabling a test class ----------------------- - -If you want to disable a complete test class you -can set the class-level attribute ``disabled``. -For example, in order to avoid running some tests on Win32:: - - class TestPosixOnly: - disabled = sys.platform == 'win32' - - def test_xxx(self): - ... - -.. _`test generators`: funcargs.html#test-generators - -.. _`generative tests`: - -generative tests: yielding parametrized tests -==================================================== - -Deprecated since 1.0 in favour of `test generators`_. - -*Generative tests* are test methods that are *generator functions* which -``yield`` callables and their arguments. This is useful for running a -test function multiple times against different parameters. Example:: - - def test_generative(): - for x in (42,17,49): - yield check, x - - def check(arg): - assert arg % 7 == 0 # second generated tests fails! - -Note that ``test_generative()`` will cause three tests -to get run, notably ``check(42)``, ``check(17)`` and ``check(49)`` -of which the middle one will obviously fail. - -To make it easier to distinguish the generated tests it is possible to specify an explicit name for them, like for example:: - - def test_generative(): - for x in (42,17,49): - yield "case %d" % x, check, x - easy to extend ========================================= -Since 1.0 py.test has advanced `extension mechanisms`_ -and a growing `list of plugins`_. +py.test has advanced `extension mechanisms`_ +with a growing `list of default plugins`_. One can can easily modify or add aspects for for purposes such as: * reporting extensions * customizing collection and execution of tests -* running non-python tests -* managing custom test state setup +* running and managing non-python tests +* managing domain-specific test state setup -.. _`list of plugins`: plugin/index.html -.. _`extension mechanisms`: extend.html +.. _`list of default plugins`: plugin/index.html +.. _`extension mechanisms`: customize.html#extensions .. _`reStructured Text`: http://docutils.sourceforge.net .. _`Python debugger`: http://docs.python.org/lib/module-pdb.html diff --git a/doc/test/funcargs.txt b/doc/test/funcargs.txt index 2f6a904b9..edb4032ef 100644 --- a/doc/test/funcargs.txt +++ b/doc/test/funcargs.txt @@ -2,8 +2,15 @@ **funcargs**: test function arguments FTW ========================================================== +.. contents:: + :local: + :depth: 2 + +Goals of the "funcarg" mechanism +========================================== + Since version 1.0 py.test features the "funcarg" mechanism which -allows a test function to take arguments independently provided +allows a Python test function to take arguments independently provided by factory functions. Factory functions allow to encapsulate all setup and fixture glue code into nicely separated objects and provide a natural way for writing python test functions. @@ -14,7 +21,6 @@ Compared to `xUnit style`_ the new mechanism is meant to: * bring new flexibility and power to test state management * naturally extend towards parametrizing test functions with multiple argument sets - (superseding `old-style generative tests`_) * enable creation of zero-boilerplate test helper objects that interact with the execution of a test function, see the `blog post about the monkeypatch funcarg`_. @@ -26,272 +32,17 @@ the mechanism you are welcome to checkout `contact possibilities`_ page. .. _`blog post about the monkeypatch funcarg`: http://tetamap.wordpress.com/2009/03/03/monkeypatching-in-unit-tests-done-right/ .. _`xUnit style`: xunit_setup.html -.. _`old-style generative tests`: features.html#generative-tests - -.. _`funcarg factory`: - -funcarg factories: setting up test function arguments -============================================================== - -Test functions can specify one ore more arguments ("funcargs") -and a test module or plugin can define functions that provide -the function argument. Let's look at a simple self-contained -example that you can put into a test module: - -.. sourcecode:: python - - # ./test_simplefactory.py - def pytest_funcarg__myfuncarg(request): - return 42 - - def test_function(myfuncarg): - assert myfuncarg == 17 - -If you run this with ``py.test test_simplefactory.py`` you see something like this: - -.. sourcecode:: python - - ============================ test session starts ============================ - python: platform linux2 -- Python 2.6.2 - test object 1: /home/hpk/hg/py/trunk/example/funcarg/test_simplefactory.py - - test_simplefactory.py F - - ================================= FAILURES ================================== - _______________________________ test_function _______________________________ - - myfuncarg = 42 - - def test_function(myfuncarg): - > assert myfuncarg == 17 - E assert 42 == 17 - - test_simplefactory.py:6: AssertionError - ========================= 1 failed in 0.11 seconds ========================== - - -This means that the test function got executed and the assertion failed. -Here is how py.test comes to execute this test function: - -1. py.test discovers the ``test_function`` because of the ``test_`` prefix. - The test function needs a function argument named ``myfuncarg``. - A matching factory function is discovered by looking for the special - name ``pytest_funcarg__myfuncarg``. - -2. ``pytest_funcarg__myfuncarg(request)`` is called and - returns the value for ``myfuncarg``. - -3. ``test_function(42)`` call is executed. - -Note that if you misspell a function argument or want -to use one that isn't available, an error with a list of -available function argument is provided. - -For more interesting factory functions that make good use of the -`request object`_ please see the `application setup tutorial example`_. - -.. _`request object`: - -funcarg factory request objects ------------------------------------------- - -Request objects are passed to funcarg factories and allow -to access test configuration, test context and `useful caching -and finalization helpers`_. Here is a list of attributes: - -``request.function``: python function object requesting the argument - -``request.cls``: class object where the test function is defined in or None. - -``request.module``: module object where the test function is defined in. - -``request.config``: access to command line opts and general config - -``request.param``: if exists was passed by a `parametrizing test generator`_ - -.. _`useful caching and finalization helpers`: - - -registering funcarg related finalizers/cleanup ----------------------------------------------------- - -.. sourcecode:: python - - def addfinalizer(func): - """ call a finalizer function when test function finishes. """ - -Calling ``request.addfinalizer()`` is useful for scheduling teardown -functions. Here is an example for providing a ``myfile`` -object that is to be closed when the execution of a -test function finishes. - -.. sourcecode:: python - - def pytest_funcarg__myfile(self, request): - # ... create and open a unique per-function "myfile" object ... - request.addfinalizer(lambda: myfile.close()) - return myfile - - -managing fixtures across test modules and test runs ----------------------------------------------------------- - -.. sourcecode:: python - - def cached_setup(setup, teardown=None, scope="module", extrakey=None): - """ cache and return result of calling setup(). - - The scope and the ``extrakey`` determine the cache key. - The scope also determines when teardown(result) - will be called. valid scopes are: - scope == 'function': when the single test function run finishes. - scope == 'module': when tests in a different module are run - scope == 'session': when tests of the session have run. - """ - -Calling ``request.cached_setup()`` helps you to manage fixture -objects across several scopes. For example, for creating a Database object -that is to be setup only once during a test session you can use the helper -like this: - -.. sourcecode:: python - - def pytest_funcarg__database(request): - return request.cached_setup( - setup=lambda: Database("..."), - teardown=lambda val: val.close(), - scope="session" - ) - - -requesting values of other funcargs ---------------------------------------------- - -.. sourcecode:: python - - def getfuncargvalue(name): - """ Lookup and call function argument factory for the given name. - Each function argument is only created once per function setup. - """ - -``request.getfuncargvalue(name)`` calls another funcarg factory function. -You can use this function if you want to `decorate a funcarg`_, i.e. -you want to provide the "normal" value but add something -extra. If a factory cannot be found a ``request.Error`` -exception will be raised. - -.. _`test generators`: -.. _`parametrizing test generator`: - -generating parametrized tests with funcargs -=========================================================== - -You can parametrize multiple runs of the same test -function by adding new test function calls with different -function argument values. Let's look at a simple self-contained -example: - -.. sourcecode:: python - - # ./test_example.py - def pytest_generate_tests(metafunc): - if "numiter" in metafunc.funcargnames: - for i in range(10): - metafunc.addcall(funcargs=dict(numiter=i)) - - def test_func(numiter): - assert numiter < 9 - -If you run this with ``py.test test_example.py`` you'll get: - -.. sourcecode:: python - - ================================= test session starts ================================= - python: platform linux2 -- Python 2.6.2 - test object 1: /home/hpk/hg/py/trunk/test_example.py - - test_example.py .........F - - ====================================== FAILURES ======================================= - _______________________________ test_func.test_func[9] ________________________________ - - numiter = 9 - - def test_func(numiter): - > assert numiter < 9 - E assert 9 < 9 - - /home/hpk/hg/py/trunk/test_example.py:10: AssertionError - - -Here is what happens in detail: - -1. ``pytest_generate_tests(metafunc)`` hook is called once for each test - function. It adds ten new function calls with explicit function arguments. - -2. **execute tests**: ``test_func(numiter)`` is called ten times with - ten different arguments. - -.. _`metafunc object`: - -test generators and metafunc objects -------------------------------------------- - -metafunc objects are passed to the ``pytest_generate_tests`` hook. -They help to inspect a testfunction and to generate tests -according to test configuration or values specified -in the class or module where a test function is defined: - -``metafunc.funcargnames``: set of required function arguments for given function - -``metafunc.function``: underlying python test function - -``metafunc.cls``: class object where the test function is defined in or None. - -``metafunc.module``: the module object where the test function is defined in. - -``metafunc.config``: access to command line opts and general config - - -the ``metafunc.addcall()`` method ------------------------------------------------ - -.. sourcecode:: python - - def addcall(funcargs={}, id=None, param=None): - """ trigger a new test function call. """ - -``funcargs`` can be a dictionary of argument names -mapped to values - providing it is called *direct parametrization*. - -If you provide an `id`` it will be used for reporting -and identification purposes. If you don't supply an `id` -the stringified counter of the list of added calls will be used. -``id`` values needs to be unique between all -invocations for a given test function. - -``param`` if specified will be seen by any -`funcarg factory`_ as a ``request.param`` attribute. -Setting it is called *indirect parametrization*. - -Indirect parametrization is preferable if test values are -expensive to setup or can only be created in certain environments. -Test generators and thus ``addcall()`` invocations are performed -during test collection which is separate from the actual test -setup and test run phase. With distributed testing collection -and test setup/run happens in different process. - .. _`tutorial examples`: -Funcarg Tutorial Examples +Tutorial Examples ======================================= .. _`application setup tutorial example`: .. _appsetup: -application specific test setup +application specific test setup and fixtures --------------------------------------------------------- Here is a basic useful step-wise example for handling application @@ -376,8 +127,9 @@ confused as to what the concrete question or answers actually mean, please see here_ :) Otherwise proceed to step 2. .. _here: http://uncyclopedia.wikia.com/wiki/The_Hitchhiker's_Guide_to_the_Galaxy -.. _`local plugin`: extend.html#local-plugin +.. _`local plugin`: customize.html#local-plugin +.. _`tut-cmdlineoption`: step 2: adding a command line option ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -521,25 +273,262 @@ the `py.path.local`_ class which provides many of the os.path methods in a convenient way. .. _`py.path.local`: ../path.html#local -.. _`conftest plugin`: extend.html#conftestplugin +.. _`conftest plugin`: customize.html#conftestplugin + +.. _`funcarg factory`: + +funcarg factories: setting up test function arguments +============================================================== + +Test functions can specify one ore more arguments ("funcargs") +and a test module or plugin can define functions that provide +the function argument. Let's look at a simple self-contained +example that you can put into a test module: + +.. sourcecode:: python + + # ./test_simplefactory.py + def pytest_funcarg__myfuncarg(request): + return 42 + + def test_function(myfuncarg): + assert myfuncarg == 17 + +If you run this with ``py.test test_simplefactory.py`` you see something like this: + +.. sourcecode:: python + + =========================== test session starts ============================ + python: platform linux2 -- Python 2.6.2 + test object 1: /home/hpk/hg/py/trunk/example/funcarg/test_simplefactory.py + + test_simplefactory.py F + + ================================ FAILURES ================================== + ______________________________ test_function _______________________________ + + myfuncarg = 42 + + def test_function(myfuncarg): + > assert myfuncarg == 17 + E assert 42 == 17 + + test_simplefactory.py:6: AssertionError + ======================== 1 failed in 0.11 seconds ========================== -Questions and Answers -================================== +This means that the test function got executed and the assertion failed. +Here is how py.test comes to execute this test function: -.. _`why pytest_pyfuncarg__ methods?`: +1. py.test discovers the ``test_function`` because of the ``test_`` prefix. + The test function needs a function argument named ``myfuncarg``. + A matching factory function is discovered by looking for the special + name ``pytest_funcarg__myfuncarg``. -Why ``pytest_funcarg__*`` methods? ------------------------------------- +2. ``pytest_funcarg__myfuncarg(request)`` is called and + returns the value for ``myfuncarg``. + +3. ``test_function(42)`` call is executed. + +Note that if you misspell a function argument or want +to use one that isn't available, an error with a list of +available function argument is provided. + +For more interesting factory functions that make good use of the +`request object`_ please see the `application setup tutorial example`_. + +.. _`request object`: + +funcarg factory request objects +------------------------------------------ + +Request objects are passed to funcarg factories and allow +to access test configuration, test context and `useful caching +and finalization helpers`_. Here is a list of attributes: + +``request.function``: python function object requesting the argument + +``request.cls``: class object where the test function is defined in or None. + +``request.module``: module object where the test function is defined in. + +``request.config``: access to command line opts and general config + +``request.param``: if exists was passed by a previous `metafunc.addcall`_ + +.. _`useful caching and finalization helpers`: + + +registering funcarg related finalizers/cleanup +---------------------------------------------------- + +.. sourcecode:: python + + def addfinalizer(func): + """ call a finalizer function when test function finishes. """ + +Calling ``request.addfinalizer()`` is useful for scheduling teardown +functions. Here is an example for providing a ``myfile`` +object that is to be closed when the execution of a +test function finishes. + +.. sourcecode:: python + + def pytest_funcarg__myfile(self, request): + # ... create and open a unique per-function "myfile" object ... + request.addfinalizer(lambda: myfile.close()) + return myfile + + +managing fixtures across test modules and test runs +---------------------------------------------------------- + +.. sourcecode:: python + + def cached_setup(setup, teardown=None, scope="module", extrakey=None): + """ cache and return result of calling setup(). + + The scope and the ``extrakey`` determine the cache key. + The scope also determines when teardown(result) + will be called. valid scopes are: + scope == 'function': when the single test function run finishes. + scope == 'module': when tests in a different module are run + scope == 'session': when tests of the session have run. + """ + +Calling ``request.cached_setup()`` helps you to manage fixture +objects across several scopes. For example, for creating a Database object +that is to be setup only once during a test session you can use the helper +like this: + +.. sourcecode:: python + + def pytest_funcarg__database(request): + return request.cached_setup( + setup=lambda: Database("..."), + teardown=lambda val: val.close(), + scope="session" + ) + + +requesting values of other funcargs +--------------------------------------------- + +.. sourcecode:: python + + def getfuncargvalue(name): + """ Lookup and call function argument factory for the given name. + Each function argument is only created once per function setup. + """ + +``request.getfuncargvalue(name)`` calls another funcarg factory function. +You can use this function if you want to `decorate a funcarg`_, i.e. +you want to provide the "normal" value but add something +extra. If a factory cannot be found a ``request.Error`` +exception will be raised. + +.. _`test generators`: +.. _`parametrizing-tests`: + +generating parametrized tests +=========================================================== + +You can parametrize multiple runs of the same test +function by adding new test function calls with different +function argument values. Let's look at a simple self-contained +example: + +.. sourcecode:: python + + # ./test_example.py + def pytest_generate_tests(metafunc): + if "numiter" in metafunc.funcargnames: + for i in range(10): + metafunc.addcall(funcargs=dict(numiter=i)) + + def test_func(numiter): + assert numiter < 9 + +If you run this with ``py.test test_example.py`` you'll get: + +.. sourcecode:: python + + ============================= test session starts ========================== + python: platform linux2 -- Python 2.6.2 + test object 1: /home/hpk/hg/py/trunk/test_example.py + + test_example.py .........F + + ================================ FAILURES ================================== + __________________________ test_func.test_func[9] __________________________ + + numiter = 9 + + def test_func(numiter): + > assert numiter < 9 + E assert 9 < 9 + + /home/hpk/hg/py/trunk/test_example.py:10: AssertionError + + +Here is what happens in detail: + +1. ``pytest_generate_tests(metafunc)`` hook is called once for each test + function. It adds ten new function calls with explicit function arguments. + +2. **execute tests**: ``test_func(numiter)`` is called ten times with + ten different arguments. + +.. _`metafunc object`: + +test generators and metafunc objects +------------------------------------------- + +metafunc objects are passed to the ``pytest_generate_tests`` hook. +They help to inspect a testfunction and to generate tests +according to test configuration or values specified +in the class or module where a test function is defined: + +``metafunc.funcargnames``: set of required function arguments for given function + +``metafunc.function``: underlying python test function + +``metafunc.cls``: class object where the test function is defined in or None. + +``metafunc.module``: the module object where the test function is defined in. + +``metafunc.config``: access to command line opts and general config + + +.. _`metafunc.addcall`: + +the ``metafunc.addcall()`` method +----------------------------------------------- + +.. sourcecode:: python + + def addcall(funcargs={}, id=None, param=None): + """ trigger a new test function call. """ + +``funcargs`` can be a dictionary of argument names +mapped to values - providing it is called *direct parametrization*. + +If you provide an `id`` it will be used for reporting +and identification purposes. If you don't supply an `id` +the stringified counter of the list of added calls will be used. +``id`` values needs to be unique between all +invocations for a given test function. + +``param`` if specified will be seen by any +`funcarg factory`_ as a ``request.param`` attribute. +Setting it is called *indirect parametrization*. + +Indirect parametrization is preferable if test values are +expensive to setup or can only be created in certain environments. +Test generators and thus ``addcall()`` invocations are performed +during test collection which is separate from the actual test +setup and test run phase. With distributed testing collection +and test setup/run happens in different process. -When experimenting with funcargs we also -considered an explicit registration mechanism, i.e. calling a register -method on the config object. But lacking a good use case for this -indirection and flexibility we decided to go for `Convention over -Configuration`_ and allow to directly specify the factory. It has the -positive implication that you should be able to "grep" for -``pytest_funcarg__MYARG`` and will find all providing sites (usually -exactly one). -.. _`Convention over Configuration`: http://en.wikipedia.org/wiki/Convention_over_Configuration diff --git a/doc/test/index.txt b/doc/test/index.txt new file mode 100644 index 000000000..a9f6341e1 --- /dev/null +++ b/doc/test/index.txt @@ -0,0 +1,29 @@ +======================================= +py.test documentation index +======================================= + + +features_: overview and discussion of features. + +quickstart_: getting started with writing a simple test. + +`talks, tutorials, examples`_: tutorial examples, slides + +funcargs_: powerful parametrized test function setup + +`plugins`_: list of available plugins with usage examples and feature details. + +`distributed testing`_: ad-hoc run tests on multiple CPUs and platforms + +customize_: configuration, customization, extensions + + +.. _`plugins`: plugin/index.html +.. _`talks, tutorials, examples`: talks.html +.. _quickstart: quickstart.html +.. _features: features.html +.. _funcargs: funcargs.html +.. _customize: customize.html +.. _`distributed testing`: dist.html + + diff --git a/doc/test/mission.txt b/doc/test/mission.txt new file mode 100644 index 000000000..07f017b9b --- /dev/null +++ b/doc/test/mission.txt @@ -0,0 +1,13 @@ + +Mission +==================================== + +py.test strives to make testing a fun and no-boilerplate effort. + +The tool is distributed as part of the `py` package which contains supporting APIs that +are also useable independently. The project independent ``py.test`` command line tool helps you to: + +* rapidly collect and run tests +* run unit- or doctests, functional or integration tests +* distribute tests to multiple environments +* use local or global plugins for custom test types and setup diff --git a/doc/test/plugin/capture.txt b/doc/test/plugin/capture.txt index 39d44cc16..d333c4bc9 100644 --- a/doc/test/plugin/capture.txt +++ b/doc/test/plugin/capture.txt @@ -120,12 +120,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_capture.py`_ plugin source code 2. put it somewhere as ``pytest_capture.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/plugin/doctest.txt b/doc/test/plugin/doctest.txt index 95d525b48..1f2894de1 100644 --- a/doc/test/plugin/doctest.txt +++ b/doc/test/plugin/doctest.txt @@ -29,12 +29,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_doctest.py`_ plugin source code 2. put it somewhere as ``pytest_doctest.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/plugin/figleaf.txt b/doc/test/plugin/figleaf.txt index 737b116ce..79ebafedb 100644 --- a/doc/test/plugin/figleaf.txt +++ b/doc/test/plugin/figleaf.txt @@ -24,12 +24,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_figleaf.py`_ plugin source code 2. put it somewhere as ``pytest_figleaf.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/plugin/hooklog.txt b/doc/test/plugin/hooklog.txt index 4fb4d8f40..687bf1ed3 100644 --- a/doc/test/plugin/hooklog.txt +++ b/doc/test/plugin/hooklog.txt @@ -20,12 +20,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_hooklog.py`_ plugin source code 2. put it somewhere as ``pytest_hooklog.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/plugin/hookspec.txt b/doc/test/plugin/hookspec.txt index f6d2ad099..847a68303 100644 --- a/doc/test/plugin/hookspec.txt +++ b/doc/test/plugin/hookspec.txt @@ -97,7 +97,7 @@ hook specification sourcecode """ called before test session finishes. """ pytest__teardown_final.firstresult = True - def pytest__teardown_final_logerror(rep): + def pytest__teardown_final_logerror(report): """ called if runtest_teardown_final failed. """ # ------------------------------------------------------------------------- @@ -114,8 +114,8 @@ hook specification sourcecode # hooks for influencing reporting (invoked from pytest_terminal) # ------------------------------------------------------------------------- - def pytest_report_teststatus(rep): - """ return shortletter and verbose word. """ + def pytest_report_teststatus(report): + """ return result-category, shortletter and verbose word for reporting.""" pytest_report_teststatus.firstresult = True def pytest_terminal_summary(terminalreporter): diff --git a/doc/test/plugin/index.txt b/doc/test/plugin/index.txt index 84f93d37b..22a0e5944 100644 --- a/doc/test/plugin/index.txt +++ b/doc/test/plugin/index.txt @@ -16,11 +16,13 @@ recwarn_ helpers for asserting deprecation and other warnings. Plugins for other testing styles and languages ============================================== +oejskit_ run javascript tests in real life browsers + unittest_ automatically discover and run traditional "unittest.py" style tests. -doctest_ collect and execute doctests from modules and test files. +nose_ nose-compatibility plugin: allow to run nose test suites natively. -oejskit_ run javascript tests in real life browsers +doctest_ collect and execute doctests from modules and test files. restdoc_ perform ReST syntax, local and remote reference tests on .rst/.txt files. diff --git a/doc/test/plugin/keyword.txt b/doc/test/plugin/keyword.txt index 91756ac5b..d330a6ce7 100644 --- a/doc/test/plugin/keyword.txt +++ b/doc/test/plugin/keyword.txt @@ -35,12 +35,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_keyword.py`_ plugin source code 2. put it somewhere as ``pytest_keyword.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/plugin/links.txt b/doc/test/plugin/links.txt index 270e90c2b..b2e560406 100644 --- a/doc/test/plugin/links.txt +++ b/doc/test/plugin/links.txt @@ -1,33 +1,35 @@ .. _`terminal`: terminal.html -.. _`pytest_recwarn.py`: http://bitbucket.org/hpk42/py-trunk/raw/caf81a2cdf5083be3bfb715d7e56273330a54843/py/test/plugin/pytest_recwarn.py +.. _`pytest_recwarn.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_recwarn.py .. _`unittest`: unittest.html -.. _`pytest_monkeypatch.py`: http://bitbucket.org/hpk42/py-trunk/raw/caf81a2cdf5083be3bfb715d7e56273330a54843/py/test/plugin/pytest_monkeypatch.py -.. _`pytest_keyword.py`: http://bitbucket.org/hpk42/py-trunk/raw/caf81a2cdf5083be3bfb715d7e56273330a54843/py/test/plugin/pytest_keyword.py +.. _`pytest_monkeypatch.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_monkeypatch.py +.. _`pytest_keyword.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_keyword.py .. _`pastebin`: pastebin.html .. _`plugins`: index.html -.. _`pytest_capture.py`: http://bitbucket.org/hpk42/py-trunk/raw/caf81a2cdf5083be3bfb715d7e56273330a54843/py/test/plugin/pytest_capture.py -.. _`pytest_doctest.py`: http://bitbucket.org/hpk42/py-trunk/raw/caf81a2cdf5083be3bfb715d7e56273330a54843/py/test/plugin/pytest_doctest.py +.. _`pytest_capture.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_capture.py +.. _`pytest_doctest.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_doctest.py .. _`capture`: capture.html -.. _`hooklog`: hooklog.html -.. _`pytest_restdoc.py`: http://bitbucket.org/hpk42/py-trunk/raw/caf81a2cdf5083be3bfb715d7e56273330a54843/py/test/plugin/pytest_restdoc.py -.. _`pytest_hooklog.py`: http://bitbucket.org/hpk42/py-trunk/raw/caf81a2cdf5083be3bfb715d7e56273330a54843/py/test/plugin/pytest_hooklog.py -.. _`pytest_pastebin.py`: http://bitbucket.org/hpk42/py-trunk/raw/caf81a2cdf5083be3bfb715d7e56273330a54843/py/test/plugin/pytest_pastebin.py -.. _`pytest_figleaf.py`: http://bitbucket.org/hpk42/py-trunk/raw/caf81a2cdf5083be3bfb715d7e56273330a54843/py/test/plugin/pytest_figleaf.py +.. _`pytest_nose.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_nose.py +.. _`pytest_restdoc.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_restdoc.py .. _`xfail`: xfail.html -.. _`contact`: ../../contact.html +.. _`pytest_pastebin.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_pastebin.py +.. _`pytest_figleaf.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_figleaf.py +.. _`pytest_hooklog.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_hooklog.py .. _`checkout the py.test development version`: ../../download.html#checkout .. _`oejskit`: oejskit.html -.. _`pytest_xfail.py`: http://bitbucket.org/hpk42/py-trunk/raw/caf81a2cdf5083be3bfb715d7e56273330a54843/py/test/plugin/pytest_xfail.py +.. _`doctest`: doctest.html +.. _`get in contact`: ../../contact.html +.. _`pytest_xfail.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_xfail.py .. _`figleaf`: figleaf.html -.. _`extend`: ../extend.html -.. _`pytest_terminal.py`: http://bitbucket.org/hpk42/py-trunk/raw/caf81a2cdf5083be3bfb715d7e56273330a54843/py/test/plugin/pytest_terminal.py +.. _`customize`: ../customize.html +.. _`hooklog`: hooklog.html +.. _`pytest_terminal.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_terminal.py .. _`recwarn`: recwarn.html -.. _`pytest_pdb.py`: http://bitbucket.org/hpk42/py-trunk/raw/caf81a2cdf5083be3bfb715d7e56273330a54843/py/test/plugin/pytest_pdb.py +.. _`pytest_pdb.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_pdb.py .. _`monkeypatch`: monkeypatch.html .. _`resultlog`: resultlog.html .. _`keyword`: keyword.html .. _`restdoc`: restdoc.html -.. _`pytest_unittest.py`: http://bitbucket.org/hpk42/py-trunk/raw/caf81a2cdf5083be3bfb715d7e56273330a54843/py/test/plugin/pytest_unittest.py -.. _`doctest`: doctest.html -.. _`pytest_resultlog.py`: http://bitbucket.org/hpk42/py-trunk/raw/caf81a2cdf5083be3bfb715d7e56273330a54843/py/test/plugin/pytest_resultlog.py +.. _`pytest_unittest.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_unittest.py +.. _`nose`: nose.html +.. _`pytest_resultlog.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.0.1/py/test/plugin/pytest_resultlog.py .. _`pdb`: pdb.html diff --git a/doc/test/plugin/monkeypatch.txt b/doc/test/plugin/monkeypatch.txt index b1f40a37c..0df8582f6 100644 --- a/doc/test/plugin/monkeypatch.txt +++ b/doc/test/plugin/monkeypatch.txt @@ -10,7 +10,7 @@ safely patch object attributes, dicts and environment variables. Usage ---------------- -Use the `monkeypatch funcarg`_ to safely patch the environment +Use the `monkeypatch funcarg`_ to safely patch environment variables, object attributes or dictionaries. For example, if you want to set the environment variable ``ENV1`` and patch the ``os.path.abspath`` function to return a particular value during a test @@ -26,7 +26,16 @@ function execution you can write it down like this: The function argument will do the modifications and memorize the old state. After the test function finished execution all modifications will be reverted. See the `monkeypatch blog post`_ -for an extensive discussion. +for an extensive discussion. + +To add to a possibly existing environment parameter you +can use this example: + +.. sourcecode:: python + + def test_mypath_finding(monkeypatch): + monkeypatch.setenv('PATH', 'x/y', prepend=":") + # x/y will be at the beginning of $PATH .. _`monkeypatch blog post`: http://tetamap.wordpress.com/2009/03/03/monkeypatching-in-unit-tests-done-right/ @@ -50,12 +59,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_monkeypatch.py`_ plugin source code 2. put it somewhere as ``pytest_monkeypatch.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/plugin/nose.txt b/doc/test/plugin/nose.txt index 1f5775a11..46ae3e290 100644 --- a/doc/test/plugin/nose.txt +++ b/doc/test/plugin/nose.txt @@ -53,12 +53,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_nose.py`_ plugin source code 2. put it somewhere as ``pytest_nose.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/plugin/pastebin.txt b/doc/test/plugin/pastebin.txt index ffbd125ae..ddaaadd78 100644 --- a/doc/test/plugin/pastebin.txt +++ b/doc/test/plugin/pastebin.txt @@ -14,7 +14,7 @@ Usage py.test --pastebin=failed -This will submit full failure information to a remote Paste service and +This will submit test run information to a remote Paste service and provide a URL for each failure. You may select tests as usual or add for example ``-x`` if you only want to send one particular failure. @@ -35,12 +35,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_pastebin.py`_ plugin source code 2. put it somewhere as ``pytest_pastebin.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/plugin/pdb.txt b/doc/test/plugin/pdb.txt index bea0f5084..e824ebe0d 100644 --- a/doc/test/plugin/pdb.txt +++ b/doc/test/plugin/pdb.txt @@ -20,12 +20,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_pdb.py`_ plugin source code 2. put it somewhere as ``pytest_pdb.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/plugin/recwarn.txt b/doc/test/plugin/recwarn.txt index 98915f9d0..e91ad7cd8 100644 --- a/doc/test/plugin/recwarn.txt +++ b/doc/test/plugin/recwarn.txt @@ -50,12 +50,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_recwarn.py`_ plugin source code 2. put it somewhere as ``pytest_recwarn.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/plugin/restdoc.txt b/doc/test/plugin/restdoc.txt index 1e9d21bb8..9f1e6c83e 100644 --- a/doc/test/plugin/restdoc.txt +++ b/doc/test/plugin/restdoc.txt @@ -24,12 +24,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_restdoc.py`_ plugin source code 2. put it somewhere as ``pytest_restdoc.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/plugin/resultlog.txt b/doc/test/plugin/resultlog.txt index 8cdfd32ee..d8f7741bf 100644 --- a/doc/test/plugin/resultlog.txt +++ b/doc/test/plugin/resultlog.txt @@ -20,12 +20,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_resultlog.py`_ plugin source code 2. put it somewhere as ``pytest_resultlog.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/plugin/terminal.txt b/doc/test/plugin/terminal.txt index 20f999153..59bbe047b 100644 --- a/doc/test/plugin/terminal.txt +++ b/doc/test/plugin/terminal.txt @@ -28,12 +28,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_terminal.py`_ plugin source code 2. put it somewhere as ``pytest_terminal.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/plugin/unittest.txt b/doc/test/plugin/unittest.txt index 68d245fca..a8ec3161c 100644 --- a/doc/test/plugin/unittest.txt +++ b/doc/test/plugin/unittest.txt @@ -23,12 +23,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_unittest.py`_ plugin source code 2. put it somewhere as ``pytest_unittest.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/plugin/xfail.txt b/doc/test/plugin/xfail.txt index 8e5dd72af..86be831bd 100644 --- a/doc/test/plugin/xfail.txt +++ b/doc/test/plugin/xfail.txt @@ -25,12 +25,10 @@ Start improving this plugin in 30 seconds ========================================= -Do you find the above documentation or the plugin itself lacking? - 1. Download `pytest_xfail.py`_ plugin source code 2. put it somewhere as ``pytest_xfail.py`` into your import path 3. a subsequent ``py.test`` run will use your local version -Further information: extend_ documentation, other plugins_ or contact_. +Checkout customize_, other plugins_ or `get in contact`_. .. include:: links.txt diff --git a/doc/test/quickstart.txt b/doc/test/quickstart.txt index da7e530fb..4efb9864b 100644 --- a/doc/test/quickstart.txt +++ b/doc/test/quickstart.txt @@ -7,27 +7,61 @@ Quickstart .. _here: ../download.html#no-setuptools -This document assumes basic python knowledge and a working `setuptools -installation`_ (otherwise see here_). You can install -the py lib and py.test by typing:: + +With a `setuptools installation`_ (otherwise see here_) you can type:: easy_install -U py -Now open a file ``test_sample.py`` file and put the following -example content into it:: +On Linux systems you may need to execute this as the superuser and +on Windows you might need to write down the full path to ``easy_install``. + +Now create a file ``test_sample.py`` with the following content: + +.. sourcecode:: python # content of test_sample.py + def func(x): + return x + 1 def test_answer(): - assert 42 == 43 + assert f(3) == 5 You can now run the test file like this:: - py.test test_sample.py + py.test test_sample.py -and will see an error report on the failing assert statement. -For further information please refer to `features`_ -or checkout the `tutorials`_ page for more introduction material. +and will see output like this: +.. sourcecode:: python + + =========================== test session starts ============================ + python: platform linux2 -- Python 2.6.2 + test object 1: test_sample.py + + test_sample.py F + + ================================= FAILURES ================================= + _______________________________ test_answer ________________________________ + + def test_answer(): + > assert func(3) == 5 + E assert 4 == 5 + E + where 4 = func(3) + + test_sample.py:6: AssertionError + ========================= 1 failed in 0.08 seconds ========================= + +This output contains Python interpreter information, a list of test objects, +a progress report and important details of the failure. + +**Where to go from here** + +`tutorials`_: a collection of starting points with code examples + +`features`_: overview and description of test features + +`contact`_: many ways for feedback and questions + +.. _`contact`: ../contact.html .. _`automatically collected`: features.html#autocollect .. _download: ../download.html .. _features: features.html diff --git a/doc/test/talks.txt b/doc/test/talks.txt index e6f2348e2..6d4ceb241 100644 --- a/doc/test/talks.txt +++ b/doc/test/talks.txt @@ -9,25 +9,27 @@ tutorial examples and blog postings function arguments: -- `application setup in test functions with funcargs`_ (doc link) +- `application setup in test functions with funcargs`_ +- `making funcargs dependendent on command line options`_ - `monkey patching done right`_ (blog post, consult `monkeypatch plugin`_ for actual 1.0 API) test parametrization: -- `generating parametrized tests with funcargs`_ (doc link) -- `parametrizing tests, generalized`_ (blog entry) -- `putting test-hooks into local or global plugins`_ (blog entry) +- `generating parametrized tests with funcargs`_ +- `parametrizing tests, generalized`_ (blog post) +- `putting test-hooks into local or global plugins`_ (blog post) distributed testing: - `simultanously test your code on all platforms`_ (blog entry) -plugins: +plugin specific examples: -- usage examples are in most of the referenced `plugins`_ docs +- `many examples in the docs for plugins`_ -.. _plugins: plugin/index.html +.. _`making funcargs dependendent on command line options`: funcargs.html#tut-cmdlineoption +.. _`many examples in the docs for plugins`: plugin/index.html .. _`monkeypatch plugin`: plugin/monkeypatch.html .. _`application setup in test functions with funcargs`: funcargs.html#appsetup .. _`simultanously test your code on all platforms`: http://tetamap.wordpress.com/2009/03/23/new-simultanously-test-your-code-on-all-platforms/ diff --git a/doc/test/test.html b/doc/test/test.html new file mode 100644 index 000000000..7d00f718a --- /dev/null +++ b/doc/test/test.html @@ -0,0 +1,18 @@ + + + + + + + + + + + diff --git a/doc/test/test.txt b/doc/test/test.txt deleted file mode 100644 index 115815f93..000000000 --- a/doc/test/test.txt +++ /dev/null @@ -1,38 +0,0 @@ -======================================= -py.test documentation index -======================================= - -the project independent ``py.test`` command line tool helps you to: - -* rapidly collect and run tests -* run unit- or doctests, functional or integration tests -* distribute tests to multiple environments -* use local or global plugins for custom test types and setup - -quickstart_: for getting started immediately. - -features_: a walk through basic features and usage. - -`talks, tutorials, examples`_: tutorial examples, slides - -`available plugins`_: list of py.test plugins - -funcargs_: powerful parametrized test function setup - -`distributed testing`_: distribute test runs to other machines and platforms. - -extend_: intro to extend and customize py.test runs - -config_: ``conftest.py`` files and the config object - - -.. _`available plugins`: plugin/index.html -.. _`talks, tutorials, examples`: talks.html -.. _quickstart: quickstart.html -.. _features: features.html -.. _funcargs: funcargs.html -.. _extend: extend.html -.. _config: config.html -.. _`distributed testing`: dist.html - - diff --git a/doc/test/xunit_setup.txt b/doc/test/xunit_setup.txt index aec020cb0..0a94794cb 100644 --- a/doc/test/xunit_setup.txt +++ b/doc/test/xunit_setup.txt @@ -7,12 +7,10 @@ xUnit style setup Note: - Since version 1.0 py.test offers funcargs_ for both - simple and complex test setup needs. Especially - for functional and integration, but also for unit testing, it is - highly recommended that you use this new method. + Since version 1.0 funcargs_ present the recommended way + to manage flexible and scalable test setups. -Python, Java and other languages have a tradition +Python, Java and many other languages have a tradition of using xUnit_ style testing. This typically involves the call of a ``setup`` method before a test function is run and ``teardown`` after diff --git a/doc/xml.txt b/doc/xml.txt index cf9ceeca0..ba4a2ef73 100644 --- a/doc/xml.txt +++ b/doc/xml.txt @@ -166,4 +166,4 @@ complete the probably request-specific serialization of your Tags. Hum, it's probably harder to explain this than to actually code it :-) -.. _`py.test`: test/test.html +.. _`py.test`: test/index.html diff --git a/example/assertion/failure_demo.py b/example/assertion/failure_demo.py index 0481c93dd..b2e3a4f03 100644 --- a/example/assertion/failure_demo.py +++ b/example/assertion/failure_demo.py @@ -11,6 +11,13 @@ def otherfunc_multi(a,b): assert (a == b) +def test_generative(param1, param2): + assert param1 * 2 < param2 + +def pytest_generate_tests(metafunc): + if 'param1' in metafunc.funcargnames: + metafunc.addcall(funcargs=dict(param1=3, param2=6)) + class TestFailing(object): def test_simple(self): def f(): @@ -96,14 +103,9 @@ class TestFailing(object): if namenotexi: pass - def test_generator(self): - yield None - def func1(self): assert 41 == 42 - def test_generator2(self): - yield self.func1 # thanks to Matthew Scott for this test def test_dynamic_compile_shows_nicely(): diff --git a/makepluginlist.py b/makepluginlist.py index c4bf6f2ba..dbd707f6d 100644 --- a/makepluginlist.py +++ b/makepluginlist.py @@ -194,24 +194,25 @@ class PluginDoc(RestWriter): # basename)) self.h1("Start improving this plugin in 30 seconds") self.para(py.code.Source(""" - Do you find the above documentation or the plugin itself lacking? - 1. Download `%s`_ plugin source code 2. put it somewhere as ``%s`` into your import path 3. a subsequent ``py.test`` run will use your local version - Further information: extend_ documentation, other plugins_ or contact_. + Checkout customize_, other plugins_ or `get in contact`_. """ % (basename, basename))) # your work appreciated if you offer back your version. In this case # it probably makes sense if you `checkout the py.test # development version`_ and apply your changes to the plugin # version in there. + #self.links.append((basename, + # "http://bitbucket.org/hpk42/py-trunk/raw/%s/" + # "py/test/plugin/%s" %(hg_changeset, basename))) self.links.append((basename, "http://bitbucket.org/hpk42/py-trunk/raw/%s/" - "py/test/plugin/%s" %(hg_changeset, basename))) - self.links.append(('extend', '../extend.html')) + "py/test/plugin/%s" %(pyversion, basename))) + self.links.append(('customize', '../customize.html')) self.links.append(('plugins', 'index.html')) - self.links.append(('contact', '../../contact.html')) + self.links.append(('get in contact', '../../contact.html')) self.links.append(('checkout the py.test development version', '../../download.html#checkout')) @@ -269,6 +270,7 @@ if __name__ == "__main__": _config.pluginmanager.do_configure(_config) pydir = py.path.local(py.__file__).dirpath() + pyversion = py.version cmd = "hg tip --template '{node}'" old = pydir.dirpath().chdir() diff --git a/py/__init__.py b/py/__init__.py index 09b6de283..e94966656 100644 --- a/py/__init__.py +++ b/py/__init__.py @@ -20,7 +20,7 @@ For questions please check out http://pylib.org/contact.html from initpkg import initpkg trunk = None -version = trunk or "1.0.x" +version = trunk or "1.0.1" initpkg(__name__, description = "py.test and pylib: advanced testing tool and networking lib", diff --git a/py/test/plugin/pytest_pastebin.py b/py/test/plugin/pytest_pastebin.py index 6ad8567fd..b9d67e86a 100644 --- a/py/test/plugin/pytest_pastebin.py +++ b/py/test/plugin/pytest_pastebin.py @@ -8,7 +8,7 @@ Usage py.test --pastebin=failed -This will submit full failure information to a remote Paste service and +This will submit test run information to a remote Paste service and provide a URL for each failure. You may select tests as usual or add for example ``-x`` if you only want to send one particular failure. diff --git a/py/test/plugin/pytest_terminal.py b/py/test/plugin/pytest_terminal.py index 8d905b4e0..203353174 100644 --- a/py/test/plugin/pytest_terminal.py +++ b/py/test/plugin/pytest_terminal.py @@ -228,9 +228,9 @@ class TerminalReporter: verinfo = ".".join(map(str, sys.version_info[:3])) msg = "python: platform %s -- Python %s" % (sys.platform, verinfo) - if self.config.option.verbose or self.config.option.debug: - msg += " -- " + str(sys.executable) + if self.config.option.verbose or self.config.option.debug or getattr(self.config.option, 'pastebin', None): msg += " -- pytest-%s" % (py.__version__) + msg += " -- " + str(sys.executable) self.write_line(msg) if self.config.option.debug or self.config.option.traceconfig: diff --git a/setup.py b/setup.py index 7fe1a3374..612037db7 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ def main(): name='py', description='py.test and pylib: advanced testing tool and networking lib', long_description = long_description, - version= trunk or '1.0.x', + version= trunk or '1.0.1', url='http://pylib.org', license='MIT license', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'],