From 73d99008449536b0e170d5eca08ccbc889a5d1d0 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 26 May 2010 14:48:27 +0200 Subject: [PATCH] add Meme's pytest-cov plugin to the plugin index page --HG-- branch : trunk --- bin-for-dist/makepluginlist.py | 3 +- doc/test/plugin/cov.txt | 250 +++++++++++++++++++++++++++++++++ doc/test/plugin/index.txt | 2 + doc/test/plugin/links.txt | 1 + 4 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 doc/test/plugin/cov.txt diff --git a/bin-for-dist/makepluginlist.py b/bin-for-dist/makepluginlist.py index 321a0f362..698c69193 100644 --- a/bin-for-dist/makepluginlist.py +++ b/bin-for-dist/makepluginlist.py @@ -5,7 +5,7 @@ WIDTH = 75 plugins = [ ('advanced python testing', 'skipping mark pdb figleaf ' - 'monkeypatch coverage capture capturelog recwarn tmpdir',), + 'monkeypatch coverage cov capture capturelog recwarn tmpdir',), ('distributed testing, CI and deployment', 'xdist pastebin junitxml resultlog genscript',), ('testing domains and conventions codecheckers', @@ -24,6 +24,7 @@ externals = { 'figleaf': None, 'capturelog': None, 'coverage': None, + 'cov': None, 'codecheckers': None, 'django': "for testing django applications", } diff --git a/doc/test/plugin/cov.txt b/doc/test/plugin/cov.txt new file mode 100644 index 000000000..4b91fdb55 --- /dev/null +++ b/doc/test/plugin/cov.txt @@ -0,0 +1,250 @@ + +produce code coverage reports using the 'coverage' package, including support for distributed testing. +====================================================================================================== + + +.. contents:: + :local: + +This plugin produces coverage reports using the coverage package. It +supports centralised testing and distributed testing in both load and +each modes. + +All features offered by the coverage package should be available, +either through this plugin or through coverage's own config file. + + +Installation +------------ + +The `pytest-cov pypi`_ package may be installed / uninstalled with pip:: + + pip install pytest-cov + pip uninstall pytest-cov + +Alternatively easy_install can be used:: + + easy_install pytest-cov + +.. _`pytest-cov pypi`: http://pypi.python.org/pypi/pytest-cov/ + + +Usage +----- + +Centralised Testing +~~~~~~~~~~~~~~~~~~~ + +Running centralised testing:: + + py.test --cov myproj tests/ + +Shows a terminal report:: + + -------------------- coverage: platform linux2, python 2.6.4-final-0 --------------------- + Name Stmts Exec Cover Missing + -------------------------------------------------- + myproj/__init__ 2 2 100% + myproj/myproj 257 244 94% 24-26, 99, 149, 233-236, 297-298, 369-370 + myproj/feature4286 94 87 92% 183-188, 197 + -------------------------------------------------- + TOTAL 353 333 94% + + +Distributed Testing +~~~~~~~~~~~~~~~~~~~ + +Distributed testing with dist mode set to load:: + + py.test --cov myproj -n 2 tests/ + +The results from the slaves will be combined like so:: + + -------------------- coverage: platform linux2, python 2.6.4-final-0 --------------------- + Name Stmts Exec Cover Missing + -------------------------------------------------- + myproj/__init__ 2 2 100% + myproj/myproj 257 244 94% 24-26, 99, 149, 233-236, 297-298, 369-370 + myproj/feature4286 94 87 92% 183-188, 197 + -------------------------------------------------- + TOTAL 353 333 94% + + +Distributed testing in each mode:: + + py.test --cov myproj --dist=each + --tx=popen//python=/usr/local/python265/bin/python + --tx=popen//python=/usr/local/python27b1/bin/python + tests/ + +Will produce a report for each slave:: + + -------------------- coverage: platform linux2, python 2.6.5-final-0 --------------------- + Name Stmts Exec Cover Missing + -------------------------------------------------- + myproj/__init__ 2 2 100% + myproj/myproj 257 244 94% 24-26, 99, 149, 233-236, 297-298, 369-370 + myproj/feature4286 94 87 92% 183-188, 197 + -------------------------------------------------- + TOTAL 353 333 94% + --------------------- coverage: platform linux2, python 2.7.0-beta-1 --------------------- + Name Stmts Exec Cover Missing + -------------------------------------------------- + myproj/__init__ 2 2 100% + myproj/myproj 257 244 94% 24-26, 99, 149, 233-236, 297-298, 369-370 + myproj/feature4286 94 87 92% 183-188, 197 + -------------------------------------------------- + TOTAL 353 333 94% + + +Distributed testing in each mode can also produce a single combined +report. This is useful to get coverage information spanning things +such as all python versions:: + + py.test --cov myproj --cov-combine-each --dist=each + --tx=popen//python=/usr/local/python265/bin/python + --tx=popen//python=/usr/local/python27b1/bin/python + tests/ + +Which looks like:: + + ---------------------------------------- coverage ---------------------------------------- + platform linux2, python 2.6.5-final-0 + platform linux2, python 2.7.0-beta-1 + Name Stmts Exec Cover Missing + -------------------------------------------------- + myproj/__init__ 2 2 100% + myproj/myproj 257 244 94% 24-26, 99, 149, 233-236, 297-298, 369-370 + myproj/feature4286 94 87 92% 183-188, 197 + -------------------------------------------------- + TOTAL 353 333 94% + + +Reporting +--------- + +By default a terminal report is output. This report can be disabled +if desired, such as when results are going to a continuous integration +system and the terminal output won't be seen. + +In addition and without rerunning tests it is possible to generate +annotated source code, a html report and an xml report. + +The directories for annotated source code and html reports can be +specified as can the file name for the xml report. + +Since testing often takes a non trivial amount of time at the end of +testing any / all of the reports may be generated. + + +Coverage Data File +------------------ + +During testing there may be many data files with coverage data. These +will have unique suffixes and will be combined at the end of testing. + +Upon completion, for --dist=load (and also for --dist=each when the +--cov-combine-each option is used) there will only be one data file. + +For --dist=each there may be many data files where each one will have +the platform / python version info appended to the name. + +These data files are left at the end of testing so that it is possible +to use normal coverage tools to examine them. + +At the beginning of testing any data files that are about to be used +will first be erased so ensure the data is clean for each test run. + +It is possible to set the name of the data file. If needed the +platform / python version will be appended automatically to this name. + + +Coverage Config File +-------------------- + +Coverage by default will read its own config file. An alternative +file name may be specified or reading config can be disabled entirely. + +Care has been taken to ensure that the coverage env vars and config +file options work the same under this plugin as they do under coverage +itself. + +Since options may be specified in different ways the order of +precedence between pytest-cov and coverage from highest to lowest is: + +1. pytest command line +2. pytest env var +3. pytest conftest +4. coverage env var +5. coverage config file +6. coverage default + + +Limitations +----------- + +For distributed testing the slaves must have the pytest-cov package +installed. This is needed since the plugin must be registered through +setuptools / distribute for pytest to start the plugin on the slave. + + +Acknowledgements +---------------- + +Holger Krekel for pytest with its distributed testing support. + +Ned Batchelder for coverage and its ability to combine the coverage +results of parallel runs. + +Whilst this plugin has been built fresh from the ground up to support +distributed testing it has been influenced by the work done on +pytest-coverage (Ross Lawley, James Mills, Holger Krekel) and +nose-cover (Jason Pellerin) which are other coverage plugins for +pytest and nose respectively. + +No doubt others have contributed to these tools as well. + +command line options +-------------------- + + +``--cov-on`` + enable coverage, only needed if not specifying any --cov options +``--cov=package`` + collect coverage for the specified package (multi-allowed) +``--cov-no-terminal`` + disable printing a report on the terminal +``--cov-annotate`` + generate an annotated source code report +``--cov-html`` + generate a html report +``--cov-xml`` + generate an xml report +``--cov-annotate-dir=dir`` + directory for the annotate report, default: %default +``--cov-html-dir=dir`` + directory for the html report, default: coverage_html +``--cov-xml-file=path`` + file for the xml report, default: coverage.xml +``--cov-data-file=path`` + file containing coverage data, default: .coverage +``--cov-combine-each`` + for dist=each mode produce a single combined report +``--cov-branch`` + enable branch coverage +``--cov-pylib`` + enable python library coverage +``--cov-timid`` + enable slower and simpler tracing +``--cov-no-missing-lines`` + disable showing missing lines, only relevant to the terminal report +``--cov-no-missing-files`` + disable showing message about missing source files +``--cov-omit=prefix1,prefix2,...`` + ignore files with these prefixes +``--cov-no-config`` + disable coverage reading its config file +``--cov-config-file=path`` + config file for coverage, default: %default + +.. include:: links.txt diff --git a/doc/test/plugin/index.txt b/doc/test/plugin/index.txt index e309b65b7..853a4dce6 100644 --- a/doc/test/plugin/index.txt +++ b/doc/test/plugin/index.txt @@ -14,6 +14,8 @@ monkeypatch_ safely patch object attributes, dicts and environment variables. coverage_ (external) Write and report coverage data with the 'coverage' package. +cov_ (external) produce code coverage reports using the 'coverage' package, including support for distributed testing. + capture_ configurable per-test stdout/stderr capturing mechanisms. capturelog_ (external) capture output of logging module. diff --git a/doc/test/plugin/links.txt b/doc/test/plugin/links.txt index 11e55618d..0f3d88e1d 100644 --- a/doc/test/plugin/links.txt +++ b/doc/test/plugin/links.txt @@ -38,6 +38,7 @@ .. _`monkeypatch`: monkeypatch.html .. _`coverage`: coverage.html .. _`resultlog`: resultlog.html +.. _`cov`: cov.html .. _`pytest_junitxml.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.3.1/py/_plugin/pytest_junitxml.py .. _`django`: django.html .. _`pytest_unittest.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.3.1/py/_plugin/pytest_unittest.py