diff --git a/doc/en/Makefile b/doc/en/Makefile index 78fa53233..c6ffc0ec9 100644 --- a/doc/en/Makefile +++ b/doc/en/Makefile @@ -12,7 +12,7 @@ PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -SITETARGET=latest +SITETARGET=dev .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest diff --git a/doc/en/example/markers.txt b/doc/en/example/markers.txt index 6b2adc520..8bf353577 100644 --- a/doc/en/example/markers.txt +++ b/doc/en/example/markers.txt @@ -19,6 +19,8 @@ You can "mark" a test function with custom metadata like this:: pass # perform some webtest test for your app def test_something_quick(): pass + def test_another(): + pass .. versionadded:: 2.2 @@ -26,25 +28,82 @@ You can then restrict a test run to only run tests marked with ``webtest``:: $ py.test -v -m webtest =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 -- /home/hpk/p/pytest/.tox/regen/bin/python - collecting ... collected 2 items + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev3 -- /home/hpk/venv/1/bin/python + cachedir: /tmp/doc-exec-196/.cache + plugins: pep8, cache, xdist + collecting ... collected 3 items test_server.py:3: test_send_http PASSED - =================== 1 tests deselected by "-m 'webtest'" =================== - ================== 1 passed, 1 deselected in 0.01 seconds ================== + =================== 2 tests deselected by "-m 'webtest'" =================== + ================== 1 passed, 2 deselected in 0.01 seconds ================== Or the inverse, running all tests except the webtest ones:: $ py.test -v -m "not webtest" =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 -- /home/hpk/p/pytest/.tox/regen/bin/python - collecting ... collected 2 items + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev3 -- /home/hpk/venv/1/bin/python + cachedir: /tmp/doc-exec-196/.cache + plugins: pep8, cache, xdist + collecting ... collected 3 items test_server.py:6: test_something_quick PASSED + test_server.py:8: test_another PASSED ================= 1 tests deselected by "-m 'not webtest'" ================= - ================== 1 passed, 1 deselected in 0.01 seconds ================== + ================== 2 passed, 1 deselected in 0.02 seconds ================== + +Using ``-k expr`` to select tests based on their name +------------------------------------------------------- + +.. versionadded: 2.0/2.3.4 + +You can use the ``-k`` command line option to specify an expression +which implements a substring match on the test names instead of the +exact match on markers that ``-m`` provides. This makes it easy to +select tests based on their names:: + + $ py.test -v -k http # running with the above defined example module + =========================== test session starts ============================ + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev3 -- /home/hpk/venv/1/bin/python + cachedir: /tmp/doc-exec-196/.cache + plugins: pep8, cache, xdist + collecting ... collected 3 items + + test_server.py:3: test_send_http PASSED + + =================== 2 tests deselected by '-ksend_http' ==================== + ================== 1 passed, 2 deselected in 0.01 seconds ================== + +And you can also run all tests except the ones that match the keyword:: + + $ py.test -k "not send_http" -v + =========================== test session starts ============================ + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev3 -- /home/hpk/venv/1/bin/python + cachedir: /tmp/doc-exec-196/.cache + plugins: pep8, cache, xdist + collecting ... collected 3 items + + test_server.py:6: test_something_quick PASSED + test_server.py:8: test_another PASSED + + ================= 1 tests deselected by '-knot send_http' ================== + ================== 2 passed, 1 deselected in 0.01 seconds ================== + +Or to select "http" and "quick" tests:: + + $ py.test -k "http or quick" -v + =========================== test session starts ============================ + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev3 -- /home/hpk/venv/1/bin/python + cachedir: /tmp/doc-exec-196/.cache + plugins: pep8, cache, xdist + collecting ... collected 3 items + + test_server.py:3: test_send_http PASSED + test_server.py:6: test_something_quick PASSED + + ================= 1 tests deselected by '-khttp or quick' ================== + ================== 2 passed, 1 deselected in 0.01 seconds ================== Registering markers ------------------------------------- @@ -137,46 +196,6 @@ in which case it will be applied to all functions and methods defined in the module. -Using ``-k TEXT`` to select tests ----------------------------------------------------- - -You can use the ``-k`` command line option to only run tests with names matching -the given argument:: - - $ py.test -k send_http # running with the above defined examples - =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 - collected 4 items - - test_server.py . - - =================== 3 tests deselected by '-ksend_http' ==================== - ================== 1 passed, 3 deselected in 0.01 seconds ================== - -And you can also run all tests except the ones that match the keyword:: - - $ py.test -k-send_http - =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 - collected 4 items - - test_mark_classlevel.py .. - test_server.py . - - =================== 1 tests deselected by '-k-send_http' =================== - ================== 3 passed, 1 deselected in 0.01 seconds ================== - -Or to only select the class:: - - $ py.test -kTestClass - =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 - collected 4 items - - test_mark_classlevel.py .. - - =================== 2 tests deselected by '-kTestClass' ==================== - ================== 2 passed, 2 deselected in 0.01 seconds ================== .. _`adding a custom marker from a plugin`: @@ -223,7 +242,8 @@ the test needs:: $ py.test -E stage2 =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev3 + plugins: pep8, cache, xdist collected 1 items test_someenv.py s @@ -234,7 +254,8 @@ and here is one that specifies exactly the environment needed:: $ py.test -E stage1 =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev3 + plugins: pep8, cache, xdist collected 1 items test_someenv.py . @@ -351,25 +372,112 @@ then you will see two test skipped and two executed tests as expected:: $ py.test -rs # this option reports skip reasons =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev3 + plugins: pep8, cache, xdist collected 4 items test_plat.py s.s. ========================= short test summary info ========================== - SKIP [2] /tmp/doc-exec-135/conftest.py:12: cannot run on platform linux2 + SKIP [2] /tmp/doc-exec-196/conftest.py:12: cannot run on platform linux2 - =================== 2 passed, 2 skipped in 0.01 seconds ==================== + =================== 2 passed, 2 skipped in 0.02 seconds ==================== Note that if you specify a platform via the marker-command line option like this:: $ py.test -m linux2 =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev3 + plugins: pep8, cache, xdist collected 4 items test_plat.py . =================== 3 tests deselected by "-m 'linux2'" ==================== - ================== 1 passed, 3 deselected in 0.01 seconds ================== + ================== 1 passed, 3 deselected in 0.02 seconds ================== then the unmarked-tests will not be run. It is thus a way to restrict the run to the specific tests. + +Automatically adding markers based on test names +-------------------------------------------------------- + +.. regendoc:wipe + +If you a test suite where test function names indicate a certain +type of test, you can implement a hook that automatically defines +markers so that you can use the ``-m`` option with it. Let's look +at this test module:: + + # content of test_module.py + + def test_interface_simple(): + assert 0 + + def test_interface_complex(): + assert 0 + + def test_event_simple(): + assert 0 + + def test_something_else(): + assert 0 + +We want to dynamically define two markers and can do it in a +``conftest.py`` plugin:: + + # content of conftest.py + + import pytest + def pytest_collection_modifyitems(items): + for item in items: + if "interface" in item.nodeid: + item.keywords["interface"] = pytest.mark.interface + elif "event" in item.nodeid: + item.keywords["event"] = pytest.mark.event + +We can now use the ``-m option`` to select one set:: + + $ py.test -m interface --tb=short + =========================== test session starts ============================ + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev3 + plugins: pep8, cache, xdist + collected 4 items + + test_module.py FF + + ================================= FAILURES ================================= + __________________________ test_interface_simple ___________________________ + test_module.py:3: in test_interface_simple + > assert 0 + E assert 0 + __________________________ test_interface_complex __________________________ + test_module.py:6: in test_interface_complex + > assert 0 + E assert 0 + ================== 2 tests deselected by "-m 'interface'" ================== + ================== 2 failed, 2 deselected in 0.02 seconds ================== + +or to select both "event" and "interface" tests:: + + $ py.test -m "interface or event" --tb=short + =========================== test session starts ============================ + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev3 + plugins: pep8, cache, xdist + collected 4 items + + test_module.py FFF + + ================================= FAILURES ================================= + __________________________ test_interface_simple ___________________________ + test_module.py:3: in test_interface_simple + > assert 0 + E assert 0 + __________________________ test_interface_complex __________________________ + test_module.py:6: in test_interface_complex + > assert 0 + E assert 0 + ____________________________ test_event_simple _____________________________ + test_module.py:9: in test_event_simple + > assert 0 + E assert 0 + ============= 1 tests deselected by "-m 'interface or event'" ============== + ================== 3 failed, 1 deselected in 0.02 seconds ==================