diff --git a/doc/en/conf.py b/doc/en/conf.py index e4347a72b..b78b0bef6 100644 --- a/doc/en/conf.py +++ b/doc/en/conf.py @@ -17,7 +17,7 @@ # # The full version, including alpha/beta/rc tags. # The short X.Y version. -version = release = "2.3.3.2" +version = release = "2.3.3.3" import sys, os diff --git a/doc/en/example/simple.txt b/doc/en/example/simple.txt index f109bfffc..18dcf26a1 100644 --- a/doc/en/example/simple.txt +++ b/doc/en/example/simple.txt @@ -106,7 +106,7 @@ directory with the above conftest.py:: $ py.test =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev1 collected 0 items ============================= in 0.00 seconds ============================= @@ -150,12 +150,12 @@ and when running it will see a skipped "slow" test:: $ py.test -rs # "-rs" means report details on the little 's' =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev1 collected 2 items test_module.py .s ========================= short test summary info ========================== - SKIP [1] /tmp/doc-exec-140/conftest.py:9: need --runslow option to run + SKIP [1] /tmp/doc-exec-156/conftest.py:9: need --runslow option to run =================== 1 passed, 1 skipped in 0.01 seconds ==================== @@ -163,7 +163,7 @@ Or run it including the ``slow`` marked test:: $ py.test --runslow =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev1 collected 2 items test_module.py .. @@ -253,7 +253,7 @@ which will add the string to the test header accordingly:: $ py.test =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev1 project deps: mylib-1.1 collected 0 items @@ -276,7 +276,7 @@ which will add info only when run with "--v":: $ py.test -v =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 -- /home/hpk/p/pytest/.tox/regen/bin/python + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev1 -- /home/hpk/venv/0/bin/python info1: did you know that ... did you? collecting ... collected 0 items @@ -287,7 +287,7 @@ and nothing when run plainly:: $ py.test =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev1 collected 0 items ============================= in 0.00 seconds ============================= @@ -319,7 +319,7 @@ Now we can profile which test functions execute the slowest:: $ py.test --durations=3 =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev1 collected 3 items test_some_are_slow.py ... @@ -380,7 +380,7 @@ If we run this:: $ py.test -rx =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev1 collected 4 items test_step.py .Fx. @@ -388,7 +388,7 @@ If we run this:: ================================= FAILURES ================================= ____________________ TestUserHandling.test_modification ____________________ - self = + self = def test_modification(self): > assert 0 @@ -398,7 +398,7 @@ If we run this:: ========================= short test summary info ========================== XFAIL test_step.py::TestUserHandling::()::test_deletion reason: previous test failed (test_modification) - ============== 1 failed, 2 passed, 1 xfailed in 0.02 seconds =============== + ============== 1 failed, 2 passed, 1 xfailed in 0.01 seconds =============== We'll see that ``test_deletion`` was not executed because ``test_modification`` failed. It is reported as an "expected failure". @@ -450,42 +450,52 @@ We can run this:: $ py.test =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.3 - collected 3 items + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev1 + collected 7 items + test_step.py .Fx. a/test_db.py F a/test_db2.py F b/test_error.py E ================================== ERRORS ================================== _______________________ ERROR at setup of test_root ________________________ - file /tmp/doc-exec-133/b/test_error.py, line 1 + file /tmp/doc-exec-156/b/test_error.py, line 1 def test_root(db): # no db here, will error out fixture 'db' not found available fixtures: pytestconfig, recwarn, monkeypatch, capfd, capsys, tmpdir use 'py.test --fixtures [testpath]' for help on them. - /tmp/doc-exec-133/b/test_error.py:1 + /tmp/doc-exec-156/b/test_error.py:1 ================================= FAILURES ================================= + ____________________ TestUserHandling.test_modification ____________________ + + self = + + def test_modification(self): + > assert 0 + E assert 0 + + test_step.py:9: AssertionError _________________________________ test_a1 __________________________________ - db = + db = def test_a1(db): > assert 0, db # to show value - E AssertionError: + E AssertionError: a/test_db.py:2: AssertionError _________________________________ test_a2 __________________________________ - db = + db = def test_a2(db): > assert 0, db # to show value - E AssertionError: + E AssertionError: a/test_db2.py:2: AssertionError - ==================== 2 failed, 1 error in 0.02 seconds ===================== + ========== 3 failed, 2 passed, 1 xfailed, 1 error in 0.03 seconds ========== The two test modules in the ``a`` directory see the same ``db`` fixture instance while the one test in the sister-directory ``b`` doesn't see it. We could of course @@ -495,3 +505,77 @@ it (unless you use "autouse" fixture which are always executed ahead of the firs executing). +post-process test reports / failures +--------------------------------------- + +If you want to postprocess test reports and need access to the executing +environment you can implement a hook that gets called when the test +"report" object is about to be created. Here we write out all failing +test calls and also access a fixture (if it was used by the test) in +case you want to query/look at it during your post processing. In our +case we just write some informations out to a ``failures`` file:: + + # content of conftest.py + + import pytest + import os.path + + @pytest.mark.tryfirst + def pytest_runtest_makereport(item, call, __multicall__): + # execute all other hooks to obtain the report object + rep = __multicall__.execute() + + # we only look at actual failing test calls, not setup/teardown + if rep.when == "call" and rep.failed: + mode = "a" if os.path.exists("failures") else "w" + with open("failures", mode) as f: + # let's also access a fixture for the fun of it + if "tmpdir" in item.funcargs: + extra = " (%s)" % item.funcargs["tmpdir"] + else: + extra = "" + + f.write(rep.nodeid + extra + "\n") + return rep + +if you then have failing tests:: + + # content of test_module.py + def test_fail1(tmpdir): + assert 0 + def test_fail2(): + assert 0 + +and run them:: + + $ py.test test_module.py + =========================== test session starts ============================ + platform linux2 -- Python 2.7.3 -- pytest-2.3.4.dev1 + collected 2 items + + test_module.py FF + + ================================= FAILURES ================================= + ________________________________ test_fail1 ________________________________ + + tmpdir = local('/tmp/pytest-3/test_fail10') + + def test_fail1(tmpdir): + > assert 0 + E assert 0 + + test_module.py:2: AssertionError + ________________________________ test_fail2 ________________________________ + + def test_fail2(): + > assert 0 + E assert 0 + + test_module.py:4: AssertionError + ========================= 2 failed in 0.01 seconds ========================= + +you will have a "failures" file which contains the failing test ids:: + + $ cat failures + test_module.py::test_fail1 (/tmp/pytest-3/test_fail10) + test_module.py::test_fail2