From 66ed2d123a61f58c9c4bb0085456361ecb16876c Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sat, 14 Jul 2012 12:06:58 +0200 Subject: [PATCH] add a little example on how to group test execution by parametrized resource --- doc/en/Makefile | 2 +- doc/en/example/parametrize.txt | 124 ++++++++++++++++++++++++++------- 2 files changed, 99 insertions(+), 27 deletions(-) diff --git a/doc/en/Makefile b/doc/en/Makefile index 4223b770e..9de7a8ba8 100644 --- a/doc/en/Makefile +++ b/doc/en/Makefile @@ -40,7 +40,7 @@ clean: -rm -rf $(BUILDDIR)/* install: html - rsync -avz _build/html/ pytest.org:/www/pytest.org/latest + rsync -avz _build/html/ pytest.org:/www/pytest.org/dev installpdf: latexpdf @scp $(BUILDDIR)/latex/pytest.pdf pytest.org:/www/pytest.org/latest diff --git a/doc/en/example/parametrize.txt b/doc/en/example/parametrize.txt index e4173a346..942867dbe 100644 --- a/doc/en/example/parametrize.txt +++ b/doc/en/example/parametrize.txt @@ -38,8 +38,8 @@ function is called three times. Let's run it:: $ py.test -q collecting ... collected 3 items ..F - ================================= FAILURES ================================= - ____________________________ test_eval[6*9-42] _____________________________ + =================================== FAILURES =================================== + ______________________________ test_eval[6*9-42] _______________________________ input = '6*9', expected = 42 @@ -54,7 +54,7 @@ function is called three times. Let's run it:: E + where 54 = eval('6*9') test_expectation.py:8: AssertionError - 1 failed, 2 passed in 0.01 seconds + 1 failed, 2 passed in 0.02 seconds As expected only one pair of input/output values fails the simple test function. @@ -96,7 +96,7 @@ This means that we only run 2 tests if we do not pass ``--all``:: $ py.test -q test_compute.py collecting ... collected 2 items .. - 2 passed in 0.01 seconds + 2 passed in 0.02 seconds We run only two computations, so we see two dots. let's run the full monty:: @@ -104,8 +104,8 @@ let's run the full monty:: $ py.test -q --all collecting ... collected 5 items ....F - ================================= FAILURES ================================= - _____________________________ test_compute[4] ______________________________ + =================================== FAILURES =================================== + _______________________________ test_compute[4] ________________________________ param1 = 4 @@ -153,20 +153,22 @@ only have to work a bit to construct the correct arguments for pytest's this is a fully self-contained example which you can run with:: $ py.test test_scenarios.py - =========================== test session starts ============================ - platform linux2 -- Python 2.7.1 -- pytest-2.2.4 + ============================= test session starts ============================== + platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev2 + plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov collecting ... collected 2 items test_scenarios.py .. - ========================= 2 passed in 0.01 seconds ========================= + =========================== 2 passed in 0.02 seconds =========================== If you just collect tests you'll also nicely see 'advanced' and 'basic' as variants for the test function:: $ py.test --collectonly test_scenarios.py - =========================== test session starts ============================ - platform linux2 -- Python 2.7.1 -- pytest-2.2.4 + ============================= test session starts ============================== + platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev2 + plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov collecting ... collected 2 items @@ -174,7 +176,7 @@ If you just collect tests you'll also nicely see 'advanced' and 'basic' as varia - ============================= in 0.00 seconds ============================= + =============================== in 0.01 seconds =============================== Deferring the setup of parametrized resources --------------------------------------------------- @@ -221,24 +223,25 @@ creates a database object for the actual test invocations:: Let's first see how it looks like at collection time:: $ py.test test_backends.py --collectonly - =========================== test session starts ============================ - platform linux2 -- Python 2.7.1 -- pytest-2.2.4 + ============================= test session starts ============================== + platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev2 + plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov collecting ... collected 2 items - ============================= in 0.00 seconds ============================= + =============================== in 0.01 seconds =============================== And then when we run the test:: $ py.test -q test_backends.py collecting ... collected 2 items .F - ================================= FAILURES ================================= - _________________________ test_db_initialized[d2] __________________________ + =================================== FAILURES =================================== + ___________________________ test_db_initialized[d2] ____________________________ - db = + db = def test_db_initialized(db): # a dummy test @@ -247,7 +250,7 @@ And then when we run the test:: E Failed: deliberately failing for demo purposes test_backends.py:6: Failed - 1 failed, 1 passed in 0.01 seconds + 1 failed, 1 passed in 0.02 seconds The first invocation with ``db == "DB1"`` passed while the second with ``db == "DB2"`` failed. Our ``pytest_funcarg__db`` factory has instantiated each of the DB values during the setup phase while the ``pytest_generate_tests`` generated two according calls to the ``test_db_initialized`` during the collection phase. @@ -292,17 +295,17 @@ argument sets to use for each test function. Let's run it:: $ py.test -q collecting ... collected 3 items F.. - ================================= FAILURES ================================= - ________________________ TestClass.test_equals[1-2] ________________________ + =================================== FAILURES =================================== + __________________________ TestClass.test_equals[1-2] __________________________ - self = , a = 1, b = 2 + self = , a = 1, b = 2 def test_equals(self, a, b): > assert a == b E assert 1 == 2 test_parametrize.py:18: AssertionError - 1 failed, 2 passed in 0.01 seconds + 1 failed, 2 passed in 0.02 seconds Indirect parametrization with multiple resources -------------------------------------------------------------- @@ -323,6 +326,75 @@ Running it results in some skips if we don't have all the python interpreters in . $ py.test -rs -q multipython.py collecting ... collected 75 items ............sss............sss............sss............ssssssssssssssssss - ========================= short test summary info ========================== - SKIP [27] /home/hpk/p/pytest/doc/example/multipython.py:36: 'python2.8' not found - 48 passed, 27 skipped in 1.71 seconds + =========================== short test summary info ============================ + SKIP [27] /home/hpk/p/pytest/doc/en/example/multipython.py:36: 'python2.8' not found + 48 passed, 27 skipped in 1.89 seconds + +.. regendoc:wipe + +Grouping test execution by parameter +----------------------------------------- + +By default pytest will execute test functions by executing all its parametrized invocations. If you rather want to group execution by parameter, you can +use something like the following ``conftest.py`` example. It uses +a parametrized "session" object:: + + # content of conftest.py + def pytest_collection_modifyitems(items): + def cmp(item1, item2): + param1 = item1.callspec.getparam("session") + param2 = item2.callspec.getparam("session") + if param1 < param2: + return -1 + elif param1 > param2: + return 1 + return 0 + items.sort(cmp=cmp) + + def pytest_generate_tests(metafunc): + if "session" in metafunc.funcargnames: + metafunc.parametrize("session", [1,2], indirect=True) + + class Session: + def __init__(self, num): + self.num = num + + def pytest_funcarg__session(request): + return Session(request.param) + + +If you know have a test file like this:: + + # content of test_session.py + def test_hello(session): + pass + + def test_world(session): + pass + + class TestClass: + def test_method1(self, session): + pass + def test_method2(self, session): + pass + +then a subsequent execution will order the running of tests by +parameter value:: + + $ py.test -v + ============================= test session starts ============================== + platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev2 -- /home/hpk/venv/1/bin/python + cachedir: /home/hpk/tmp/doc-exec-313/.cache + plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov + collecting ... collected 8 items + + test_session.py:1: test_hello[1] PASSED + test_session.py:4: test_world[1] PASSED + test_session.py:8: TestClass.test_method1[1] PASSED + test_session.py:10: TestClass.test_method2[1] PASSED + test_session.py:1: test_hello[2] PASSED + test_session.py:4: test_world[2] PASSED + test_session.py:8: TestClass.test_method1[2] PASSED + test_session.py:10: TestClass.test_method2[2] PASSED + + =========================== 8 passed in 0.02 seconds ===========================