From 738f14a48a305921b2f812310f894fd1b2a8a038 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 21 Sep 2012 09:39:54 +0200 Subject: [PATCH] improve the parametrization scenario example to sort by id, rather than by file-order, see also: http://stackoverflow.com/questions/12521924/pytest-running-scenarios-in-the-correct-order-in-the-class --- CHANGELOG | 2 ++ _pytest/__init__.py | 2 +- _pytest/python.py | 4 +++ doc/en/example/parametrize.txt | 53 ++++++++++++++++++++-------------- setup.py | 2 +- testing/test_python.py | 38 ++++++++++++++++++++++++ 6 files changed, 77 insertions(+), 24 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 5a2ea45c9..e4b7c3b73 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,6 +6,8 @@ Changes between 2.2.4 and 2.3.0.dev node.markers allows reading and manipulating of MarkInfo objects previously attached with @pytest.mark.* or request.applymarker or setattr(node.markers, name, pytest.mark.*) calls. +- introduce re-ordering of tests by resource and parametrization setup + which takes precedence to the usual file-ordering - fix issue185 monkeypatching time.time does not cause pytest to fail - fix issue172 duplicate call of pytest.setup-decoratored setup_module functions diff --git a/_pytest/__init__.py b/_pytest/__init__.py index 0f7edfaf7..4e0bab1cd 100644 --- a/_pytest/__init__.py +++ b/_pytest/__init__.py @@ -1,2 +1,2 @@ # -__version__ = '2.3.0.dev14' +__version__ = '2.3.0.dev15' diff --git a/_pytest/python.py b/_pytest/python.py index d77b35768..77d91efae 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -574,6 +574,10 @@ class CallSpec2(object): for arg,val in zip(argnames, valset): self._checkargnotcontained(arg) getattr(self, valtype)[arg] = val + # we want self.params to be always set because of + # parametrize_sorted() which groups tests by params/scope + if valtype == "funcargs": + self.params[arg] = id self._arg2scopenum[arg] = scopenum self._idlist.append(id) diff --git a/doc/en/example/parametrize.txt b/doc/en/example/parametrize.txt index 6512d8b70..cc43ac419 100644 --- a/doc/en/example/parametrize.txt +++ b/doc/en/example/parametrize.txt @@ -96,7 +96,7 @@ This means that we only run 2 tests if we do not pass ``--all``:: $ py.test -q test_compute.py collecting ... collected 2 items .. - 2 passed in 0.02 seconds + 2 passed in 0.01 seconds We run only two computations, so we see two dots. let's run the full monty:: @@ -139,7 +139,7 @@ only have to work a bit to construct the correct arguments for pytest's items = scenario[1].items() argnames = [x[0] for x in items] argvalues.append(([x[1] for x in items])) - metafunc.parametrize(argnames, argvalues, ids=idlist) + metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class") scenario1 = ('basic', {'attribute': 'value'}) scenario2 = ('advanced', {'attribute': 'value2'}) @@ -147,36 +147,45 @@ only have to work a bit to construct the correct arguments for pytest's class TestSampleWithScenarios: scenarios = [scenario1, scenario2] - def test_demo(self, attribute): + def test_demo1(self, attribute): + assert isinstance(attribute, str) + + def test_demo2(self, attribute): assert isinstance(attribute, str) this is a fully self-contained example which you can run with:: $ py.test test_scenarios.py =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev3 - plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov - collecting ... collected 2 items + platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev14 + plugins: xdist, bugzilla, cache, oejskit, cli, timeout, pep8, cov + collecting ... collected 4 items - test_scenarios.py .. + test_scenarios.py .... - ========================= 2 passed in 0.02 seconds ========================= + ========================= 4 passed in 0.02 seconds ========================= If you just collect tests you'll also nicely see 'advanced' and 'basic' as variants for the test function:: $ py.test --collectonly test_scenarios.py =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev3 - plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov - collecting ... collected 2 items + platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev14 + plugins: xdist, bugzilla, cache, oejskit, cli, timeout, pep8, cov + collecting ... collected 4 items - - + + + + - ============================= in 0.02 seconds ============================= + ============================= in 0.01 seconds ============================= + +Note that we told ``metafunc.parametrize()`` that your scenario values +should be considered class-scoped. With pytest-2.3 this leads to a +resource-based ordering. Deferring the setup of parametrized resources --------------------------------------------------- @@ -224,14 +233,14 @@ Let's first see how it looks like at collection time:: $ py.test test_backends.py --collectonly =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev3 - plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov + platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev14 + plugins: xdist, bugzilla, cache, oejskit, cli, timeout, pep8, cov collecting ... collected 2 items - ============================= in 0.02 seconds ============================= + ============================= in 0.01 seconds ============================= And then when we run the test:: @@ -241,7 +250,7 @@ And then when we run the test:: ================================= FAILURES ================================= _________________________ test_db_initialized[d2] __________________________ - db = + db = def test_db_initialized(db): # a dummy test @@ -250,7 +259,7 @@ And then when we run the test:: E Failed: deliberately failing for demo purposes test_backends.py:6: Failed - 1 failed, 1 passed in 0.02 seconds + 1 failed, 1 passed in 0.01 seconds The first invocation with ``db == "DB1"`` passed while the second with ``db == "DB2"`` failed. Our ``pytest_funcarg__db`` factory has instantiated each of the DB values during the setup phase while the ``pytest_generate_tests`` generated two according calls to the ``test_db_initialized`` during the collection phase. @@ -298,7 +307,7 @@ argument sets to use for each test function. Let's run it:: ================================= FAILURES ================================= ________________________ TestClass.test_equals[1-2] ________________________ - self = , a = 1, b = 2 + self = , a = 1, b = 2 def test_equals(self, a, b): > assert a == b @@ -327,5 +336,5 @@ Running it results in some skips if we don't have all the python interpreters in collecting ... collected 75 items ............sss............sss............sss............ssssssssssssssssss ========================= short test summary info ========================== - SKIP [27] /home/hpk/p/pytest/doc/en/example/multipython.py:36: 'python2.8' not found - 48 passed, 27 skipped in 1.70 seconds + SKIP [27] /home/hpk/p/pytest/doc/en/example/multipython.py:21: 'python2.8' not found + 48 passed, 27 skipped in 3.11 seconds diff --git a/setup.py b/setup.py index 0db860a99..8f32832c0 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def main(): name='pytest', description='py.test: simple powerful testing with Python', long_description = long_description, - version='2.3.0.dev14', + version='2.3.0.dev15', url='http://pytest.org', license='MIT license', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], diff --git a/testing/test_python.py b/testing/test_python.py index 9a159c847..21c95a75c 100644 --- a/testing/test_python.py +++ b/testing/test_python.py @@ -1095,6 +1095,44 @@ class TestMetafunc: "*6 fail*", ]) + def test_parametrize_class_scenarios(self, testdir): + testdir.makepyfile(""" + # same as doc/en/example/parametrize scenario example + def pytest_generate_tests(metafunc): + idlist = [] + argvalues = [] + for scenario in metafunc.cls.scenarios: + idlist.append(scenario[0]) + items = scenario[1].items() + argnames = [x[0] for x in items] + argvalues.append(([x[1] for x in items])) + metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class") + + class Test(object): + scenarios = [['1', {'arg': {1: 2}, "arg2": "value2"}], + ['2', {'arg':'value2', "arg2": "value2"}]] + + def test_1(self, arg, arg2): + pass + + def test_2(self, arg2, arg): + pass + + def test_3(self, arg, arg2): + pass + """) + result = testdir.runpytest("-v") + assert result.ret == 0 + result.stdout.fnmatch_lines(""" + *test_1*1* + *test_2*1* + *test_3*1* + *test_1*2* + *test_2*2* + *test_3*2* + *6 passed* + """) + class TestMetafuncFunctional: def test_attributes(self, testdir): p = testdir.makepyfile("""