From 4e35c00ab005da542e91a488da58fa3638618168 Mon Sep 17 00:00:00 2001 From: Brianna Laugher Date: Thu, 17 Apr 2014 15:08:49 -0400 Subject: [PATCH 1/3] issue351: Add ability to specify parametrize ids as a callable, to generate custom test ids. + tests, docs --HG-- branch : issue351 --- _pytest/python.py | 53 +++++++++++++++++------- doc/en/example/parametrize.txt | 75 ++++++++++++++++++++++++++++++++++ testing/python/metafunc.py | 46 +++++++++++++++++++++ 3 files changed, 159 insertions(+), 15 deletions(-) diff --git a/_pytest/python.py b/_pytest/python.py index 01925f539..0f70ef84b 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -771,9 +771,14 @@ class Metafunc(FuncargnamesCompatAttr): function so that it can perform more expensive setups during the setup phase of a test rather than at collection time. - :arg ids: list of string ids each corresponding to the argvalues so - that they are part of the test id. If no ids are provided they will - be generated automatically from the argvalues. + :arg ids: list of string ids, or a callable. + If strings, each is corresponding to the argvalues so that they are + part of the test id. + If callable, it should take one argument (a single argvalue) and return + a string or return None. If None, the automatically generated id for that + argument will be used. + If no ids are provided they will be generated automatically from + the argvalues. :arg scope: if specified it denotes the scope of the parameters. The scope is used for grouping tests by parameter instances. @@ -813,11 +818,15 @@ class Metafunc(FuncargnamesCompatAttr): raise ValueError("%r uses no fixture %r" %( self.function, arg)) valtype = indirect and "params" or "funcargs" + idfn = None + if callable(ids): + idfn = ids + ids = None if ids and len(ids) != len(argvalues): raise ValueError('%d tests specified with %d ids' %( len(argvalues), len(ids))) if not ids: - ids = idmaker(argnames, argvalues) + ids = idmaker(argnames, argvalues, idfn) newcalls = [] for callspec in self._calls or [CallSpec2(self)]: for param_index, valset in enumerate(argvalues): @@ -865,17 +874,31 @@ class Metafunc(FuncargnamesCompatAttr): cs.setall(funcargs, id, param) self._calls.append(cs) -def idmaker(argnames, argvalues): - idlist = [] - for valindex, valset in enumerate(argvalues): - this_id = [] - for nameindex, val in enumerate(valset): - if not isinstance(val, (float, int, str, bool, NoneType)): - this_id.append(str(argnames[nameindex])+str(valindex)) - else: - this_id.append(str(val)) - idlist.append("-".join(this_id)) - return idlist + +def _idval(val, argname, idx, idfn): + if idfn: + try: + s = idfn(val) + if s: + return s + except Exception: + pass + if isinstance(val, (float, int, str, bool, NoneType)): + return str(val) + return str(argname)+str(idx) + +def _idvalset(idx, valset, argnames, idfn): + this_id = [_idval(val, argname, idx, idfn) + for val, argname in zip(valset, argnames)] + return "-".join(this_id) + +def idmaker(argnames, argvalues, idfn=None): + ids = [_idvalset(valindex, valset, argnames, idfn) + for valindex, valset in enumerate(argvalues)] + if len(set(ids)) < len(ids): + # the user may have provided a bad idfn which means the ids are not unique + ids = ["{}".format(i) + testid for i, testid in enumerate(ids)] + return ids def showfixtures(config): from _pytest.main import wrap_session diff --git a/doc/en/example/parametrize.txt b/doc/en/example/parametrize.txt index ebecd9fa4..f457d5c5c 100644 --- a/doc/en/example/parametrize.txt +++ b/doc/en/example/parametrize.txt @@ -68,6 +68,81 @@ let's run the full monty:: As expected when running the full range of ``param1`` values we'll get an error on the last one. + +Different options for test IDs +------------------------------------ + +pytest will build a string that is the test ID for each set of values in a +parametrized test. These IDs can be used with "-k" to select specific cases +to run, and they will also identify the specific case when one is failing. +Running pytest with --collect-only will show the generated IDs. + +Numbers, strings, booleans and None will have their usual string representation +used in the test ID. For other objects, pytest will make a string based on +the argument name:: + + # contents of test_time.py + + from datetime import datetime, timedelta + + testdata = [(datetime(2001, 12, 12), datetime(2001, 12, 11), timedelta(1)), + (datetime(2001, 12, 11), datetime(2001, 12, 12), timedelta(-1)), + ] + + + @pytest.mark.parametrize("a,b,expected", testdata) + def test_timedistance_v0(a, b, expected): + diff = a - b + assert diff == expected + + + @pytest.mark.parametrize("a,b,expected", testdata, ids=["forward", "backward"]) + def test_timedistance_v1(a, b, expected): + diff = a - b + assert diff == expected + + + def idfn(val): + if isinstance(val, (datetime,)): + # note this wouldn't show any hours/minutes/seconds + return val.strftime('%Y%m%d') + + + @pytest.mark.parametrize("a,b,expected", testdata, ids=idfn) + def test_timedistance_v2(a, b, expected): + diff = a - b + assert diff == expected + + +In ``test_timedistance_v0``, we let pytest generate the test IDs. + +In ``test_timedistance_v1``, we specified ``ids`` as a list of strings which were +used as the test IDs. These are succinct, but can be a pain to maintain. + +In ``test_timedistance_v2``, we specified ``ids`` as a function that can generate a +string representation to make part of the test ID. So our ``datetime`` values use the +label generated by ``idfn``, but because we didn't generate a label for ``timedelta`` +objects, they are still using the default pytest representation:: + + + $ py.test test_time.py --collect-only + ============================ test session starts ============================= + platform linux2 -- Python 2.7.3 -- py-1.4.20 -- pytest-2.6.0.dev1 + plugins: cache + collected 6 items + + + + + + + + + ============================== in 0.04 seconds =============================== + + + + A quick port of "testscenarios" ------------------------------------ diff --git a/testing/python/metafunc.py b/testing/python/metafunc.py index 464c9e937..651e3846e 100644 --- a/testing/python/metafunc.py +++ b/testing/python/metafunc.py @@ -151,6 +151,52 @@ class TestMetafunc: "a6-b6", "a7-b7"] + @pytest.mark.issue351 + def test_idmaker_idfn(self): + from _pytest.python import idmaker + def ids(val): + if isinstance(val, Exception): + return repr(val) + + result = idmaker(("a", "b"), [(10.0, IndexError()), + (20, KeyError()), + ("three", [1, 2, 3]), + ], idfn=ids) + assert result == ["10.0-IndexError()", + "20-KeyError()", + "three-b2", + ] + + @pytest.mark.issue351 + def test_idmaker_idfn_unique_names(self): + from _pytest.python import idmaker + def ids(val): + return 'a' + + result = idmaker(("a", "b"), [(10.0, IndexError()), + (20, KeyError()), + ("three", [1, 2, 3]), + ], idfn=ids) + assert result == ["0a-a", + "1a-a", + "2a-a", + ] + + @pytest.mark.issue351 + def test_idmaker_idfn_exception(self): + from _pytest.python import idmaker + def ids(val): + raise Exception("bad code") + + result = idmaker(("a", "b"), [(10.0, IndexError()), + (20, KeyError()), + ("three", [1, 2, 3]), + ], idfn=ids) + assert result == ["10.0-b0", + "20-b1", + "three-b2", + ] + def test_addcall_and_parametrize(self): def func(x, y): pass metafunc = self.Metafunc(func) From ab005a4261927acbe96e2d76d5e0f6864cb51bd0 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 8 Oct 2014 00:11:32 +0100 Subject: [PATCH 2/3] Functional tests for id function --HG-- branch : issue351 --- testing/python/integration.py | 55 ++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/testing/python/integration.py b/testing/python/integration.py index e27f27d52..b01efa522 100644 --- a/testing/python/integration.py +++ b/testing/python/integration.py @@ -262,5 +262,58 @@ class TestNoselikeTestAttribute: call = reprec.getcalls("pytest_collection_modifyitems")[0] assert len(call.items) == 1 assert call.items[0].cls.__name__ == "TC" - + +@pytest.mark.issue351 +class TestParameterize: + + def test_idfn_marker(self, testdir): + testdir.makepyfile(""" + import pytest + + def idfn(param): + if param == 0: + return 'spam' + elif param == 1: + return 'ham' + else: + return None + + @pytest.mark.parametrize('a,b', [(0, 2), (1, 2)], ids=idfn) + def test_params(a, b): + pass + """) + res = testdir.runpytest('--collect-only') + res.stdout.fnmatch_lines([ + "*spam-2*", + "*ham-2*", + ]) + + def test_idfn_fixture(self, testdir): + testdir.makepyfile(""" + import pytest + + def idfn(param): + if param == 0: + return 'spam' + elif param == 1: + return 'ham' + else: + return None + + @pytest.fixture(params=[0, 1], ids=idfn) + def a(request): + return request.param + + @pytest.fixture(params=[1, 2], ids=idfn) + def b(request): + return request.param + + def test_params(a, b): + pass + """) + res = testdir.runpytest('--collect-only') + res.stdout.fnmatch_lines([ + "*spam-2*", + "*ham-2*", + ]) From 89de87dce1e16a653003c33c463c184c6987bd9c Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 8 Oct 2014 00:43:27 +0100 Subject: [PATCH 3/3] Document the ids keyword for fixture parametrisation --HG-- branch : issue351 --- doc/en/example/parametrize.txt | 4 +-- doc/en/fixture.txt | 55 ++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/doc/en/example/parametrize.txt b/doc/en/example/parametrize.txt index f457d5c5c..f2697ed69 100644 --- a/doc/en/example/parametrize.txt +++ b/doc/en/example/parametrize.txt @@ -73,9 +73,9 @@ Different options for test IDs ------------------------------------ pytest will build a string that is the test ID for each set of values in a -parametrized test. These IDs can be used with "-k" to select specific cases +parametrized test. These IDs can be used with ``-k`` to select specific cases to run, and they will also identify the specific case when one is failing. -Running pytest with --collect-only will show the generated IDs. +Running pytest with ``--collect-only`` will show the generated IDs. Numbers, strings, booleans and None will have their usual string representation used in the test ID. For other objects, pytest will make a string based on diff --git a/doc/en/fixture.txt b/doc/en/fixture.txt index 7ab780100..6b0499097 100644 --- a/doc/en/fixture.txt +++ b/doc/en/fixture.txt @@ -431,6 +431,61 @@ We see that our two test functions each ran twice, against the different connection the second test fails in ``test_ehlo`` because a different server string is expected than what arrived. +pytest will build a string that is the test ID for each fixture value +in a parametrized fixture, e.g. ``test_ehlo[merlinux.eu]`` and +``test_ehlo[mail.python.org]`` in the above examples. These IDs can +be used with ``-k`` to select specific cases to run, and they will +also identify the specific case when one is failing. Running pytest +with ``--collect-only`` will show the generated IDs. + +Numbers, strings, booleans and None will have their usual string +representation used in the test ID. For other objects, pytest will +make a string based on the argument name. It is possible to customise +the string used in a test ID for a certain fixture value by using the +``ids`` keyword argument:: + + import pytest + + @pytest.fixture(params=[0, 1], ids=["spam", "ham"]) + def a(request): + return request.param + + def test_a(a): + pass + + def idfn(fixture_value): + if fixture_value == 0: + return "eggs" + else: + return None + + @pytest.fixture(params=[0, 1], ids=idfn) + def b(request): + return request.param + + def test_b(b): + pass + +The above shows how ``ids`` can be either a list of strings to use or +a function which will be called with the fixture value and then +has to return a string to use. In the latter case if the function +return ``None`` then pytest's auto-generated ID will be used. + +Running the above tests results in the following test IDs being used:: + + $ py.test --collect-only + ========================== test session starts ========================== + platform linux2 -- Python 2.7.6 -- py-1.4.25.dev2 -- pytest-2.6.0.dev1 + plugins: xdist + collected 4 items + + + + + + + =========================== in 0.05 seconds ============================ + .. _`interdependent fixtures`: