Merged in pfctdayelise/pytest/issue351 (pull request #161)
Fixes issue351: Add ability to specify parametrize ids as a callable, to generate custom test ids. + tests, docs Hg branch merge
This commit is contained in:
Floris Bruynooghe
commit
8d19ccb56f
|
|
@ -798,9 +798,14 @@ class Metafunc(FuncargnamesCompatAttr):
|
|||
function so that it can perform more expensive setups during the
|
||||
setup phase of a test rather than at collection time.
|
||||
|
||||
:arg ids: list of string ids each corresponding to the argvalues so
|
||||
that they are part of the test id. If no ids are provided they will
|
||||
be generated automatically from the argvalues.
|
||||
:arg ids: list of string ids, or a callable.
|
||||
If strings, each is corresponding to the argvalues so that they are
|
||||
part of the test id.
|
||||
If callable, it should take one argument (a single argvalue) and return
|
||||
a string or return None. If None, the automatically generated id for that
|
||||
argument will be used.
|
||||
If no ids are provided they will be generated automatically from
|
||||
the argvalues.
|
||||
|
||||
:arg scope: if specified it denotes the scope of the parameters.
|
||||
The scope is used for grouping tests by parameter instances.
|
||||
|
|
@ -840,11 +845,15 @@ class Metafunc(FuncargnamesCompatAttr):
|
|||
raise ValueError("%r uses no fixture %r" %(
|
||||
self.function, arg))
|
||||
valtype = indirect and "params" or "funcargs"
|
||||
idfn = None
|
||||
if callable(ids):
|
||||
idfn = ids
|
||||
ids = None
|
||||
if ids and len(ids) != len(argvalues):
|
||||
raise ValueError('%d tests specified with %d ids' %(
|
||||
len(argvalues), len(ids)))
|
||||
if not ids:
|
||||
ids = idmaker(argnames, argvalues)
|
||||
ids = idmaker(argnames, argvalues, idfn)
|
||||
newcalls = []
|
||||
for callspec in self._calls or [CallSpec2(self)]:
|
||||
for param_index, valset in enumerate(argvalues):
|
||||
|
|
@ -892,17 +901,31 @@ class Metafunc(FuncargnamesCompatAttr):
|
|||
cs.setall(funcargs, id, param)
|
||||
self._calls.append(cs)
|
||||
|
||||
def idmaker(argnames, argvalues):
|
||||
idlist = []
|
||||
for valindex, valset in enumerate(argvalues):
|
||||
this_id = []
|
||||
for nameindex, val in enumerate(valset):
|
||||
if not isinstance(val, (float, int, str, bool, NoneType)):
|
||||
this_id.append(str(argnames[nameindex])+str(valindex))
|
||||
else:
|
||||
this_id.append(str(val))
|
||||
idlist.append("-".join(this_id))
|
||||
return idlist
|
||||
|
||||
def _idval(val, argname, idx, idfn):
|
||||
if idfn:
|
||||
try:
|
||||
s = idfn(val)
|
||||
if s:
|
||||
return s
|
||||
except Exception:
|
||||
pass
|
||||
if isinstance(val, (float, int, str, bool, NoneType)):
|
||||
return str(val)
|
||||
return str(argname)+str(idx)
|
||||
|
||||
def _idvalset(idx, valset, argnames, idfn):
|
||||
this_id = [_idval(val, argname, idx, idfn)
|
||||
for val, argname in zip(valset, argnames)]
|
||||
return "-".join(this_id)
|
||||
|
||||
def idmaker(argnames, argvalues, idfn=None):
|
||||
ids = [_idvalset(valindex, valset, argnames, idfn)
|
||||
for valindex, valset in enumerate(argvalues)]
|
||||
if len(set(ids)) < len(ids):
|
||||
# the user may have provided a bad idfn which means the ids are not unique
|
||||
ids = ["{}".format(i) + testid for i, testid in enumerate(ids)]
|
||||
return ids
|
||||
|
||||
def showfixtures(config):
|
||||
from _pytest.main import wrap_session
|
||||
|
|
|
|||
|
|
@ -68,6 +68,81 @@ let's run the full monty::
|
|||
As expected when running the full range of ``param1`` values
|
||||
we'll get an error on the last one.
|
||||
|
||||
|
||||
Different options for test IDs
|
||||
------------------------------------
|
||||
|
||||
pytest will build a string that is the test ID for each set of values in a
|
||||
parametrized test. These IDs can be used with ``-k`` to select specific cases
|
||||
to run, and they will also identify the specific case when one is failing.
|
||||
Running pytest with ``--collect-only`` will show the generated IDs.
|
||||
|
||||
Numbers, strings, booleans and None will have their usual string representation
|
||||
used in the test ID. For other objects, pytest will make a string based on
|
||||
the argument name::
|
||||
|
||||
# contents of test_time.py
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
testdata = [(datetime(2001, 12, 12), datetime(2001, 12, 11), timedelta(1)),
|
||||
(datetime(2001, 12, 11), datetime(2001, 12, 12), timedelta(-1)),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("a,b,expected", testdata)
|
||||
def test_timedistance_v0(a, b, expected):
|
||||
diff = a - b
|
||||
assert diff == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("a,b,expected", testdata, ids=["forward", "backward"])
|
||||
def test_timedistance_v1(a, b, expected):
|
||||
diff = a - b
|
||||
assert diff == expected
|
||||
|
||||
|
||||
def idfn(val):
|
||||
if isinstance(val, (datetime,)):
|
||||
# note this wouldn't show any hours/minutes/seconds
|
||||
return val.strftime('%Y%m%d')
|
||||
|
||||
|
||||
@pytest.mark.parametrize("a,b,expected", testdata, ids=idfn)
|
||||
def test_timedistance_v2(a, b, expected):
|
||||
diff = a - b
|
||||
assert diff == expected
|
||||
|
||||
|
||||
In ``test_timedistance_v0``, we let pytest generate the test IDs.
|
||||
|
||||
In ``test_timedistance_v1``, we specified ``ids`` as a list of strings which were
|
||||
used as the test IDs. These are succinct, but can be a pain to maintain.
|
||||
|
||||
In ``test_timedistance_v2``, we specified ``ids`` as a function that can generate a
|
||||
string representation to make part of the test ID. So our ``datetime`` values use the
|
||||
label generated by ``idfn``, but because we didn't generate a label for ``timedelta``
|
||||
objects, they are still using the default pytest representation::
|
||||
|
||||
|
||||
$ py.test test_time.py --collect-only
|
||||
============================ test session starts =============================
|
||||
platform linux2 -- Python 2.7.3 -- py-1.4.20 -- pytest-2.6.0.dev1
|
||||
plugins: cache
|
||||
collected 6 items
|
||||
<Module 'test_time.py'>
|
||||
<Function 'test_timedistance_v0[a0-b0-expected0]'>
|
||||
<Function 'test_timedistance_v0[a1-b1-expected1]'>
|
||||
<Function 'test_timedistance_v1[forward]'>
|
||||
<Function 'test_timedistance_v1[backward]'>
|
||||
<Function 'test_timedistance_v2[20011212-20011211-expected0]'>
|
||||
<Function 'test_timedistance_v2[20011211-20011212-expected1]'>
|
||||
|
||||
============================== in 0.04 seconds ===============================
|
||||
|
||||
|
||||
|
||||
|
||||
A quick port of "testscenarios"
|
||||
------------------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -429,6 +429,61 @@ We see that our two test functions each ran twice, against the different
|
|||
connection the second test fails in ``test_ehlo`` because a
|
||||
different server string is expected than what arrived.
|
||||
|
||||
pytest will build a string that is the test ID for each fixture value
|
||||
in a parametrized fixture, e.g. ``test_ehlo[merlinux.eu]`` and
|
||||
``test_ehlo[mail.python.org]`` in the above examples. These IDs can
|
||||
be used with ``-k`` to select specific cases to run, and they will
|
||||
also identify the specific case when one is failing. Running pytest
|
||||
with ``--collect-only`` will show the generated IDs.
|
||||
|
||||
Numbers, strings, booleans and None will have their usual string
|
||||
representation used in the test ID. For other objects, pytest will
|
||||
make a string based on the argument name. It is possible to customise
|
||||
the string used in a test ID for a certain fixture value by using the
|
||||
``ids`` keyword argument::
|
||||
|
||||
import pytest
|
||||
|
||||
@pytest.fixture(params=[0, 1], ids=["spam", "ham"])
|
||||
def a(request):
|
||||
return request.param
|
||||
|
||||
def test_a(a):
|
||||
pass
|
||||
|
||||
def idfn(fixture_value):
|
||||
if fixture_value == 0:
|
||||
return "eggs"
|
||||
else:
|
||||
return None
|
||||
|
||||
@pytest.fixture(params=[0, 1], ids=idfn)
|
||||
def b(request):
|
||||
return request.param
|
||||
|
||||
def test_b(b):
|
||||
pass
|
||||
|
||||
The above shows how ``ids`` can be either a list of strings to use or
|
||||
a function which will be called with the fixture value and then
|
||||
has to return a string to use. In the latter case if the function
|
||||
return ``None`` then pytest's auto-generated ID will be used.
|
||||
|
||||
Running the above tests results in the following test IDs being used::
|
||||
|
||||
$ py.test --collect-only
|
||||
========================== test session starts ==========================
|
||||
platform linux2 -- Python 2.7.6 -- py-1.4.25.dev2 -- pytest-2.6.0.dev1
|
||||
plugins: xdist
|
||||
collected 4 items
|
||||
<Module 'test_ids.py'>
|
||||
<Function 'test_a[spam]'>
|
||||
<Function 'test_a[ham]'>
|
||||
<Function 'test_b[eggs]'>
|
||||
<Function 'test_b[1]'>
|
||||
|
||||
=========================== in 0.05 seconds ============================
|
||||
|
||||
|
||||
.. _`interdependent fixtures`:
|
||||
|
||||
|
|
|
|||
|
|
@ -282,5 +282,58 @@ class TestNoselikeTestAttribute:
|
|||
call = reprec.getcalls("pytest_collection_modifyitems")[0]
|
||||
assert len(call.items) == 1
|
||||
assert call.items[0].cls.__name__ == "TC"
|
||||
|
||||
|
||||
|
||||
@pytest.mark.issue351
|
||||
class TestParameterize:
|
||||
|
||||
def test_idfn_marker(self, testdir):
|
||||
testdir.makepyfile("""
|
||||
import pytest
|
||||
|
||||
def idfn(param):
|
||||
if param == 0:
|
||||
return 'spam'
|
||||
elif param == 1:
|
||||
return 'ham'
|
||||
else:
|
||||
return None
|
||||
|
||||
@pytest.mark.parametrize('a,b', [(0, 2), (1, 2)], ids=idfn)
|
||||
def test_params(a, b):
|
||||
pass
|
||||
""")
|
||||
res = testdir.runpytest('--collect-only')
|
||||
res.stdout.fnmatch_lines([
|
||||
"*spam-2*",
|
||||
"*ham-2*",
|
||||
])
|
||||
|
||||
def test_idfn_fixture(self, testdir):
|
||||
testdir.makepyfile("""
|
||||
import pytest
|
||||
|
||||
def idfn(param):
|
||||
if param == 0:
|
||||
return 'spam'
|
||||
elif param == 1:
|
||||
return 'ham'
|
||||
else:
|
||||
return None
|
||||
|
||||
@pytest.fixture(params=[0, 1], ids=idfn)
|
||||
def a(request):
|
||||
return request.param
|
||||
|
||||
@pytest.fixture(params=[1, 2], ids=idfn)
|
||||
def b(request):
|
||||
return request.param
|
||||
|
||||
def test_params(a, b):
|
||||
pass
|
||||
""")
|
||||
res = testdir.runpytest('--collect-only')
|
||||
res.stdout.fnmatch_lines([
|
||||
"*spam-2*",
|
||||
"*ham-2*",
|
||||
])
|
||||
|
|
|
|||
|
|
@ -151,6 +151,52 @@ class TestMetafunc:
|
|||
"a6-b6",
|
||||
"a7-b7"]
|
||||
|
||||
@pytest.mark.issue351
|
||||
def test_idmaker_idfn(self):
|
||||
from _pytest.python import idmaker
|
||||
def ids(val):
|
||||
if isinstance(val, Exception):
|
||||
return repr(val)
|
||||
|
||||
result = idmaker(("a", "b"), [(10.0, IndexError()),
|
||||
(20, KeyError()),
|
||||
("three", [1, 2, 3]),
|
||||
], idfn=ids)
|
||||
assert result == ["10.0-IndexError()",
|
||||
"20-KeyError()",
|
||||
"three-b2",
|
||||
]
|
||||
|
||||
@pytest.mark.issue351
|
||||
def test_idmaker_idfn_unique_names(self):
|
||||
from _pytest.python import idmaker
|
||||
def ids(val):
|
||||
return 'a'
|
||||
|
||||
result = idmaker(("a", "b"), [(10.0, IndexError()),
|
||||
(20, KeyError()),
|
||||
("three", [1, 2, 3]),
|
||||
], idfn=ids)
|
||||
assert result == ["0a-a",
|
||||
"1a-a",
|
||||
"2a-a",
|
||||
]
|
||||
|
||||
@pytest.mark.issue351
|
||||
def test_idmaker_idfn_exception(self):
|
||||
from _pytest.python import idmaker
|
||||
def ids(val):
|
||||
raise Exception("bad code")
|
||||
|
||||
result = idmaker(("a", "b"), [(10.0, IndexError()),
|
||||
(20, KeyError()),
|
||||
("three", [1, 2, 3]),
|
||||
], idfn=ids)
|
||||
assert result == ["10.0-b0",
|
||||
"20-b1",
|
||||
"three-b2",
|
||||
]
|
||||
|
||||
def test_addcall_and_parametrize(self):
|
||||
def func(x, y): pass
|
||||
metafunc = self.Metafunc(func)
|
||||
|
|
|
|||
Loading…
Reference in New Issue