Merge pull request #1921 from RonnyPfannschmidt/marked-value
introduce pytest.Marked as holder for marked parameter values
This commit is contained in:
Florian Bruhin
GitHub
commit
de8607deb2
|
|
@ -16,6 +16,9 @@ New Features
|
|||
* ``pytest.raises`` now asserts that the error message matches a text or regex
|
||||
with the ``match`` keyword argument. Thanks `@Kriechi`_ for the PR.
|
||||
|
||||
* ``pytest.param`` can be used to declare test parameter sets with marks and test ids.
|
||||
Thanks `@RonnyPfannschmidt`_ for the PR.
|
||||
|
||||
|
||||
Changes
|
||||
-------
|
||||
|
|
|
|||
|
|
@ -6,17 +6,72 @@ from collections import namedtuple
|
|||
from operator import attrgetter
|
||||
from .compat import imap
|
||||
|
||||
|
||||
def alias(name):
|
||||
return property(attrgetter(name), doc='alias for ' + name)
|
||||
|
||||
|
||||
class ParameterSet(namedtuple('ParameterSet', 'values, marks, id')):
|
||||
@classmethod
|
||||
def param(cls, *values, **kw):
|
||||
marks = kw.pop('marks', ())
|
||||
if isinstance(marks, MarkDecorator):
|
||||
marks = marks,
|
||||
else:
|
||||
assert isinstance(marks, (tuple, list, set))
|
||||
|
||||
def param_extract_id(id=None):
|
||||
return id
|
||||
|
||||
id = param_extract_id(**kw)
|
||||
return cls(values, marks, id)
|
||||
|
||||
@classmethod
|
||||
def extract_from(cls, parameterset, legacy_force_tuple=False):
|
||||
"""
|
||||
:param parameterset:
|
||||
a legacy style parameterset that may or may not be a tuple,
|
||||
and may or may not be wrapped into a mess of mark objects
|
||||
|
||||
:param legacy_force_tuple:
|
||||
enforce tuple wrapping so single argument tuple values
|
||||
don't get decomposed and break tests
|
||||
|
||||
"""
|
||||
|
||||
if isinstance(parameterset, cls):
|
||||
return parameterset
|
||||
if not isinstance(parameterset, MarkDecorator) and legacy_force_tuple:
|
||||
return cls.param(parameterset)
|
||||
|
||||
newmarks = []
|
||||
argval = parameterset
|
||||
while isinstance(argval, MarkDecorator):
|
||||
newmarks.append(MarkDecorator(Mark(
|
||||
argval.markname, argval.args[:-1], argval.kwargs)))
|
||||
argval = argval.args[-1]
|
||||
assert not isinstance(argval, ParameterSet)
|
||||
if legacy_force_tuple:
|
||||
argval = argval,
|
||||
|
||||
return cls(argval, marks=newmarks, id=None)
|
||||
|
||||
@property
|
||||
def deprecated_arg_dict(self):
|
||||
return dict((mark.name, mark) for mark in self.marks)
|
||||
|
||||
|
||||
class MarkerError(Exception):
|
||||
|
||||
"""Error in use of a pytest marker/attribute."""
|
||||
|
||||
|
||||
|
||||
def pytest_namespace():
|
||||
return {'mark': MarkGenerator()}
|
||||
return {
|
||||
'mark': MarkGenerator(),
|
||||
'param': ParameterSet.param,
|
||||
}
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
|
|
@ -212,6 +267,7 @@ def istestfunc(func):
|
|||
return hasattr(func, "__call__") and \
|
||||
getattr(func, "__name__", "<lambda>") != "<lambda>"
|
||||
|
||||
|
||||
class MarkDecorator(object):
|
||||
""" A decorator for test functions and test classes. When applied
|
||||
it will create :class:`MarkInfo` objects which may be
|
||||
|
|
@ -257,8 +313,11 @@ class MarkDecorator(object):
|
|||
def markname(self):
|
||||
return self.name # for backward-compat (2.4.1 had this attr)
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.mark == other.mark
|
||||
|
||||
def __repr__(self):
|
||||
return "<MarkDecorator %r>" % self.mark
|
||||
return "<MarkDecorator %r>" % (self.mark,)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
""" if passed a single callable argument: decorate it with mark info.
|
||||
|
|
@ -291,19 +350,7 @@ class MarkDecorator(object):
|
|||
return self.__class__(self.mark.combined_with(mark))
|
||||
|
||||
|
||||
def extract_argvalue(maybe_marked_args):
|
||||
# TODO: incorrect mark data, the old code wanst able to collect lists
|
||||
# individual parametrized argument sets can be wrapped in a series
|
||||
# of markers in which case we unwrap the values and apply the mark
|
||||
# at Function init
|
||||
newmarks = {}
|
||||
argval = maybe_marked_args
|
||||
while isinstance(argval, MarkDecorator):
|
||||
newmark = MarkDecorator(Mark(
|
||||
argval.markname, argval.args[:-1], argval.kwargs))
|
||||
newmarks[newmark.name] = newmark
|
||||
argval = argval.args[-1]
|
||||
return argval, newmarks
|
||||
|
||||
|
||||
|
||||
class Mark(namedtuple('Mark', 'name, args, kwargs')):
|
||||
|
|
|
|||
|
|
@ -788,36 +788,35 @@ class Metafunc(fixtures.FuncargnamesCompatAttr):
|
|||
to set a dynamic scope using test context or configuration.
|
||||
"""
|
||||
from _pytest.fixtures import scope2index
|
||||
from _pytest.mark import extract_argvalue
|
||||
from _pytest.mark import ParameterSet
|
||||
from py.io import saferepr
|
||||
|
||||
unwrapped_argvalues = []
|
||||
newkeywords = []
|
||||
for maybe_marked_args in argvalues:
|
||||
argval, newmarks = extract_argvalue(maybe_marked_args)
|
||||
unwrapped_argvalues.append(argval)
|
||||
newkeywords.append(newmarks)
|
||||
argvalues = unwrapped_argvalues
|
||||
|
||||
if not isinstance(argnames, (tuple, list)):
|
||||
argnames = [x.strip() for x in argnames.split(",") if x.strip()]
|
||||
if len(argnames) == 1:
|
||||
argvalues = [(val,) for val in argvalues]
|
||||
if not argvalues:
|
||||
argvalues = [(NOTSET,) * len(argnames)]
|
||||
# we passed a empty list to parameterize, skip that test
|
||||
#
|
||||
force_tuple = len(argnames) == 1
|
||||
else:
|
||||
force_tuple = False
|
||||
parameters = [
|
||||
ParameterSet.extract_from(x, legacy_force_tuple=force_tuple)
|
||||
for x in argvalues]
|
||||
del argvalues
|
||||
|
||||
|
||||
if not parameters:
|
||||
fs, lineno = getfslineno(self.function)
|
||||
newmark = pytest.mark.skip(
|
||||
reason = "got empty parameter set %r, function %s at %s:%d" % (
|
||||
argnames, self.function.__name__, fs, lineno))
|
||||
newkeywords = [{newmark.markname: newmark}]
|
||||
argnames, self.function.__name__, fs, lineno)
|
||||
mark = pytest.mark.skip(reason=reason)
|
||||
parameters.append(ParameterSet(
|
||||
values=(NOTSET,) * len(argnames),
|
||||
marks=[mark],
|
||||
id=None,
|
||||
))
|
||||
|
||||
if scope is None:
|
||||
scope = _find_parametrized_scope(argnames, self._arg2fixturedefs, indirect)
|
||||
|
||||
scopenum = scope2index(
|
||||
scope, descr='call to {0}'.format(self.parametrize))
|
||||
scopenum = scope2index(scope, descr='call to {0}'.format(self.parametrize))
|
||||
valtypes = {}
|
||||
for arg in argnames:
|
||||
if arg not in self.fixturenames:
|
||||
|
|
@ -845,22 +844,22 @@ class Metafunc(fixtures.FuncargnamesCompatAttr):
|
|||
idfn = ids
|
||||
ids = None
|
||||
if ids:
|
||||
if len(ids) != len(argvalues):
|
||||
if len(ids) != len(parameters):
|
||||
raise ValueError('%d tests specified with %d ids' % (
|
||||
len(argvalues), len(ids)))
|
||||
len(parameters), len(ids)))
|
||||
for id_value in ids:
|
||||
if id_value is not None and not isinstance(id_value, py.builtin._basestring):
|
||||
msg = 'ids must be list of strings, found: %s (type: %s)'
|
||||
raise ValueError(msg % (saferepr(id_value), type(id_value).__name__))
|
||||
ids = idmaker(argnames, argvalues, idfn, ids, self.config)
|
||||
ids = idmaker(argnames, parameters, idfn, ids, self.config)
|
||||
newcalls = []
|
||||
for callspec in self._calls or [CallSpec2(self)]:
|
||||
elements = zip(ids, argvalues, newkeywords, count())
|
||||
for a_id, valset, keywords, param_index in elements:
|
||||
assert len(valset) == len(argnames)
|
||||
elements = zip(ids, parameters, count())
|
||||
for a_id, param, param_index in elements:
|
||||
assert len(param.values) == len(argnames)
|
||||
newcallspec = callspec.copy(self)
|
||||
newcallspec.setmulti(valtypes, argnames, valset, a_id,
|
||||
keywords, scopenum, param_index)
|
||||
newcallspec.setmulti(valtypes, argnames, param.values, a_id,
|
||||
param.deprecated_arg_dict, scopenum, param_index)
|
||||
newcalls.append(newcallspec)
|
||||
self._calls = newcalls
|
||||
|
||||
|
|
@ -959,17 +958,19 @@ def _idval(val, argname, idx, idfn, config=None):
|
|||
return val.__name__
|
||||
return str(argname)+str(idx)
|
||||
|
||||
def _idvalset(idx, valset, argnames, idfn, ids, config=None):
|
||||
def _idvalset(idx, parameterset, argnames, idfn, ids, config=None):
|
||||
if parameterset.id is not None:
|
||||
return parameterset.id
|
||||
if ids is None or (idx >= len(ids) or ids[idx] is None):
|
||||
this_id = [_idval(val, argname, idx, idfn, config)
|
||||
for val, argname in zip(valset, argnames)]
|
||||
for val, argname in zip(parameterset.values, argnames)]
|
||||
return "-".join(this_id)
|
||||
else:
|
||||
return _escape_strings(ids[idx])
|
||||
|
||||
def idmaker(argnames, argvalues, idfn=None, ids=None, config=None):
|
||||
ids = [_idvalset(valindex, valset, argnames, idfn, ids, config)
|
||||
for valindex, valset in enumerate(argvalues)]
|
||||
def idmaker(argnames, parametersets, idfn=None, ids=None, config=None):
|
||||
ids = [_idvalset(valindex, parameterset, argnames, idfn, ids, config)
|
||||
for valindex, parameterset in enumerate(parametersets)]
|
||||
if len(set(ids)) != len(ids):
|
||||
# The ids are not unique
|
||||
duplicates = [testid for testid in ids if ids.count(testid) > 1]
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ them in turn::
|
|||
|
||||
$ pytest
|
||||
======= test session starts ========
|
||||
platform linux -- Python 3.5.2, pytest-3.0.7, py-1.4.32, pluggy-0.4.0
|
||||
platform linux -- Python 3.5.2, pytest-3.0.3, py-1.4.31, pluggy-0.4.0
|
||||
rootdir: $REGENDOC_TMPDIR, inifile:
|
||||
collected 3 items
|
||||
|
||||
|
|
@ -73,7 +73,7 @@ them in turn::
|
|||
])
|
||||
def test_eval(test_input, expected):
|
||||
> assert eval(test_input) == expected
|
||||
E AssertionError: assert 54 == 42
|
||||
E assert 54 == 42
|
||||
E + where 54 = eval('6*9')
|
||||
|
||||
test_expectation.py:8: AssertionError
|
||||
|
|
@ -94,16 +94,37 @@ for example with the builtin ``mark.xfail``::
|
|||
@pytest.mark.parametrize("test_input,expected", [
|
||||
("3+5", 8),
|
||||
("2+4", 6),
|
||||
pytest.mark.xfail(("6*9", 42)),
|
||||
pytest.param("6*9", 42,
|
||||
marks=pytest.mark.xfail),
|
||||
])
|
||||
def test_eval(test_input, expected):
|
||||
assert eval(test_input) == expected
|
||||
|
||||
.. note::
|
||||
|
||||
prior to version 3.1 the supported mechanism for marking values
|
||||
used the syntax::
|
||||
|
||||
import pytest
|
||||
@pytest.mark.parametrize("test_input,expected", [
|
||||
("3+5", 8),
|
||||
("2+4", 6),
|
||||
pytest.mark.xfail(("6*9", 42),),
|
||||
])
|
||||
def test_eval(test_input, expected):
|
||||
assert eval(test_input) == expected
|
||||
|
||||
|
||||
This was an initial hack to support the feature but soon was demonstrated to be incomplete,
|
||||
broken for passing functions or applying multiple marks with the same name but different parameters.
|
||||
The old syntax will be removed in pytest-4.0.
|
||||
|
||||
|
||||
Let's run this::
|
||||
|
||||
$ pytest
|
||||
======= test session starts ========
|
||||
platform linux -- Python 3.5.2, pytest-3.0.7, py-1.4.32, pluggy-0.4.0
|
||||
platform linux -- Python 3.5.2, pytest-3.0.3, py-1.4.31, pluggy-0.4.0
|
||||
rootdir: $REGENDOC_TMPDIR, inifile:
|
||||
collected 3 items
|
||||
|
||||
|
|
@ -186,7 +207,7 @@ Let's also run with a stringinput that will lead to a failing test::
|
|||
|
||||
def test_valid_string(stringinput):
|
||||
> assert stringinput.isalpha()
|
||||
E AssertionError: assert False
|
||||
E assert False
|
||||
E + where False = <built-in method isalpha of str object at 0xdeadbeef>()
|
||||
E + where <built-in method isalpha of str object at 0xdeadbeef> = '!'.isalpha
|
||||
|
||||
|
|
|
|||
|
|
@ -207,37 +207,40 @@ class TestMetafunc(object):
|
|||
@pytest.mark.issue250
|
||||
def test_idmaker_autoname(self):
|
||||
from _pytest.python import idmaker
|
||||
result = idmaker(("a", "b"), [("string", 1.0),
|
||||
("st-ring", 2.0)])
|
||||
result = idmaker(("a", "b"), [pytest.param("string", 1.0),
|
||||
pytest.param("st-ring", 2.0)])
|
||||
assert result == ["string-1.0", "st-ring-2.0"]
|
||||
|
||||
result = idmaker(("a", "b"), [(object(), 1.0),
|
||||
(object(), object())])
|
||||
result = idmaker(("a", "b"), [pytest.param(object(), 1.0),
|
||||
pytest.param(object(), object())])
|
||||
assert result == ["a0-1.0", "a1-b1"]
|
||||
# unicode mixing, issue250
|
||||
result = idmaker((py.builtin._totext("a"), "b"), [({}, b'\xc3\xb4')])
|
||||
result = idmaker(
|
||||
(py.builtin._totext("a"), "b"),
|
||||
[pytest.param({}, b'\xc3\xb4')])
|
||||
assert result == ['a0-\\xc3\\xb4']
|
||||
|
||||
def test_idmaker_with_bytes_regex(self):
|
||||
from _pytest.python import idmaker
|
||||
result = idmaker(("a"), [(re.compile(b'foo'), 1.0)])
|
||||
result = idmaker(("a"), [pytest.param(re.compile(b'foo'), 1.0)])
|
||||
assert result == ["foo"]
|
||||
|
||||
def test_idmaker_native_strings(self):
|
||||
from _pytest.python import idmaker
|
||||
totext = py.builtin._totext
|
||||
result = idmaker(("a", "b"), [(1.0, -1.1),
|
||||
(2, -202),
|
||||
("three", "three hundred"),
|
||||
(True, False),
|
||||
(None, None),
|
||||
(re.compile('foo'), re.compile('bar')),
|
||||
(str, int),
|
||||
(list("six"), [66, 66]),
|
||||
(set([7]), set("seven")),
|
||||
(tuple("eight"), (8, -8, 8)),
|
||||
(b'\xc3\xb4', b"name"),
|
||||
(b'\xc3\xb4', totext("other")),
|
||||
result = idmaker(("a", "b"), [
|
||||
pytest.param(1.0, -1.1),
|
||||
pytest.param(2, -202),
|
||||
pytest.param("three", "three hundred"),
|
||||
pytest.param(True, False),
|
||||
pytest.param(None, None),
|
||||
pytest.param(re.compile('foo'), re.compile('bar')),
|
||||
pytest.param(str, int),
|
||||
pytest.param(list("six"), [66, 66]),
|
||||
pytest.param(set([7]), set("seven")),
|
||||
pytest.param(tuple("eight"), (8, -8, 8)),
|
||||
pytest.param(b'\xc3\xb4', b"name"),
|
||||
pytest.param(b'\xc3\xb4', totext("other")),
|
||||
])
|
||||
assert result == ["1.0--1.1",
|
||||
"2--202",
|
||||
|
|
@ -257,7 +260,7 @@ class TestMetafunc(object):
|
|||
from _pytest.python import idmaker
|
||||
enum = pytest.importorskip("enum")
|
||||
e = enum.Enum("Foo", "one, two")
|
||||
result = idmaker(("a", "b"), [(e.one, e.two)])
|
||||
result = idmaker(("a", "b"), [pytest.param(e.one, e.two)])
|
||||
assert result == ["Foo.one-Foo.two"]
|
||||
|
||||
@pytest.mark.issue351
|
||||
|
|
@ -268,9 +271,10 @@ class TestMetafunc(object):
|
|||
if isinstance(val, Exception):
|
||||
return repr(val)
|
||||
|
||||
result = idmaker(("a", "b"), [(10.0, IndexError()),
|
||||
(20, KeyError()),
|
||||
("three", [1, 2, 3]),
|
||||
result = idmaker(("a", "b"), [
|
||||
pytest.param(10.0, IndexError()),
|
||||
pytest.param(20, KeyError()),
|
||||
pytest.param("three", [1, 2, 3]),
|
||||
], idfn=ids)
|
||||
assert result == ["10.0-IndexError()",
|
||||
"20-KeyError()",
|
||||
|
|
@ -284,9 +288,9 @@ class TestMetafunc(object):
|
|||
def ids(val):
|
||||
return 'a'
|
||||
|
||||
result = idmaker(("a", "b"), [(10.0, IndexError()),
|
||||
(20, KeyError()),
|
||||
("three", [1, 2, 3]),
|
||||
result = idmaker(("a", "b"), [pytest.param(10.0, IndexError()),
|
||||
pytest.param(20, KeyError()),
|
||||
pytest.param("three", [1, 2, 3]),
|
||||
], idfn=ids)
|
||||
assert result == ["a-a0",
|
||||
"a-a1",
|
||||
|
|
@ -306,9 +310,10 @@ class TestMetafunc(object):
|
|||
|
||||
rec = WarningsRecorder()
|
||||
with rec:
|
||||
idmaker(("a", "b"), [(10.0, IndexError()),
|
||||
(20, KeyError()),
|
||||
("three", [1, 2, 3]),
|
||||
idmaker(("a", "b"), [
|
||||
pytest.param(10.0, IndexError()),
|
||||
pytest.param(20, KeyError()),
|
||||
pytest.param("three", [1, 2, 3]),
|
||||
], idfn=ids)
|
||||
|
||||
assert [str(i.message) for i in rec.list] == [
|
||||
|
|
@ -351,14 +356,21 @@ class TestMetafunc(object):
|
|||
|
||||
def test_idmaker_with_ids(self):
|
||||
from _pytest.python import idmaker
|
||||
result = idmaker(("a", "b"), [(1, 2),
|
||||
(3, 4)],
|
||||
result = idmaker(("a", "b"), [pytest.param(1, 2),
|
||||
pytest.param(3, 4)],
|
||||
ids=["a", None])
|
||||
assert result == ["a", "3-4"]
|
||||
|
||||
def test_idmaker_with_paramset_id(self):
|
||||
from _pytest.python import idmaker
|
||||
result = idmaker(("a", "b"), [pytest.param(1, 2, id="me"),
|
||||
pytest.param(3, 4, id="you")],
|
||||
ids=["a", None])
|
||||
assert result == ["me", "you"]
|
||||
|
||||
def test_idmaker_with_ids_unique_names(self):
|
||||
from _pytest.python import idmaker
|
||||
result = idmaker(("a"), [1,2,3,4,5],
|
||||
result = idmaker(("a"), map(pytest.param, [1,2,3,4,5]),
|
||||
ids=["a", "a", "b", "c", "b"])
|
||||
assert result == ["a0", "a1", "b0", "c", "b1"]
|
||||
|
||||
|
|
@ -1438,6 +1450,31 @@ class TestMarkersWithParametrization(object):
|
|||
reprec = testdir.inline_run()
|
||||
reprec.assertoutcome(passed=2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('strict', [True, False])
|
||||
def test_parametrize_marked_value(self, testdir, strict):
|
||||
s = """
|
||||
import pytest
|
||||
|
||||
@pytest.mark.parametrize(("n", "expected"), [
|
||||
pytest.param(
|
||||
2,3,
|
||||
marks=pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict}),
|
||||
),
|
||||
pytest.param(
|
||||
2,3,
|
||||
marks=[pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict})],
|
||||
),
|
||||
])
|
||||
def test_increment(n, expected):
|
||||
assert n + 1 == expected
|
||||
""".format(strict=strict)
|
||||
testdir.makepyfile(s)
|
||||
reprec = testdir.inline_run()
|
||||
passed, failed = (0, 2) if strict else (2, 0)
|
||||
reprec.assertoutcome(passed=passed, failed=failed)
|
||||
|
||||
|
||||
def test_pytest_make_parametrize_id(self, testdir):
|
||||
testdir.makeconftest("""
|
||||
def pytest_make_parametrize_id(config, val):
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
from __future__ import absolute_import, division, print_function
|
||||
import os
|
||||
import sys
|
||||
|
||||
import py, pytest
|
||||
from _pytest.mark import MarkGenerator as Mark
|
||||
import pytest
|
||||
from _pytest.mark import MarkGenerator as Mark, ParameterSet
|
||||
|
||||
class TestMark(object):
|
||||
def test_markinfo_repr(self):
|
||||
|
|
@ -10,9 +11,11 @@ class TestMark(object):
|
|||
m = MarkInfo(Mark("hello", (1,2), {}))
|
||||
repr(m)
|
||||
|
||||
def test_pytest_exists_in_namespace_all(self):
|
||||
assert 'mark' in py.test.__all__
|
||||
assert 'mark' in pytest.__all__
|
||||
@pytest.mark.parametrize('attr', ['mark', 'param'])
|
||||
@pytest.mark.parametrize('modulename', ['py.test', 'pytest'])
|
||||
def test_pytest_exists_in_namespace_all(self, attr, modulename):
|
||||
module = sys.modules[modulename]
|
||||
assert attr in module.__all__
|
||||
|
||||
def test_pytest_mark_notcallable(self):
|
||||
mark = Mark()
|
||||
|
|
@ -739,3 +742,16 @@ class TestKeywordSelection(object):
|
|||
|
||||
assert_test_is_not_selected("__")
|
||||
assert_test_is_not_selected("()")
|
||||
|
||||
|
||||
@pytest.mark.parametrize('argval, expected', [
|
||||
(pytest.mark.skip()((1, 2)),
|
||||
ParameterSet(values=(1, 2), marks=[pytest.mark.skip], id=None)),
|
||||
(pytest.mark.xfail(pytest.mark.skip()((1, 2))),
|
||||
ParameterSet(values=(1, 2),
|
||||
marks=[pytest.mark.xfail, pytest.mark.skip], id=None)),
|
||||
|
||||
])
|
||||
def test_parameterset_extractfrom(argval, expected):
|
||||
extracted = ParameterSet.extract_from(argval)
|
||||
assert extracted == expected
|
||||
|
|
|
|||
Loading…
Reference in New Issue