From e368fb4b29912fe618d523b2ca9197016738d7d6 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 7 Sep 2016 11:00:27 +0200 Subject: [PATCH] implement pytest.param this allows a clear addition of parameterization parameters that carry along marks instead of nesting multiple mark objects and destroying the possibility of creating function valued parameters, it just folders everything together into one object carrfying parameters, and the marks. --- _pytest/mark.py | 77 +++++++++++++++++++++++------ _pytest/python.py | 71 +++++++++++++-------------- doc/en/parametrize.rst | 51 ++++++++++++++------ testing/python/metafunc.py | 99 ++++++++++++++++++++++++++------------ testing/test_mark.py | 28 ++++++++--- 5 files changed, 224 insertions(+), 102 deletions(-) diff --git a/_pytest/mark.py b/_pytest/mark.py index 643f43ce0..a06b02b14 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -6,17 +6,72 @@ from collections import namedtuple from operator import attrgetter from .compat import imap + def alias(name): return property(attrgetter(name), doc='alias for ' + name) +class ParameterSet(namedtuple('ParameterSet', 'values, marks, id')): + @classmethod + def param(cls, *values, **kw): + marks = kw.pop('marks', ()) + if isinstance(marks, MarkDecorator): + marks = marks, + else: + assert isinstance(marks, (tuple, list, set)) + + def param_extract_id(id=None): + return id + + id = param_extract_id(**kw) + return cls(values, marks, id) + + @classmethod + def extract_from(cls, parameterset, legacy_force_tuple=False): + """ + :param parameterset: + a legacy style parameterset that may or may not be a tuple, + and may or may not be wrapped into a mess of mark objects + + :param legacy_force_tuple: + enforce tuple wrapping so single argument tuple values + don't get decomposed and break tests + + """ + + if isinstance(parameterset, cls): + return parameterset + if not isinstance(parameterset, MarkDecorator) and legacy_force_tuple: + return cls.param(parameterset) + + newmarks = [] + argval = parameterset + while isinstance(argval, MarkDecorator): + newmarks.append(MarkDecorator(Mark( + argval.markname, argval.args[:-1], argval.kwargs))) + argval = argval.args[-1] + assert not isinstance(argval, ParameterSet) + if legacy_force_tuple: + argval = argval, + + return cls(argval, marks=newmarks, id=None) + + @property + def deprecated_arg_dict(self): + return dict((mark.name, mark) for mark in self.marks) + + class MarkerError(Exception): """Error in use of a pytest marker/attribute.""" + def pytest_namespace(): - return {'mark': MarkGenerator()} + return { + 'mark': MarkGenerator(), + 'param': ParameterSet.param, + } def pytest_addoption(parser): @@ -212,6 +267,7 @@ def istestfunc(func): return hasattr(func, "__call__") and \ getattr(func, "__name__", "") != "" + class MarkDecorator(object): """ A decorator for test functions and test classes. When applied it will create :class:`MarkInfo` objects which may be @@ -257,8 +313,11 @@ class MarkDecorator(object): def markname(self): return self.name # for backward-compat (2.4.1 had this attr) + def __eq__(self, other): + return self.mark == other.mark + def __repr__(self): - return "" % self.mark + return "" % (self.mark,) def __call__(self, *args, **kwargs): """ if passed a single callable argument: decorate it with mark info. @@ -291,19 +350,7 @@ class MarkDecorator(object): return self.__class__(self.mark.combined_with(mark)) -def extract_argvalue(maybe_marked_args): - # TODO: incorrect mark data, the old code wanst able to collect lists - # individual parametrized argument sets can be wrapped in a series - # of markers in which case we unwrap the values and apply the mark - # at Function init - newmarks = {} - argval = maybe_marked_args - while isinstance(argval, MarkDecorator): - newmark = MarkDecorator(Mark( - argval.markname, argval.args[:-1], argval.kwargs)) - newmarks[newmark.name] = newmark - argval = argval.args[-1] - return argval, newmarks + class Mark(namedtuple('Mark', 'name, args, kwargs')): diff --git a/_pytest/python.py b/_pytest/python.py index 7d94ed61a..81eed00f0 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -788,36 +788,35 @@ class Metafunc(fixtures.FuncargnamesCompatAttr): to set a dynamic scope using test context or configuration. """ from _pytest.fixtures import scope2index - from _pytest.mark import extract_argvalue + from _pytest.mark import ParameterSet from py.io import saferepr - unwrapped_argvalues = [] - newkeywords = [] - for maybe_marked_args in argvalues: - argval, newmarks = extract_argvalue(maybe_marked_args) - unwrapped_argvalues.append(argval) - newkeywords.append(newmarks) - argvalues = unwrapped_argvalues - if not isinstance(argnames, (tuple, list)): argnames = [x.strip() for x in argnames.split(",") if x.strip()] - if len(argnames) == 1: - argvalues = [(val,) for val in argvalues] - if not argvalues: - argvalues = [(NOTSET,) * len(argnames)] - # we passed a empty list to parameterize, skip that test - # + force_tuple = len(argnames) == 1 + else: + force_tuple = False + parameters = [ + ParameterSet.extract_from(x, legacy_force_tuple=force_tuple) + for x in argvalues] + del argvalues + + + if not parameters: fs, lineno = getfslineno(self.function) - newmark = pytest.mark.skip( - reason="got empty parameter set %r, function %s at %s:%d" % ( - argnames, self.function.__name__, fs, lineno)) - newkeywords = [{newmark.markname: newmark}] + reason = "got empty parameter set %r, function %s at %s:%d" % ( + argnames, self.function.__name__, fs, lineno) + mark = pytest.mark.skip(reason=reason) + parameters.append(ParameterSet( + values=(NOTSET,) * len(argnames), + marks=[mark], + id=None, + )) if scope is None: scope = _find_parametrized_scope(argnames, self._arg2fixturedefs, indirect) - scopenum = scope2index( - scope, descr='call to {0}'.format(self.parametrize)) + scopenum = scope2index(scope, descr='call to {0}'.format(self.parametrize)) valtypes = {} for arg in argnames: if arg not in self.fixturenames: @@ -845,22 +844,22 @@ class Metafunc(fixtures.FuncargnamesCompatAttr): idfn = ids ids = None if ids: - if len(ids) != len(argvalues): - raise ValueError('%d tests specified with %d ids' %( - len(argvalues), len(ids))) + if len(ids) != len(parameters): + raise ValueError('%d tests specified with %d ids' % ( + len(parameters), len(ids))) for id_value in ids: if id_value is not None and not isinstance(id_value, py.builtin._basestring): msg = 'ids must be list of strings, found: %s (type: %s)' raise ValueError(msg % (saferepr(id_value), type(id_value).__name__)) - ids = idmaker(argnames, argvalues, idfn, ids, self.config) + ids = idmaker(argnames, parameters, idfn, ids, self.config) newcalls = [] for callspec in self._calls or [CallSpec2(self)]: - elements = zip(ids, argvalues, newkeywords, count()) - for a_id, valset, keywords, param_index in elements: - assert len(valset) == len(argnames) + elements = zip(ids, parameters, count()) + for a_id, param, param_index in elements: + assert len(param.values) == len(argnames) newcallspec = callspec.copy(self) - newcallspec.setmulti(valtypes, argnames, valset, a_id, - keywords, scopenum, param_index) + newcallspec.setmulti(valtypes, argnames, param.values, a_id, + param.deprecated_arg_dict, scopenum, param_index) newcalls.append(newcallspec) self._calls = newcalls @@ -959,17 +958,19 @@ def _idval(val, argname, idx, idfn, config=None): return val.__name__ return str(argname)+str(idx) -def _idvalset(idx, valset, argnames, idfn, ids, config=None): +def _idvalset(idx, parameterset, argnames, idfn, ids, config=None): + if parameterset.id is not None: + return parameterset.id if ids is None or (idx >= len(ids) or ids[idx] is None): this_id = [_idval(val, argname, idx, idfn, config) - for val, argname in zip(valset, argnames)] + for val, argname in zip(parameterset.values, argnames)] return "-".join(this_id) else: return _escape_strings(ids[idx]) -def idmaker(argnames, argvalues, idfn=None, ids=None, config=None): - ids = [_idvalset(valindex, valset, argnames, idfn, ids, config) - for valindex, valset in enumerate(argvalues)] +def idmaker(argnames, parametersets, idfn=None, ids=None, config=None): + ids = [_idvalset(valindex, parameterset, argnames, idfn, ids, config) + for valindex, parameterset in enumerate(parametersets)] if len(set(ids)) != len(ids): # The ids are not unique duplicates = [testid for testid in ids if ids.count(testid) > 1] diff --git a/doc/en/parametrize.rst b/doc/en/parametrize.rst index b93099e7d..10de065c7 100644 --- a/doc/en/parametrize.rst +++ b/doc/en/parametrize.rst @@ -55,17 +55,17 @@ them in turn:: $ pytest ======= test session starts ======== - platform linux -- Python 3.5.2, pytest-3.0.7, py-1.4.32, pluggy-0.4.0 + platform linux -- Python 3.5.2, pytest-3.0.3, py-1.4.31, pluggy-0.4.0 rootdir: $REGENDOC_TMPDIR, inifile: collected 3 items - + test_expectation.py ..F - + ======= FAILURES ======== _______ test_eval[6*9-42] ________ - + test_input = '6*9', expected = 42 - + @pytest.mark.parametrize("test_input,expected", [ ("3+5", 8), ("2+4", 6), @@ -73,9 +73,9 @@ them in turn:: ]) def test_eval(test_input, expected): > assert eval(test_input) == expected - E AssertionError: assert 54 == 42 + E assert 54 == 42 E + where 54 = eval('6*9') - + test_expectation.py:8: AssertionError ======= 1 failed, 2 passed in 0.12 seconds ======== @@ -94,21 +94,42 @@ for example with the builtin ``mark.xfail``:: @pytest.mark.parametrize("test_input,expected", [ ("3+5", 8), ("2+4", 6), - pytest.mark.xfail(("6*9", 42)), + pytest.param("6*9", 42, + marks=pytest.mark.xfail), ]) def test_eval(test_input, expected): assert eval(test_input) == expected +.. note:: + + prior to version 3.1 the supported mechanism for marking values + used the syntax:: + + import pytest + @pytest.mark.parametrize("test_input,expected", [ + ("3+5", 8), + ("2+4", 6), + pytest.mark.xfail(("6*9", 42),), + ]) + def test_eval(test_input, expected): + assert eval(test_input) == expected + + + This was an initial hack to support the feature but soon was demonstrated to be incomplete, + broken for passing functions or applying multiple marks with the same name but different parameters. + The old syntax will be removed in pytest-4.0. + + Let's run this:: $ pytest ======= test session starts ======== - platform linux -- Python 3.5.2, pytest-3.0.7, py-1.4.32, pluggy-0.4.0 + platform linux -- Python 3.5.2, pytest-3.0.3, py-1.4.31, pluggy-0.4.0 rootdir: $REGENDOC_TMPDIR, inifile: collected 3 items - + test_expectation.py ..x - + ======= 2 passed, 1 xfailed in 0.12 seconds ======== The one parameter set which caused a failure previously now @@ -181,15 +202,15 @@ Let's also run with a stringinput that will lead to a failing test:: F ======= FAILURES ======== _______ test_valid_string[!] ________ - + stringinput = '!' - + def test_valid_string(stringinput): > assert stringinput.isalpha() - E AssertionError: assert False + E assert False E + where False = () E + where = '!'.isalpha - + test_strings.py:3: AssertionError 1 failed in 0.12 seconds diff --git a/testing/python/metafunc.py b/testing/python/metafunc.py index 949b265a3..c347dc9e2 100644 --- a/testing/python/metafunc.py +++ b/testing/python/metafunc.py @@ -207,37 +207,40 @@ class TestMetafunc(object): @pytest.mark.issue250 def test_idmaker_autoname(self): from _pytest.python import idmaker - result = idmaker(("a", "b"), [("string", 1.0), - ("st-ring", 2.0)]) + result = idmaker(("a", "b"), [pytest.param("string", 1.0), + pytest.param("st-ring", 2.0)]) assert result == ["string-1.0", "st-ring-2.0"] - result = idmaker(("a", "b"), [(object(), 1.0), - (object(), object())]) + result = idmaker(("a", "b"), [pytest.param(object(), 1.0), + pytest.param(object(), object())]) assert result == ["a0-1.0", "a1-b1"] # unicode mixing, issue250 - result = idmaker((py.builtin._totext("a"), "b"), [({}, b'\xc3\xb4')]) + result = idmaker( + (py.builtin._totext("a"), "b"), + [pytest.param({}, b'\xc3\xb4')]) assert result == ['a0-\\xc3\\xb4'] def test_idmaker_with_bytes_regex(self): from _pytest.python import idmaker - result = idmaker(("a"), [(re.compile(b'foo'), 1.0)]) + result = idmaker(("a"), [pytest.param(re.compile(b'foo'), 1.0)]) assert result == ["foo"] def test_idmaker_native_strings(self): from _pytest.python import idmaker totext = py.builtin._totext - result = idmaker(("a", "b"), [(1.0, -1.1), - (2, -202), - ("three", "three hundred"), - (True, False), - (None, None), - (re.compile('foo'), re.compile('bar')), - (str, int), - (list("six"), [66, 66]), - (set([7]), set("seven")), - (tuple("eight"), (8, -8, 8)), - (b'\xc3\xb4', b"name"), - (b'\xc3\xb4', totext("other")), + result = idmaker(("a", "b"), [ + pytest.param(1.0, -1.1), + pytest.param(2, -202), + pytest.param("three", "three hundred"), + pytest.param(True, False), + pytest.param(None, None), + pytest.param(re.compile('foo'), re.compile('bar')), + pytest.param(str, int), + pytest.param(list("six"), [66, 66]), + pytest.param(set([7]), set("seven")), + pytest.param(tuple("eight"), (8, -8, 8)), + pytest.param(b'\xc3\xb4', b"name"), + pytest.param(b'\xc3\xb4', totext("other")), ]) assert result == ["1.0--1.1", "2--202", @@ -257,7 +260,7 @@ class TestMetafunc(object): from _pytest.python import idmaker enum = pytest.importorskip("enum") e = enum.Enum("Foo", "one, two") - result = idmaker(("a", "b"), [(e.one, e.two)]) + result = idmaker(("a", "b"), [pytest.param(e.one, e.two)]) assert result == ["Foo.one-Foo.two"] @pytest.mark.issue351 @@ -268,9 +271,10 @@ class TestMetafunc(object): if isinstance(val, Exception): return repr(val) - result = idmaker(("a", "b"), [(10.0, IndexError()), - (20, KeyError()), - ("three", [1, 2, 3]), + result = idmaker(("a", "b"), [ + pytest.param(10.0, IndexError()), + pytest.param(20, KeyError()), + pytest.param("three", [1, 2, 3]), ], idfn=ids) assert result == ["10.0-IndexError()", "20-KeyError()", @@ -284,9 +288,9 @@ class TestMetafunc(object): def ids(val): return 'a' - result = idmaker(("a", "b"), [(10.0, IndexError()), - (20, KeyError()), - ("three", [1, 2, 3]), + result = idmaker(("a", "b"), [pytest.param(10.0, IndexError()), + pytest.param(20, KeyError()), + pytest.param("three", [1, 2, 3]), ], idfn=ids) assert result == ["a-a0", "a-a1", @@ -306,9 +310,10 @@ class TestMetafunc(object): rec = WarningsRecorder() with rec: - idmaker(("a", "b"), [(10.0, IndexError()), - (20, KeyError()), - ("three", [1, 2, 3]), + idmaker(("a", "b"), [ + pytest.param(10.0, IndexError()), + pytest.param(20, KeyError()), + pytest.param("three", [1, 2, 3]), ], idfn=ids) assert [str(i.message) for i in rec.list] == [ @@ -351,14 +356,21 @@ class TestMetafunc(object): def test_idmaker_with_ids(self): from _pytest.python import idmaker - result = idmaker(("a", "b"), [(1, 2), - (3, 4)], + result = idmaker(("a", "b"), [pytest.param(1, 2), + pytest.param(3, 4)], ids=["a", None]) assert result == ["a", "3-4"] + def test_idmaker_with_paramset_id(self): + from _pytest.python import idmaker + result = idmaker(("a", "b"), [pytest.param(1, 2, id="me"), + pytest.param(3, 4, id="you")], + ids=["a", None]) + assert result == ["me", "you"] + def test_idmaker_with_ids_unique_names(self): from _pytest.python import idmaker - result = idmaker(("a"), [1,2,3,4,5], + result = idmaker(("a"), map(pytest.param, [1,2,3,4,5]), ids=["a", "a", "b", "c", "b"]) assert result == ["a0", "a1", "b0", "c", "b1"] @@ -1438,6 +1450,31 @@ class TestMarkersWithParametrization(object): reprec = testdir.inline_run() reprec.assertoutcome(passed=2) + + @pytest.mark.parametrize('strict', [True, False]) + def test_parametrize_marked_value(self, testdir, strict): + s = """ + import pytest + + @pytest.mark.parametrize(("n", "expected"), [ + pytest.param( + 2,3, + marks=pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict}), + ), + pytest.param( + 2,3, + marks=[pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict})], + ), + ]) + def test_increment(n, expected): + assert n + 1 == expected + """.format(strict=strict) + testdir.makepyfile(s) + reprec = testdir.inline_run() + passed, failed = (0, 2) if strict else (2, 0) + reprec.assertoutcome(passed=passed, failed=failed) + + def test_pytest_make_parametrize_id(self, testdir): testdir.makeconftest(""" def pytest_make_parametrize_id(config, val): diff --git a/testing/test_mark.py b/testing/test_mark.py index 3bec1d0d0..9a8c35bf3 100644 --- a/testing/test_mark.py +++ b/testing/test_mark.py @@ -1,8 +1,9 @@ from __future__ import absolute_import, division, print_function import os +import sys -import py, pytest -from _pytest.mark import MarkGenerator as Mark +import pytest +from _pytest.mark import MarkGenerator as Mark, ParameterSet class TestMark(object): def test_markinfo_repr(self): @@ -10,9 +11,11 @@ class TestMark(object): m = MarkInfo(Mark("hello", (1,2), {})) repr(m) - def test_pytest_exists_in_namespace_all(self): - assert 'mark' in py.test.__all__ - assert 'mark' in pytest.__all__ + @pytest.mark.parametrize('attr', ['mark', 'param']) + @pytest.mark.parametrize('modulename', ['py.test', 'pytest']) + def test_pytest_exists_in_namespace_all(self, attr, modulename): + module = sys.modules[modulename] + assert attr in module.__all__ def test_pytest_mark_notcallable(self): mark = Mark() @@ -415,7 +418,7 @@ class TestFunctional(object): """) items, rec = testdir.inline_genitems(p) for item in items: - print (item, item.keywords) + print(item, item.keywords) assert 'a' in item.keywords def test_mark_decorator_subclass_does_not_propagate_to_base(self, testdir): @@ -739,3 +742,16 @@ class TestKeywordSelection(object): assert_test_is_not_selected("__") assert_test_is_not_selected("()") + + +@pytest.mark.parametrize('argval, expected', [ + (pytest.mark.skip()((1, 2)), + ParameterSet(values=(1, 2), marks=[pytest.mark.skip], id=None)), + (pytest.mark.xfail(pytest.mark.skip()((1, 2))), + ParameterSet(values=(1, 2), + marks=[pytest.mark.xfail, pytest.mark.skip], id=None)), + +]) +def test_parameterset_extractfrom(argval, expected): + extracted = ParameterSet.extract_from(argval) + assert extracted == expected