Merge pull request #1921 from RonnyPfannschmidt/marked-value

introduce pytest.Marked as holder for marked parameter values

CHANGELOG.rst

@@ -16,6 +16,9 @@ New Features
 * ``pytest.raises`` now asserts that the error message matches a text or regex
   with the ``match`` keyword argument. Thanks `@Kriechi`_ for the PR.
 
+* ``pytest.param`` can be used to declare test parameter sets with marks and test ids.
+  Thanks `@RonnyPfannschmidt`_ for the PR.
+
 
 Changes
 -------

_pytest/mark.py

@@ -6,17 +6,72 @@ from collections import namedtuple
 from operator import attrgetter
 from .compat import imap
 
+
 def alias(name):
     return property(attrgetter(name), doc='alias for ' + name)
 
 
+class ParameterSet(namedtuple('ParameterSet', 'values, marks, id')):
+    @classmethod
+    def param(cls, *values, **kw):
+        marks = kw.pop('marks', ())
+        if isinstance(marks, MarkDecorator):
+            marks = marks,
+        else:
+            assert isinstance(marks, (tuple, list, set))
+
+        def param_extract_id(id=None):
+            return id
+
+        id = param_extract_id(**kw)
+        return cls(values, marks, id)
+
+    @classmethod
+    def extract_from(cls, parameterset, legacy_force_tuple=False):
+        """
+        :param parameterset:
+            a legacy style parameterset that may or may not be a tuple,
+            and may or may not be wrapped into a mess of mark objects
+
+        :param legacy_force_tuple:
+            enforce tuple wrapping so single argument tuple values
+            don't get decomposed and break tests
+
+        """
+
+        if isinstance(parameterset, cls):
+            return parameterset
+        if not isinstance(parameterset, MarkDecorator) and legacy_force_tuple:
+            return cls.param(parameterset)
+
+        newmarks = []
+        argval = parameterset
+        while isinstance(argval, MarkDecorator):
+            newmarks.append(MarkDecorator(Mark(
+                argval.markname, argval.args[:-1], argval.kwargs)))
+            argval = argval.args[-1]
+        assert not isinstance(argval, ParameterSet)
+        if legacy_force_tuple:
+            argval = argval,
+
+        return cls(argval, marks=newmarks, id=None)
+
+    @property
+    def deprecated_arg_dict(self):
+        return dict((mark.name, mark) for mark in self.marks)
+
+
 class MarkerError(Exception):
 
     """Error in use of a pytest marker/attribute."""
 
 
+
 def pytest_namespace():
-    return {'mark': MarkGenerator()}
+    return {
+        'mark': MarkGenerator(),
+        'param': ParameterSet.param,
+    }
 
 
 def pytest_addoption(parser):
@@ -212,6 +267,7 @@ def istestfunc(func):
     return hasattr(func, "__call__") and \
         getattr(func, "__name__", "<lambda>") != "<lambda>"
 
+
 class MarkDecorator(object):
     """ A decorator for test functions and test classes.  When applied
     it will create :class:`MarkInfo` objects which may be
@@ -257,8 +313,11 @@ class MarkDecorator(object):
     def markname(self):
         return self.name # for backward-compat (2.4.1 had this attr)
 
+    def __eq__(self, other):
+        return self.mark == other.mark
+
     def __repr__(self):
-        return "<MarkDecorator %r>" % self.mark
+        return "<MarkDecorator %r>" % (self.mark,)
 
     def __call__(self, *args, **kwargs):
         """ if passed a single callable argument: decorate it with mark info.
@@ -291,19 +350,7 @@ class MarkDecorator(object):
         return self.__class__(self.mark.combined_with(mark))
 
 
-def extract_argvalue(maybe_marked_args):
+
-    # TODO: incorrect mark data, the old code wanst able to collect lists
-    # individual parametrized argument sets can be wrapped in a series
-    # of markers in which case we unwrap the values and apply the mark
-    # at Function init
-    newmarks = {}
-    argval = maybe_marked_args
-    while isinstance(argval, MarkDecorator):
-        newmark = MarkDecorator(Mark(
-            argval.markname, argval.args[:-1], argval.kwargs))
-        newmarks[newmark.name] = newmark
-        argval = argval.args[-1]
-    return argval, newmarks
 
 
 class Mark(namedtuple('Mark', 'name, args, kwargs')):

_pytest/python.py

@@ -788,36 +788,35 @@ class Metafunc(fixtures.FuncargnamesCompatAttr):
             to set a dynamic scope using test context or configuration.
         """
         from _pytest.fixtures import scope2index
-        from _pytest.mark import extract_argvalue
+        from _pytest.mark import ParameterSet
         from py.io import saferepr
 
-        unwrapped_argvalues = []
-        newkeywords = []
-        for maybe_marked_args in argvalues:
-            argval, newmarks = extract_argvalue(maybe_marked_args)
-            unwrapped_argvalues.append(argval)
-            newkeywords.append(newmarks)
-        argvalues = unwrapped_argvalues
-
         if not isinstance(argnames, (tuple, list)):
             argnames = [x.strip() for x in argnames.split(",") if x.strip()]
-            if len(argnames) == 1:
+            force_tuple = len(argnames) == 1
-                argvalues = [(val,) for val in argvalues]
+        else:
-        if not argvalues:
+            force_tuple = False
-            argvalues = [(NOTSET,) * len(argnames)]
+        parameters = [
-            # we passed a empty list to parameterize, skip that test
+            ParameterSet.extract_from(x, legacy_force_tuple=force_tuple)
-            #
+            for x in argvalues]
+        del argvalues
+
+        
+        if not parameters:
             fs, lineno = getfslineno(self.function)
-            newmark = pytest.mark.skip(
+            reason = "got empty parameter set %r, function %s at %s:%d" % (
-                reason="got empty parameter set %r, function %s at %s:%d" % (
+                    argnames, self.function.__name__, fs, lineno)
-                    argnames, self.function.__name__, fs, lineno))
+            mark = pytest.mark.skip(reason=reason)
-            newkeywords = [{newmark.markname: newmark}]
+            parameters.append(ParameterSet(
+                values=(NOTSET,) * len(argnames),
+                marks=[mark],
+                id=None,
+            ))
 
         if scope is None:
             scope = _find_parametrized_scope(argnames, self._arg2fixturedefs, indirect)
 
-        scopenum = scope2index(
+        scopenum = scope2index(scope, descr='call to {0}'.format(self.parametrize))
-            scope, descr='call to {0}'.format(self.parametrize))
         valtypes = {}
         for arg in argnames:
             if arg not in self.fixturenames:
@@ -845,22 +844,22 @@ class Metafunc(fixtures.FuncargnamesCompatAttr):
             idfn = ids
             ids = None
         if ids:
-            if len(ids) != len(argvalues):
+            if len(ids) != len(parameters):
-                raise ValueError('%d tests specified with %d ids' %(
+                raise ValueError('%d tests specified with %d ids' % (
-                                 len(argvalues), len(ids)))
+                                 len(parameters), len(ids)))
             for id_value in ids:
                 if id_value is not None and not isinstance(id_value, py.builtin._basestring):
                     msg = 'ids must be list of strings, found: %s (type: %s)'
                     raise ValueError(msg % (saferepr(id_value), type(id_value).__name__))
-        ids = idmaker(argnames, argvalues, idfn, ids, self.config)
+        ids = idmaker(argnames, parameters, idfn, ids, self.config)
         newcalls = []
         for callspec in self._calls or [CallSpec2(self)]:
-            elements = zip(ids, argvalues, newkeywords, count())
+            elements = zip(ids, parameters, count())
-            for a_id, valset, keywords, param_index in elements:
+            for a_id, param, param_index in elements:
-                assert len(valset) == len(argnames)
+                assert len(param.values) == len(argnames)
                 newcallspec = callspec.copy(self)
-                newcallspec.setmulti(valtypes, argnames, valset, a_id,
+                newcallspec.setmulti(valtypes, argnames, param.values, a_id,
-                                     keywords, scopenum, param_index)
+                                     param.deprecated_arg_dict, scopenum, param_index)
                 newcalls.append(newcallspec)
         self._calls = newcalls
 
@@ -959,17 +958,19 @@ def _idval(val, argname, idx, idfn, config=None):
         return val.__name__
     return str(argname)+str(idx)
 
-def _idvalset(idx, valset, argnames, idfn, ids, config=None):
+def _idvalset(idx, parameterset, argnames, idfn, ids, config=None):
+    if parameterset.id is not None:
+        return parameterset.id
     if ids is None or (idx >= len(ids) or ids[idx] is None):
         this_id = [_idval(val, argname, idx, idfn, config)
-                   for val, argname in zip(valset, argnames)]
+                   for val, argname in zip(parameterset.values, argnames)]
         return "-".join(this_id)
     else:
         return _escape_strings(ids[idx])
 
-def idmaker(argnames, argvalues, idfn=None, ids=None, config=None):
+def idmaker(argnames, parametersets, idfn=None, ids=None, config=None):
-    ids = [_idvalset(valindex, valset, argnames, idfn, ids, config)
+    ids = [_idvalset(valindex, parameterset, argnames, idfn, ids, config)
-           for valindex, valset in enumerate(argvalues)]
+           for valindex, parameterset in enumerate(parametersets)]
     if len(set(ids)) != len(ids):
         # The ids are not unique
         duplicates = [testid for testid in ids if ids.count(testid) > 1]

doc/en/parametrize.rst

@@ -55,17 +55,17 @@ them in turn::
 
     $ pytest
     ======= test session starts ========
-    platform linux -- Python 3.5.2, pytest-3.0.7, py-1.4.32, pluggy-0.4.0
+    platform linux -- Python 3.5.2, pytest-3.0.3, py-1.4.31, pluggy-0.4.0
     rootdir: $REGENDOC_TMPDIR, inifile:
     collected 3 items
-    
+
     test_expectation.py ..F
-    
+
     ======= FAILURES ========
     _______ test_eval[6*9-42] ________
-    
+
     test_input = '6*9', expected = 42
-    
+
         @pytest.mark.parametrize("test_input,expected", [
             ("3+5", 8),
             ("2+4", 6),
@@ -73,9 +73,9 @@ them in turn::
         ])
         def test_eval(test_input, expected):
     >       assert eval(test_input) == expected
-    E       AssertionError: assert 54 == 42
+    E       assert 54 == 42
     E        +  where 54 = eval('6*9')
-    
+
     test_expectation.py:8: AssertionError
     ======= 1 failed, 2 passed in 0.12 seconds ========
 
@@ -94,21 +94,42 @@ for example with the builtin ``mark.xfail``::
     @pytest.mark.parametrize("test_input,expected", [
         ("3+5", 8),
         ("2+4", 6),
-        pytest.mark.xfail(("6*9", 42)),
+        pytest.param("6*9", 42,
+                     marks=pytest.mark.xfail),
     ])
     def test_eval(test_input, expected):
         assert eval(test_input) == expected
 
+.. note::
+
+  prior to version 3.1 the supported mechanism for marking values
+  used the syntax::
+
+        import pytest
+        @pytest.mark.parametrize("test_input,expected", [
+            ("3+5", 8),
+            ("2+4", 6),
+            pytest.mark.xfail(("6*9", 42),),
+        ])
+        def test_eval(test_input, expected):
+            assert eval(test_input) == expected
+
+
+  This was an initial hack to support the feature but soon was demonstrated to be incomplete,
+  broken for passing functions or applying multiple marks with the same name but different parameters.
+  The old syntax will be removed in pytest-4.0.
+
+
 Let's run this::
 
     $ pytest
     ======= test session starts ========
-    platform linux -- Python 3.5.2, pytest-3.0.7, py-1.4.32, pluggy-0.4.0
+    platform linux -- Python 3.5.2, pytest-3.0.3, py-1.4.31, pluggy-0.4.0
     rootdir: $REGENDOC_TMPDIR, inifile:
     collected 3 items
-    
+
     test_expectation.py ..x
-    
+
     ======= 2 passed, 1 xfailed in 0.12 seconds ========
 
 The one parameter set which caused a failure previously now
@@ -181,15 +202,15 @@ Let's also run with a stringinput that will lead to a failing test::
     F
     ======= FAILURES ========
     _______ test_valid_string[!] ________
-    
+
     stringinput = '!'
-    
+
         def test_valid_string(stringinput):
     >       assert stringinput.isalpha()
-    E       AssertionError: assert False
+    E       assert False
     E        +  where False = <built-in method isalpha of str object at 0xdeadbeef>()
     E        +    where <built-in method isalpha of str object at 0xdeadbeef> = '!'.isalpha
-    
+
     test_strings.py:3: AssertionError
     1 failed in 0.12 seconds
 

testing/python/metafunc.py

@@ -207,37 +207,40 @@ class TestMetafunc(object):
     @pytest.mark.issue250
     def test_idmaker_autoname(self):
         from _pytest.python import idmaker
-        result = idmaker(("a", "b"), [("string", 1.0),
+        result = idmaker(("a", "b"), [pytest.param("string", 1.0),
-                                      ("st-ring", 2.0)])
+                                      pytest.param("st-ring", 2.0)])
         assert result == ["string-1.0", "st-ring-2.0"]
 
-        result = idmaker(("a", "b"), [(object(), 1.0),
+        result = idmaker(("a", "b"), [pytest.param(object(), 1.0),
-                                      (object(), object())])
+                                      pytest.param(object(), object())])
         assert result == ["a0-1.0", "a1-b1"]
         # unicode mixing, issue250
-        result = idmaker((py.builtin._totext("a"), "b"), [({}, b'\xc3\xb4')])
+        result = idmaker(
+            (py.builtin._totext("a"), "b"),
+            [pytest.param({}, b'\xc3\xb4')])
         assert result == ['a0-\\xc3\\xb4']
 
     def test_idmaker_with_bytes_regex(self):
         from _pytest.python import idmaker
-        result = idmaker(("a"), [(re.compile(b'foo'), 1.0)])
+        result = idmaker(("a"), [pytest.param(re.compile(b'foo'), 1.0)])
         assert result == ["foo"]
 
     def test_idmaker_native_strings(self):
         from _pytest.python import idmaker
         totext = py.builtin._totext
-        result = idmaker(("a", "b"), [(1.0, -1.1),
+        result = idmaker(("a", "b"), [
-                                      (2, -202),
+            pytest.param(1.0, -1.1),
-                                      ("three", "three hundred"),
+            pytest.param(2, -202),
-                                      (True, False),
+            pytest.param("three", "three hundred"),
-                                      (None, None),
+            pytest.param(True, False),
-                                      (re.compile('foo'), re.compile('bar')),
+            pytest.param(None, None),
-                                      (str, int),
+            pytest.param(re.compile('foo'), re.compile('bar')),
-                                      (list("six"), [66, 66]),
+            pytest.param(str, int),
-                                      (set([7]), set("seven")),
+            pytest.param(list("six"), [66, 66]),
-                                      (tuple("eight"), (8, -8, 8)),
+            pytest.param(set([7]), set("seven")),
-                                      (b'\xc3\xb4', b"name"),
+            pytest.param(tuple("eight"), (8, -8, 8)),
-                                      (b'\xc3\xb4', totext("other")),
+            pytest.param(b'\xc3\xb4', b"name"),
+            pytest.param(b'\xc3\xb4', totext("other")),
         ])
         assert result == ["1.0--1.1",
                           "2--202",
@@ -257,7 +260,7 @@ class TestMetafunc(object):
         from _pytest.python import idmaker
         enum = pytest.importorskip("enum")
         e = enum.Enum("Foo", "one, two")
-        result = idmaker(("a", "b"), [(e.one, e.two)])
+        result = idmaker(("a", "b"), [pytest.param(e.one, e.two)])
         assert result == ["Foo.one-Foo.two"]
 
     @pytest.mark.issue351
@@ -268,9 +271,10 @@ class TestMetafunc(object):
             if isinstance(val, Exception):
                 return repr(val)
 
-        result = idmaker(("a", "b"), [(10.0, IndexError()),
+        result = idmaker(("a", "b"), [
-                                      (20, KeyError()),
+            pytest.param(10.0, IndexError()),
-                                      ("three", [1, 2, 3]),
+            pytest.param(20, KeyError()),
+            pytest.param("three", [1, 2, 3]),
         ], idfn=ids)
         assert result == ["10.0-IndexError()",
                           "20-KeyError()",
@@ -284,9 +288,9 @@ class TestMetafunc(object):
         def ids(val):
             return 'a'
 
-        result = idmaker(("a", "b"), [(10.0, IndexError()),
+        result = idmaker(("a", "b"), [pytest.param(10.0, IndexError()),
-                                      (20, KeyError()),
+                                      pytest.param(20, KeyError()),
-                                      ("three", [1, 2, 3]),
+                                      pytest.param("three", [1, 2, 3]),
         ], idfn=ids)
         assert result == ["a-a0",
                           "a-a1",
@@ -306,9 +310,10 @@ class TestMetafunc(object):
 
         rec = WarningsRecorder()
         with rec:
-            idmaker(("a", "b"), [(10.0, IndexError()),
+            idmaker(("a", "b"), [
-                                          (20, KeyError()),
+                pytest.param(10.0, IndexError()),
-                                          ("three", [1, 2, 3]),
+                pytest.param(20, KeyError()),
+                pytest.param("three", [1, 2, 3]),
             ], idfn=ids)
 
         assert [str(i.message) for i in rec.list] == [
@@ -351,14 +356,21 @@ class TestMetafunc(object):
 
     def test_idmaker_with_ids(self):
         from _pytest.python import idmaker
-        result = idmaker(("a", "b"), [(1, 2),
+        result = idmaker(("a", "b"), [pytest.param(1, 2),
-                                      (3, 4)],
+                                      pytest.param(3, 4)],
                          ids=["a", None])
         assert result == ["a", "3-4"]
 
+    def test_idmaker_with_paramset_id(self):
+        from _pytest.python import idmaker
+        result = idmaker(("a", "b"), [pytest.param(1, 2, id="me"),
+                                      pytest.param(3, 4, id="you")],
+                         ids=["a", None])
+        assert result == ["me", "you"]
+
     def test_idmaker_with_ids_unique_names(self):
         from _pytest.python import idmaker
-        result = idmaker(("a"), [1,2,3,4,5],
+        result = idmaker(("a"), map(pytest.param, [1,2,3,4,5]),
                          ids=["a", "a", "b", "c", "b"])
         assert result == ["a0", "a1", "b0", "c", "b1"]
 
@@ -1438,6 +1450,31 @@ class TestMarkersWithParametrization(object):
         reprec = testdir.inline_run()
         reprec.assertoutcome(passed=2)
 
+
+    @pytest.mark.parametrize('strict', [True, False])
+    def test_parametrize_marked_value(self, testdir, strict):
+        s = """
+            import pytest
+
+            @pytest.mark.parametrize(("n", "expected"), [
+                pytest.param(
+                    2,3,
+                    marks=pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict}),
+                ),
+                pytest.param(
+                    2,3,
+                    marks=[pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict})],
+                ),
+            ])
+            def test_increment(n, expected):
+                assert n + 1 == expected
+        """.format(strict=strict)
+        testdir.makepyfile(s)
+        reprec = testdir.inline_run()
+        passed, failed = (0, 2) if strict else (2, 0)
+        reprec.assertoutcome(passed=passed, failed=failed)
+
+
     def test_pytest_make_parametrize_id(self, testdir):
         testdir.makeconftest("""
             def pytest_make_parametrize_id(config, val):

testing/test_mark.py

@@ -1,8 +1,9 @@
 from __future__ import absolute_import, division, print_function
 import os
+import sys
 
-import py, pytest
+import pytest
-from _pytest.mark import MarkGenerator as Mark
+from _pytest.mark import MarkGenerator as Mark, ParameterSet
 
 class TestMark(object):
     def test_markinfo_repr(self):
@@ -10,9 +11,11 @@ class TestMark(object):
         m = MarkInfo(Mark("hello", (1,2), {}))
         repr(m)
 
-    def test_pytest_exists_in_namespace_all(self):
+    @pytest.mark.parametrize('attr', ['mark', 'param'])
-        assert 'mark' in py.test.__all__
+    @pytest.mark.parametrize('modulename', ['py.test', 'pytest'])
-        assert 'mark' in pytest.__all__
+    def test_pytest_exists_in_namespace_all(self, attr, modulename):
+        module = sys.modules[modulename]
+        assert attr in module.__all__
 
     def test_pytest_mark_notcallable(self):
         mark = Mark()
@@ -415,7 +418,7 @@ class TestFunctional(object):
         """)
         items, rec = testdir.inline_genitems(p)
         for item in items:
-            print (item, item.keywords)
+            print(item, item.keywords)
             assert 'a' in item.keywords
 
     def test_mark_decorator_subclass_does_not_propagate_to_base(self, testdir):
@@ -739,3 +742,16 @@ class TestKeywordSelection(object):
 
         assert_test_is_not_selected("__")
         assert_test_is_not_selected("()")
+
+
+@pytest.mark.parametrize('argval, expected', [
+    (pytest.mark.skip()((1, 2)),
+     ParameterSet(values=(1, 2), marks=[pytest.mark.skip], id=None)),
+    (pytest.mark.xfail(pytest.mark.skip()((1, 2))),
+     ParameterSet(values=(1, 2),
+                  marks=[pytest.mark.xfail, pytest.mark.skip], id=None)),
+
+])
+def test_parameterset_extractfrom(argval, expected):
+    extracted = ParameterSet.extract_from(argval)
+    assert extracted == expected