Merge pull request #1031 from pytest-dev/unmarshable-parametrize

Parametrized values containing non-ascii bytes break cache

CHANGELOG

@@ -9,6 +9,13 @@
 - Fix issue #766 by removing documentation references to distutils.
   Thanks Russel Winder.
 
+- Fix issue #1030: now byte-strings are escaped to produce item node ids
+  to make them always serializable.
+  Thanks Andy Freeland for the report and Bruno Oliveira for the PR.
+
+- Python 2: if unicode parametrized values are convertible to ascii, their
+  ascii representation is used for the node id.
+
 - Fix issue #411: Add __eq__ method to assertion comparison example.
   Thanks Ben Webb.
 

_pytest/python.py

@@ -32,6 +32,9 @@ exc_clear = getattr(sys, 'exc_clear', lambda: None)
 # The type of re.compile objects is not exposed in Python.
 REGEX_TYPE = type(re.compile(''))
 
+_PY3 = sys.version_info > (3, 0)
+_PY2 = not _PY3
+
 
 if hasattr(inspect, 'signature'):
     def _format_args(func):
@@ -1037,6 +1040,35 @@ class Metafunc(FuncargnamesCompatAttr):
         self._calls.append(cs)
 
 
+if _PY3:
+    def _escape_bytes(val):
+        """
+        If val is pure ascii, returns it as a str(), otherwise escapes
+        into a sequence of escaped bytes:
+        b'\xc3\xb4\xc5\xd6' -> u'\\xc3\\xb4\\xc5\\xd6'
+
+        note:
+           the obvious "v.decode('unicode-escape')" will return
+           valid utf-8 unicode if it finds them in the string, but we
+           want to return escaped bytes for any byte, even if they match
+           a utf-8 string.
+        """
+        # source: http://goo.gl/bGsnwC
+        import codecs
+        encoded_bytes, _ = codecs.escape_encode(val)
+        return encoded_bytes.decode('ascii')
+else:
+    def _escape_bytes(val):
+        """
+        In py2 bytes and str are the same, so return it unchanged if it
+        is a full ascii string, otherwise escape it into its binary form.
+        """
+        try:
+            return val.encode('ascii')
+        except UnicodeDecodeError:
+            return val.encode('string-escape')
+
+
 def _idval(val, argname, idx, idfn):
     if idfn:
         try:
@@ -1046,7 +1078,9 @@ def _idval(val, argname, idx, idfn):
         except Exception:
             pass
 
-    if isinstance(val, (float, int, str, bool, NoneType)):
+    if isinstance(val, bytes):
+        return _escape_bytes(val)
+    elif isinstance(val, (float, int, str, bool, NoneType)):
         return str(val)
     elif isinstance(val, REGEX_TYPE):
         return val.pattern
@@ -1054,6 +1088,14 @@ def _idval(val, argname, idx, idfn):
         return str(val)
     elif isclass(val) and hasattr(val, '__name__'):
         return val.__name__
+    elif _PY2 and isinstance(val, unicode):
+        # special case for python 2: if a unicode string is
+        # convertible to ascii, return it as an str() object instead
+        try:
+            return str(val)
+        except UnicodeDecodeError:
+            # fallthrough
+            pass
     return str(argname)+str(idx)
 
 def _idvalset(idx, valset, argnames, idfn):

testing/python/metafunc.py

@@ -129,11 +129,12 @@ class TestMetafunc:
                                       (object(), object())])
         assert result == ["a0-1.0", "a1-b1"]
         # unicode mixing, issue250
-        result = idmaker((py.builtin._totext("a"), "b"), [({}, '\xc3\xb4')])
-        assert result == ['a0-\xc3\xb4']
+        result = idmaker((py.builtin._totext("a"), "b"), [({}, b'\xc3\xb4')])
+        assert result == ['a0-\\xc3\\xb4']
 
     def test_idmaker_native_strings(self):
         from _pytest.python import idmaker
+        totext = py.builtin._totext
         result = idmaker(("a", "b"), [(1.0, -1.1),
                                       (2, -202),
                                       ("three", "three hundred"),
@@ -143,7 +144,9 @@ class TestMetafunc:
                                       (str, int),
                                       (list("six"), [66, 66]),
                                       (set([7]), set("seven")),
-                                      (tuple("eight"), (8, -8, 8))
+                                      (tuple("eight"), (8, -8, 8)),
+                                      (b'\xc3\xb4', b"name"),
+                                      (b'\xc3\xb4', totext("other")),
         ])
         assert result == ["1.0--1.1",
                           "2--202",
@@ -154,7 +157,10 @@ class TestMetafunc:
                           "str-int",
                           "a7-b7",
                           "a8-b8",
-                          "a9-b9"]
+                          "a9-b9",
+                          "\\xc3\\xb4-name",
+                          "\\xc3\\xb4-other",
+                          ]
 
     def test_idmaker_enum(self):
         from _pytest.python import idmaker
@@ -312,7 +318,6 @@ class TestMetafunc:
             "*uses no fixture 'y'*",
         ])
 
-    @pytest.mark.xfail
     @pytest.mark.issue714
     def test_parametrize_uses_no_fixture_error_indirect_true(self, testdir):
         testdir.makepyfile("""
@@ -333,7 +338,6 @@ class TestMetafunc:
             "*uses no fixture 'y'*",
         ])
 
-    @pytest.mark.xfail
     @pytest.mark.issue714
     def test_parametrize_indirect_uses_no_fixture_error_indirect_list(self, testdir):
         testdir.makepyfile("""

testing/test_cache.py

@@ -269,6 +269,22 @@ class TestLastFailed:
         lastfailed = config.cache.get("cache/lastfailed", -1)
         assert not lastfailed
 
+    def test_non_serializable_parametrize(self, testdir):
+        """Test that failed parametrized tests with unmarshable parameters
+        don't break pytest-cache.
+        """
+        testdir.makepyfile(r"""
+            import pytest
+
+            @pytest.mark.parametrize('val', [
+                b'\xac\x10\x02G',
+            ])
+            def test_fail(val):
+                assert False
+        """)
+        result = testdir.runpytest()
+        result.stdout.fnmatch_lines('*1 failed in*')
+
     def test_lastfailed_collectfailure(self, testdir, monkeypatch):
 
         testdir.makepyfile(test_maybe="""