Merge pull request #2442 from nicoddemus/merge-master-into-features

Merge master into features

AUTHORS

@@ -13,8 +13,8 @@ Andreas Zeidler
 Andrzej Ostrowski
 Andy Freeland
 Anthon van der Neut
-Antony Lee
 Anthony Sottile
+Antony Lee
 Armin Rigo
 Aron Curzon
 Aviv Palivoda
@@ -70,6 +70,7 @@ Grig Gheorghiu
 Grigorii Eremeev (budulianin)
 Guido Wesdorp
 Harald Armin Massa
+Hui Wang (coldnight)
 Ian Bicking
 Jaap Broekhuizen
 Jan Balster
@@ -158,8 +159,8 @@ Trevor Bekolay
 Tyler Goodlet
 Vasily Kuznetsov
 Victor Uriarte
-Vlad Dragos
 Vidar T. Fauske
 Vitaly Lashmanov
+Vlad Dragos
 Wouter van Ackooy
 Xuecong Liao

CHANGELOG.rst

@@ -1,3 +1,11 @@
+3.1.1 (unreleased)
+==================
+
+* Fix encoding errors for unicode warnings in Python 2. (towncrier: 2436.bugfix)
+
+* Fix issue with non-ascii contents in doctest text files. (towncrier: 2434.bugfix)
+
+
 3.1.0 (2017-05-22)
 ==================
 
@@ -5,11 +13,25 @@
 New Features
 ------------
 
-* The ``pytest-warnings`` plugin has been integrated into the core, so now ``pytest`` automatically
+* The ``pytest-warnings`` plugin has been integrated into the core and now ``pytest`` automatically
   captures and displays warnings at the end of the test session.
+
+  .. warning::
+
+    This feature may disrupt test suites which apply and treat warnings themselves, and can be
+    disabled in your ``pytest.ini``:
+
+    .. code-block:: ini
+
+      [pytest]
+      addopts = -p no:warnings
+
+    See the `warnings documentation page <https://docs.pytest.org/en/latest/warnings.html>`_ for more
+    information.
+
   Thanks `@nicoddemus`_ for the PR.
 
-* Added ``junit_suite_name`` ini option to specify root `<testsuite>` name for JUnit XML reports (`#533`_).
+* Added ``junit_suite_name`` ini option to specify root ``<testsuite>`` name for JUnit XML reports (`#533`_).
 
 * Added an ini option ``doctest_encoding`` to specify which encoding to use for doctest files.
   Thanks `@wheerd`_ for the PR (`#2101`_).
@@ -53,8 +75,6 @@ Changes
 * ``--pdbcls`` no longer implies ``--pdb``. This makes it possible to use
   ``addopts=--pdbcls=module.SomeClass`` on ``pytest.ini``. Thanks `@davidszotten`_ for
   the PR (`#1952`_).
-* Change exception raised by ``capture.DontReadFromInput.fileno()`` from ``ValueError``
-  to ``io.UnsupportedOperation``. Thanks `@vlad-dragos`_ for the PR.
 
 * fix `#2013`_: turn RecordedWarning into ``namedtuple``,
   to give it a comprehensible repr while preventing unwarranted modification.
@@ -86,7 +106,7 @@ Changes
   Thanks `@RonnyPfannschmidt`_ for the PR.
 
 * fix `#2391`_: consider pytest_plugins on all plugin modules
-  Thansks `@RonnyPfannschmidt`_ for the PR.
+  Thanks `@RonnyPfannschmidt`_ for the PR.
 
 
 Bug Fixes
@@ -98,7 +118,7 @@ Bug Fixes
 
 * Change capture.py's ``DontReadFromInput`` class to throw ``io.UnsupportedOperation`` errors rather
   than ValueErrors in the ``fileno`` method (`#2276`_).
-  Thanks `@metasyn`_ for the PR.
+  Thanks `@metasyn`_ and `@vlad-dragos`_ for the PR.
 
 * Fix exception formatting while importing modules when the exception message
   contains non-ascii characters (`#2336`_).

_pytest/cacheprovider.py

@@ -219,7 +219,7 @@ def cacheshow(config, session):
     basedir = config.cache._cachedir
     vdir = basedir.join("v")
     tw.sep("-", "cache values")
-    for valpath in vdir.visit(lambda x: x.isfile()):
+    for valpath in sorted(vdir.visit(lambda x: x.isfile())):
         key = valpath.relto(vdir).replace(valpath.sep, "/")
         val = config.cache.get(key, dummy)
         if val is dummy:
@@ -235,7 +235,7 @@ def cacheshow(config, session):
     ddir = basedir.join("d")
     if ddir.isdir() and ddir.listdir():
         tw.sep("-", "cache directories")
-        for p in basedir.join("d").visit():
+        for p in sorted(basedir.join("d").visit()):
             #if p.check(dir=1):
             #    print("%s/" % p.relto(basedir))
             if p.isfile():

_pytest/compat.py

@@ -126,6 +126,7 @@ if _PY3:
     import codecs
     imap = map
     STRING_TYPES = bytes, str
+    UNICODE_TYPES = str,
 
     def _escape_strings(val):
         """If val is pure ascii, returns it as a str().  Otherwise, escapes
@@ -157,6 +158,7 @@ if _PY3:
             return val.encode('unicode_escape').decode('ascii')
 else:
     STRING_TYPES = bytes, str, unicode
+    UNICODE_TYPES = unicode,
 
     from itertools import imap  # NOQA
 

_pytest/doctest.py

@@ -181,6 +181,7 @@ class DoctestTextfile(pytest.Module):
         optionflags = get_optionflags(self)
         runner = doctest.DebugRunner(verbose=0, optionflags=optionflags,
                                      checker=_get_checker())
+        _fix_spoof_python2(runner, encoding)
 
         parser = doctest.DocTestParser()
         test = parser.get_doctest(text, globs, name, filename, 0)
@@ -216,6 +217,10 @@ class DoctestModule(pytest.Module):
         optionflags = get_optionflags(self)
         runner = doctest.DebugRunner(verbose=0, optionflags=optionflags,
                                      checker=_get_checker())
+
+        encoding = self.config.getini("doctest_encoding")
+        _fix_spoof_python2(runner, encoding)
+        
         for test in finder.find(module, module.__name__):
             if test.examples:  # skip empty doctests
                 yield DoctestItem(test.name, self, runner, test)
@@ -324,6 +329,30 @@ def _get_report_choice(key):
         DOCTEST_REPORT_CHOICE_NONE: 0,
     }[key]
 
+
+def _fix_spoof_python2(runner, encoding):
+    """
+    Installs a "SpoofOut" into the given DebugRunner so it properly deals with unicode output.
+    
+    This fixes the problem related in issue #2434.
+    """
+    from _pytest.compat import _PY2
+    if not _PY2:
+        return
+
+    from doctest import _SpoofOut
+
+    class UnicodeSpoof(_SpoofOut):
+
+        def getvalue(self):
+            result = _SpoofOut.getvalue(self)
+            if encoding:
+                result = result.decode(encoding)
+            return result
+
+    runner._fakeout = UnicodeSpoof()
+
+
 @pytest.fixture(scope='session')
 def doctest_namespace():
     """

_pytest/warnings.py

@@ -5,6 +5,8 @@ from contextlib import contextmanager
 
 import pytest
 
+from _pytest import compat
+
 
 def _setoption(wmod, arg):
     """
@@ -61,11 +63,24 @@ def catch_warnings_for_item(item):
         yield
 
         for warning in log:
+            warn_msg = warning.message
+            unicode_warning = False
+
+            if compat._PY2 and any(isinstance(m, compat.UNICODE_TYPES) for m in warn_msg.args):
+                warn_msg.args = [compat.safe_str(m) for m in warn_msg.args]
+                unicode_warning = True
+
             msg = warnings.formatwarning(
-                warning.message, warning.category,
+                warn_msg, warning.category,
                 warning.filename, warning.lineno, warning.line)
             item.warn("unused", msg)
 
+            if unicode_warning:
+                warnings.warn(
+                    "This warning %s is broken as it's message is not a str instance"
+                    "(after all this is a stdlib problem workaround)" % msg,
+                    UnicodeWarning)
+
 
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_protocol(item):

doc/en/cache.rst

@@ -231,10 +231,10 @@ You can always peek at the content of the cache using the
     rootdir: $REGENDOC_TMPDIR, inifile:
     cachedir: $REGENDOC_TMPDIR/.cache
     ------------------------------- cache values -------------------------------
-    example/value contains:
-      42
     cache/lastfailed contains:
       {'test_caching.py::test_function': True}
+    example/value contains:
+      42
     
     ======= no tests ran in 0.12 seconds ========
 

doc/en/example/parametrize.rst

@@ -116,6 +116,15 @@ the argument name::
         diff = a - b
         assert diff == expected
 
+    @pytest.mark.parametrize("a,b,expected", [
+        pytest.param(datetime(2001, 12, 12), datetime(2001, 12, 11),
+                     timedelta(1), id='forward'),
+        pytest.param(datetime(2001, 12, 11), datetime(2001, 12, 12),
+                     timedelta(-1), id='backward'),
+    ])
+    def test_timedistance_v3(a, b, expected):
+        diff = a - b
+        assert diff == expected
 
 In ``test_timedistance_v0``, we let pytest generate the test IDs.
 
@@ -132,7 +141,7 @@ objects, they are still using the default pytest representation::
     ======= test session starts ========
     platform linux -- Python 3.x.y, pytest-3.x.y, py-1.x.y, pluggy-0.x.y
     rootdir: $REGENDOC_TMPDIR, inifile:
-    collected 6 items
+    collected 8 items
     <Module 'test_time.py'>
       <Function 'test_timedistance_v0[a0-b0-expected0]'>
       <Function 'test_timedistance_v0[a1-b1-expected1]'>
@@ -140,9 +149,14 @@ objects, they are still using the default pytest representation::
       <Function 'test_timedistance_v1[backward]'>
       <Function 'test_timedistance_v2[20011212-20011211-expected0]'>
       <Function 'test_timedistance_v2[20011211-20011212-expected1]'>
+      <Function 'test_timedistance_v3[forward]'>
+      <Function 'test_timedistance_v3[backward]'>
     
     ======= no tests ran in 0.12 seconds ========
 
+In ``test_timedistance_v3``, we used ``pytest.param`` to specify the test IDs
+together with the actual data, instead of listing them separately.
+
 A quick port of "testscenarios"
 ------------------------------------
 

doc/en/warnings.rst

@@ -5,6 +5,20 @@ Warnings Capture
 
 .. versionadded:: 3.1
 
+.. warning::
+    pytest captures all warnings between tests, which prevents custom warning
+    filters in existing test suites from working. If this causes problems to your test suite,
+    this plugin can be disabled in your ``pytest.ini`` file with:
+
+    .. code-block:: ini
+
+        [pytest]
+        addopts = -p no:warnings
+
+    There's an ongoing discussion about this on `#2430
+    <https://github.com/pytest-dev/pytest/issues/2430>`_.
+
+
 Starting from version ``3.1``, pytest now automatically catches all warnings during test execution
 and displays them at the end of the session::
 

testing/test_doctest.py

@@ -505,6 +505,28 @@ class TestDoctests(object):
                                     "--junit-xml=junit.xml")
         reprec.assertoutcome(failed=1)
 
+    def test_unicode_doctest(self, testdir):
+        """
+        Test case for issue 2434: DecodeError on Python 2 when doctest contains non-ascii
+        characters.
+        """
+        p = testdir.maketxtfile(test_unicode_doctest="""
+            .. doctest::
+
+                >>> print(
+                ...    "Hi\\n\\nByé")
+                Hi
+                ...
+                Byé
+                >>> 1/0  # Byé
+                1
+        """)
+        result = testdir.runpytest(p)
+        result.stdout.fnmatch_lines([
+            '*UNEXPECTED EXCEPTION: ZeroDivisionError*',
+            '*1 failed*',
+        ])
+
 
 class TestLiterals(object):
 

testing/test_warnings.py

@@ -1,3 +1,8 @@
+# -*- coding: utf8 -*-
+from __future__ import unicode_literals
+
+import sys
+
 import pytest
 
 
@@ -106,3 +111,58 @@ def test_ignore(testdir, pyfile_with_warnings, method):
     ])
     assert WARNINGS_SUMMARY_HEADER not in result.stdout.str()
 
+
+
+@pytest.mark.skipif(sys.version_info < (3, 0),
+                    reason='warnings message is unicode is ok in python3')
+def test_unicode(testdir, pyfile_with_warnings):
+    testdir.makepyfile('''
+        # -*- coding: utf8 -*-
+        import warnings
+        import pytest
+
+
+        @pytest.fixture
+        def fix():
+            warnings.warn(u"测试")
+            yield
+
+        def test_func(fix):
+            pass
+    ''')
+    result = testdir.runpytest()
+    result.stdout.fnmatch_lines([
+        '*== %s ==*' % WARNINGS_SUMMARY_HEADER,
+
+        '*test_unicode.py:8: UserWarning: \u6d4b\u8bd5',
+        '*warnings.warn(u"\u6d4b\u8bd5")',
+        '* 1 passed, 1 warnings*',
+    ])
+
+
+@pytest.mark.skipif(sys.version_info >= (3, 0),
+                    reason='warnings message is broken as it is not str instance')
+def test_py2_unicode(testdir, pyfile_with_warnings):
+    testdir.makepyfile('''
+        # -*- coding: utf8 -*-
+        import warnings
+        import pytest
+
+
+        @pytest.fixture
+        def fix():
+            warnings.warn(u"测试")
+            yield
+
+        def test_func(fix):
+            pass
+    ''')
+    result = testdir.runpytest()
+    result.stdout.fnmatch_lines([
+        '*== %s ==*' % WARNINGS_SUMMARY_HEADER,
+
+        '*test_py2_unicode.py:8: UserWarning: \u6d4b\u8bd5',
+        '*warnings.warn(u"\u6d4b\u8bd5")',
+        '*warnings.py:82: UnicodeWarning: This warning*\u6d4b\u8bd5',
+        '* 1 passed, 2 warnings*',
+    ])

tox.ini

@@ -46,6 +46,8 @@ commands=
 
 
 [testenv:linting]
+skipsdist=True
+usedevelop=True
 basepython = python2.7
 # needed to keep check-manifest working
 setenv =