Merge pull request #4692 from nicoddemus/merge-master-into-features

Merge master into features

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 exclude: doc/en/example/py2py3/test_py2.py
 repos:
 -   repo: https://github.com/ambv/black
-    rev: 18.6b4
+    rev: 18.9b0
     hooks:
     -   id: black
         args: [--safe, --quiet]
@@ -13,7 +13,7 @@ repos:
         additional_dependencies: [black==18.9b0]
         language_version: python3
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.0.0
+    rev: v2.1.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
@@ -21,20 +21,23 @@ repos:
     -   id: debug-statements
         exclude: _pytest/debugging.py
         language_version: python3
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.7.0
+    hooks:
     -   id: flake8
         language_version: python3
 -   repo: https://github.com/asottile/reorder_python_imports
-    rev: v1.3.3
+    rev: v1.3.5
     hooks:
     -   id: reorder-python-imports
         args: ['--application-directories=.:src']
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v1.10.1
+    rev: v1.11.1
     hooks:
     -   id: pyupgrade
         args: [--keep-percent-format]
 -   repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.1.0
+    rev: v1.2.0
     hooks:
     -   id: rst-backticks
 -   repo: local
@@ -51,3 +54,17 @@ repos:
         entry: 'changelog files must be named ####.(feature|bugfix|doc|deprecation|removal|vendor|trivial).rst'
         exclude: changelog/(\d+\.(feature|bugfix|doc|deprecation|removal|vendor|trivial).rst|README.rst|_template.rst)
         files: ^changelog/
+    -   id: py-deprecated
+        name: py library is deprecated
+        language: pygrep
+        entry: >
+            (?x)\bpy\.(
+                _code\.|
+                builtin\.|
+                code\.|
+                io\.(BytesIO|saferepr)|
+                path\.local\.sysfind|
+                process\.|
+                std\.
+            )
+        types: [python]

AUTHORS

@@ -7,6 +7,7 @@ Aaron Coleman
 Abdeali JK
 Abhijeet Kasurde
 Adam Johnson
+Adam Uhlir
 Ahn Ki-Wook
 Alan Velasco
 Alexander Johnson
@@ -52,6 +53,7 @@ Christian Boelsen
 Christian Theunert
 Christian Tismer
 Christopher Gilling
+Christopher Dignam
 CrazyMerlyn
 Cyrus Maden
 Dhiren Serai

changelog/4402.bugfix.rst

@@ -0,0 +1,4 @@
+Warning summary now groups warnings by message instead of by test id.
+
+This makes the output more compact and better conveys the general idea of how much code is
+actually generating warnings, instead of how many tests call that code.

changelog/4536.bugfix.rst

@@ -0,0 +1 @@
+``monkeypatch.delattr`` handles class descriptors like ``staticmethod``/``classmethod``.

changelog/4649.bugfix.rst

@@ -0,0 +1 @@
+Restore marks being considered keywords for keyword expressions.

changelog/4653.bugfix.rst

@@ -0,0 +1 @@
+``tmp_path`` fixture and other related ones provides resolved path (a.k.a real path)

changelog/4657.trivial.rst

@@ -0,0 +1 @@
+Copy saferepr from pylib

changelog/4667.bugfix.rst

@@ -0,0 +1 @@
+``pytest_terminal_summary`` uses result from ``pytest_report_teststatus`` hook, rather than hardcoded strings.

changelog/4669.bugfix.rst

@@ -0,0 +1 @@
+Correctly handle ``unittest.SkipTest`` exception containing non-ascii characters on Python 2.

doc/en/contents.rst

@@ -41,6 +41,7 @@ Full pytest documentation
 
    backwards-compatibility
    deprecations
+   py27-py34-deprecation
    historical-notes
    license
    contributing

doc/en/deprecations.rst

@@ -25,11 +25,32 @@ Below is a complete list of all pytest features which are considered deprecated.
 .. deprecated:: 4.1
 
 It is a common mistake to think this parameter will match the exception message, while in fact
-it only serves to provide a custom message in case the ``pytest.raises`` check fails. To avoid this
+it only serves to provide a custom message in case the ``pytest.raises`` check fails. To prevent
-mistake and because it is believed to be little used, pytest is deprecating it without providing
+users from making this mistake, and because it is believed to be little used, pytest is
-an alternative for the moment.
+deprecating it without providing an alternative for the moment.
 
-If you have concerns about this, please comment on `issue #3974 <https://github.com/pytest-dev/pytest/issues/3974>`__.
+If you have a valid use case for this parameter, consider that to obtain the same results
+you can just call ``pytest.fail`` manually at the end of the ``with`` statement.
+
+For example:
+
+.. code-block:: python
+
+    with pytest.raises(TimeoutError, message="Client got unexpected message"):
+        wait_for(websocket.recv(), 0.5)
+
+
+Becomes:
+
+.. code-block:: python
+
+    with pytest.raises(TimeoutError):
+        wait_for(websocket.recv(), 0.5)
+        pytest.fail("Client got unexpected message")
+
+
+If you still have concerns about this deprecation and future removal, please comment on
+`issue #3974 <https://github.com/pytest-dev/pytest/issues/3974>`__.
 
 
 ``pytest.config`` global

doc/en/example/attic.rst

@@ -24,10 +24,10 @@ example: specifying and selecting acceptance tests
                 pytest.skip("specify -A to run acceptance tests")
             self.tmpdir = request.config.mktemp(request.function.__name__, numbered=True)
 
-        def run(self, cmd):
+        def run(self, *cmd):
             """ called by test code to execute an acceptance test. """
             self.tmpdir.chdir()
-            return py.process.cmdexec(cmd)
+            return subprocess.check_output(cmd).decode()
 
 
 and the actual test function example:
@@ -36,7 +36,7 @@ and the actual test function example:
 
     def test_some_acceptance_aspect(accept):
         accept.tmpdir.mkdir("somesub")
-        result = accept.run("ls -la")
+        result = accept.run("ls", "-la")
         assert "somesub" in result
 
 If you run this test without specifying a command line option

doc/en/example/multipython.py

@@ -2,10 +2,10 @@
 module containing a parametrized tests testing cross-python
 serialization via the pickle module.
 """
+import distutils.spawn
+import subprocess
 import textwrap
 
-import py
-
 import pytest
 
 pythonlist = ["python2.7", "python3.4", "python3.5"]
@@ -24,7 +24,7 @@ def python2(request, python1):
 
 class Python(object):
     def __init__(self, version, picklefile):
-        self.pythonpath = py.path.local.sysfind(version)
+        self.pythonpath = distutils.spawn.find_executable(version)
         if not self.pythonpath:
             pytest.skip("{!r} not found".format(version))
         self.picklefile = picklefile
@@ -43,7 +43,7 @@ class Python(object):
                 )
             )
         )
-        py.process.cmdexec("{} {}".format(self.pythonpath, dumpfile))
+        subprocess.check_call((self.pythonpath, str(dumpfile)))
 
     def load_and_is_true(self, expression):
         loadfile = self.picklefile.dirpath("load.py")
@@ -63,7 +63,7 @@ class Python(object):
             )
         )
         print(loadfile)
-        py.process.cmdexec("{} {}".format(self.pythonpath, loadfile))
+        subprocess.check_call((self.pythonpath, str(loadfile)))
 
 
 @pytest.mark.parametrize("obj", [42, {}, {1: 3}])

doc/en/py27-py34-deprecation.rst

@@ -0,0 +1,22 @@
+Python 2.7 and 3.4 support plan
+===============================
+
+Python 2.7 EOL is fast approaching, with
+upstream support `ending in 2020 <https://legacy.python.org/dev/peps/pep-0373/#id4>`__.
+Python 3.4's last release is scheduled for
+`March 2019 <https://www.python.org/dev/peps/pep-0429/#release-schedule>`__. pytest is one of
+the participating projects of the https://python3statement.org.
+
+We plan to drop support for Python 2.7 and 3.4 at the same time with the release of **pytest 5.0**,
+scheduled to be released by **mid-2019**. Thanks to the `python_requires <https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires>`__ ``setuptools`` option,
+Python 2.7 and Python 3.4 users using a modern ``pip`` version
+will install the last compatible pytest ``4.X`` version automatically even if ``5.0`` or later
+are available on PyPI.
+
+During the period **from mid-2019 and 2020**, the pytest core team plans to make
+bug-fix releases of the pytest ``4.X`` series by back-porting patches to the ``4.x-maintenance``
+branch.
+
+**After 2020**, the core team will no longer actively back port-patches, but the ``4.x-maintenance``
+branch will continue to exist so the community itself can contribute patches. The
+core team will be happy to accept those patches and make new ``4.X`` releases **until mid-2020**.

setup.cfg

@@ -36,10 +36,11 @@ platforms = unix, linux, osx, cygwin, win32
 zip_safe = no
 packages =
     _pytest
-    _pytest.assertion
     _pytest._code
-    _pytest.mark
+    _pytest._io
+    _pytest.assertion
     _pytest.config
+    _pytest.mark
 
 py_modules = pytest
 python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*

src/_pytest/_code/code.py

@@ -18,6 +18,7 @@ import six
 from six import text_type
 
 import _pytest
+from _pytest._io.saferepr import saferepr
 from _pytest.compat import _PY2
 from _pytest.compat import _PY3
 from _pytest.compat import PY35
@@ -142,7 +143,7 @@ class Frame(object):
     def repr(self, object):
         """ return a 'safe' (non-recursive, one-line) string repr for 'object'
         """
-        return py.io.saferepr(object)
+        return saferepr(object)
 
     def is_true(self, object):
         return object
@@ -421,7 +422,7 @@ class ExceptionInfo(object):
         if exprinfo is None and isinstance(tup[1], AssertionError):
             exprinfo = getattr(tup[1], "msg", None)
             if exprinfo is None:
-                exprinfo = py.io.saferepr(tup[1])
+                exprinfo = saferepr(tup[1])
             if exprinfo and exprinfo.startswith(cls._assert_start_repr):
                 _striptext = "AssertionError: "
 
@@ -618,7 +619,7 @@ class FormattedExcinfo(object):
         return source
 
     def _saferepr(self, obj):
-        return py.io.saferepr(obj)
+        return saferepr(obj)
 
     def repr_args(self, entry):
         if self.funcargs:

src/_pytest/_code/source.py

@@ -237,9 +237,7 @@ def getfslineno(obj):
 def findsource(obj):
     try:
         sourcelines, lineno = inspect.findsource(obj)
-    except py.builtin._sysex:
+    except Exception:
-        raise
-    except:  # noqa
         return None, -1
     source = Source()
     source.lines = [line.rstrip() for line in sourcelines]

src/_pytest/_io/saferepr.py

@@ -0,0 +1,72 @@
+import sys
+
+from six.moves import reprlib
+
+
+class SafeRepr(reprlib.Repr):
+    """subclass of repr.Repr that limits the resulting size of repr()
+    and includes information on exceptions raised during the call.
+    """
+
+    def repr(self, x):
+        return self._callhelper(reprlib.Repr.repr, self, x)
+
+    def repr_unicode(self, x, level):
+        # Strictly speaking wrong on narrow builds
+        def repr(u):
+            if "'" not in u:
+                return u"'%s'" % u
+            elif '"' not in u:
+                return u'"%s"' % u
+            else:
+                return u"'%s'" % u.replace("'", r"\'")
+
+        s = repr(x[: self.maxstring])
+        if len(s) > self.maxstring:
+            i = max(0, (self.maxstring - 3) // 2)
+            j = max(0, self.maxstring - 3 - i)
+            s = repr(x[:i] + x[len(x) - j :])
+            s = s[:i] + "..." + s[len(s) - j :]
+        return s
+
+    def repr_instance(self, x, level):
+        return self._callhelper(repr, x)
+
+    def _callhelper(self, call, x, *args):
+        try:
+            # Try the vanilla repr and make sure that the result is a string
+            s = call(x, *args)
+        except Exception:
+            cls, e, tb = sys.exc_info()
+            exc_name = getattr(cls, "__name__", "unknown")
+            try:
+                exc_info = str(e)
+            except Exception:
+                exc_info = "unknown"
+            return '<[%s("%s") raised in repr()] %s object at 0x%x>' % (
+                exc_name,
+                exc_info,
+                x.__class__.__name__,
+                id(x),
+            )
+        else:
+            if len(s) > self.maxsize:
+                i = max(0, (self.maxsize - 3) // 2)
+                j = max(0, self.maxsize - 3 - i)
+                s = s[:i] + "..." + s[len(s) - j :]
+            return s
+
+
+def saferepr(obj, maxsize=240):
+    """return a size-limited safe repr-string for the given object.
+    Failing __repr__ functions of user instances will be represented
+    with a short exception info and 'saferepr' generally takes
+    care to never raise exceptions itself.  This function is a wrapper
+    around the Repr/reprlib functionality of the standard 2.6 lib.
+    """
+    # review exception handling
+    srepr = SafeRepr()
+    srepr.maxstring = maxsize
+    srepr.maxsize = maxsize
+    srepr.maxother = 160
+    return srepr.repr(obj)

src/_pytest/assertion/rewrite.py

@@ -19,6 +19,7 @@ import atomicwrites
 import py
 import six
 
+from _pytest._io.saferepr import saferepr
 from _pytest.assertion import util
 from _pytest.compat import spec_from_file_location
 from _pytest.pathlib import fnmatch_ex
@@ -471,7 +472,7 @@ def _saferepr(obj):
     JSON reprs.
 
     """
-    r = py.io.saferepr(obj)
+    r = saferepr(obj)
     # only occurs in python2.x, repr must return text in python3+
     if isinstance(r, bytes):
         # Represent unprintable bytes as `\x##`
@@ -490,7 +491,7 @@ def _format_assertmsg(obj):
 
     For strings this simply replaces newlines with '\n~' so that
     util.format_explanation() will preserve them instead of escaping
-    newlines.  For other objects py.io.saferepr() is used first.
+    newlines.  For other objects saferepr() is used first.
 
     """
     # reprlib appears to have a bug which means that if a string
@@ -499,7 +500,7 @@ def _format_assertmsg(obj):
     # However in either case we want to preserve the newline.
     replaces = [(u"\n", u"\n~"), (u"%", u"%%")]
     if not isinstance(obj, six.string_types):
-        obj = py.io.saferepr(obj)
+        obj = saferepr(obj)
         replaces.append((u"\\n", u"\n~"))
 
     if isinstance(obj, bytes):
@@ -665,7 +666,7 @@ class AssertionRewriter(ast.NodeVisitor):
         # Insert some special imports at the top of the module but after any
         # docstrings and __future__ imports.
         aliases = [
-            ast.alias(py.builtin.builtins.__name__, "@py_builtins"),
+            ast.alias(six.moves.builtins.__name__, "@py_builtins"),
             ast.alias("_pytest.assertion.rewrite", "@pytest_ar"),
         ]
         doc = getattr(mod, "docstring", None)
@@ -740,7 +741,7 @@ class AssertionRewriter(ast.NodeVisitor):
         return ast.Name(name, ast.Load())
 
     def display(self, expr):
-        """Call py.io.saferepr on the expression."""
+        """Call saferepr on the expression."""
         return self.helper("saferepr", expr)
 
     def helper(self, name, *args):

src/_pytest/assertion/util.py

@@ -5,11 +5,11 @@ from __future__ import print_function
 
 import pprint
 
-import py
 import six
 
 import _pytest._code
 from ..compat import Sequence
+from _pytest._io.saferepr import saferepr
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was
@@ -105,8 +105,8 @@ except NameError:
 def assertrepr_compare(config, op, left, right):
     """Return specialised explanations for some operators/operands"""
     width = 80 - 15 - len(op) - 2  # 15 chars indentation, 1 space around op
-    left_repr = py.io.saferepr(left, maxsize=int(width // 2))
+    left_repr = saferepr(left, maxsize=int(width // 2))
-    right_repr = py.io.saferepr(right, maxsize=width - len(left_repr))
+    right_repr = saferepr(right, maxsize=width - len(left_repr))
 
     summary = u"%s %s %s" % (ecu(left_repr), op, ecu(right_repr))
 
@@ -282,12 +282,12 @@ def _compare_eq_sequence(left, right, verbose=False):
     if len(left) > len(right):
         explanation += [
             u"Left contains more items, first extra item: %s"
-            % py.io.saferepr(left[len(right)])
+            % saferepr(left[len(right)])
         ]
     elif len(left) < len(right):
         explanation += [
             u"Right contains more items, first extra item: %s"
-            % py.io.saferepr(right[len(left)])
+            % saferepr(right[len(left)])
         ]
     return explanation
 
@@ -299,11 +299,11 @@ def _compare_eq_set(left, right, verbose=False):
     if diff_left:
         explanation.append(u"Extra items in the left set:")
         for item in diff_left:
-            explanation.append(py.io.saferepr(item))
+            explanation.append(saferepr(item))
     if diff_right:
         explanation.append(u"Extra items in the right set:")
         for item in diff_right:
-            explanation.append(py.io.saferepr(item))
+            explanation.append(saferepr(item))
     return explanation
 
 
@@ -320,9 +320,7 @@ def _compare_eq_dict(left, right, verbose=False):
     if diff:
         explanation += [u"Differing items:"]
         for k in diff:
-            explanation += [
+            explanation += [saferepr({k: left[k]}) + " != " + saferepr({k: right[k]})]
-                py.io.saferepr({k: left[k]}) + " != " + py.io.saferepr({k: right[k]})
-            ]
     extra_left = set(left) - set(right)
     if extra_left:
         explanation.append(u"Left contains more items:")
@@ -376,7 +374,7 @@ def _notin_text(term, text, verbose=False):
     tail = text[index + len(term) :]
     correct_text = head + tail
     diff = _diff_text(correct_text, text, verbose)
-    newdiff = [u"%s is contained here:" % py.io.saferepr(term, maxsize=42)]
+    newdiff = [u"%s is contained here:" % saferepr(term, maxsize=42)]
     for line in diff:
         if line.startswith(u"Skipping"):
             continue

src/_pytest/compat.py

@@ -17,6 +17,7 @@ import six
 from six import text_type
 
 import _pytest
+from _pytest._io.saferepr import saferepr
 from _pytest.outcomes import fail
 from _pytest.outcomes import TEST_OUTCOME
 
@@ -294,7 +295,7 @@ def get_real_func(obj):
     else:
         raise ValueError(
             ("could not find real function of {start}\nstopped at {current}").format(
-                start=py.io.saferepr(start_obj), current=py.io.saferepr(obj)
+                start=saferepr(start_obj), current=saferepr(obj)
             )
         )
     if isinstance(obj, functools.partial):

src/_pytest/config/__init__.py

@@ -437,7 +437,7 @@ class PytestPluginManager(PluginManager):
                     and not self._using_pyargs
                 ):
                     from _pytest.deprecated import (
-                        PYTEST_PLUGINS_FROM_NON_TOP_LEVEL_CONFTEST
+                        PYTEST_PLUGINS_FROM_NON_TOP_LEVEL_CONFTEST,
                     )
 
                     fail(

src/_pytest/fixtures.py

@@ -14,10 +14,10 @@ import attr
 import py
 import six
 from more_itertools import flatten
-from py._code.code import FormattedExcinfo
 
 import _pytest
 from _pytest import nodes
+from _pytest._code.code import FormattedExcinfo
 from _pytest._code.code import TerminalRepr
 from _pytest.compat import _format_args
 from _pytest.compat import _PytestWrapper
@@ -307,8 +307,8 @@ class FuncFixtureInfo(object):
     # fixture names specified via usefixtures and via autouse=True in fixture
     # definitions.
     initialnames = attr.ib(type=tuple)
-    names_closure = attr.ib()  # type: List[str]
+    names_closure = attr.ib()  # List[str]
-    name2fixturedefs = attr.ib()  # type: List[str, List[FixtureDef]]
+    name2fixturedefs = attr.ib()  # List[str, List[FixtureDef]]
 
     def prune_dependency_tree(self):
         """Recompute names_closure from initialnames and name2fixturedefs

src/_pytest/junitxml.py

@@ -244,7 +244,7 @@ class _NodeReporter(object):
         self._add_simple(Junit.skipped, "collection skipped", report.longrepr)
 
     def append_error(self, report):
-        if getattr(report, "when", None) == "teardown":
+        if report.when == "teardown":
             msg = "test teardown failure"
         else:
             msg = "test setup failure"

src/_pytest/logging.py

@@ -217,7 +217,7 @@ class LogCaptureFixture(object):
         """Creates a new funcarg."""
         self._item = item
         # dict of log name -> log level
-        self._initial_log_levels = {}  # type: Dict[str, int]
+        self._initial_log_levels = {}  # Dict[str, int]
 
     def _finalize(self):
         """Finalizes the fixture.

src/_pytest/mark/legacy.py

@@ -45,13 +45,14 @@ class KeywordMapping(object):
                 mapped_names.add(item.name)
 
         # Add the names added as extra keywords to current or parent items
-        for name in item.listextrakeywords():
+        mapped_names.update(item.listextrakeywords())
-            mapped_names.add(name)
 
         # Add the names attached to the current function through direct assignment
         if hasattr(item, "function"):
-            for name in item.function.__dict__:
+            mapped_names.update(item.function.__dict__)
-                mapped_names.add(name)
+
+        # add the markers to the keywords as we no longer handle them correctly
+        mapped_names.update(mark.name for mark in item.iter_markers())
 
         return cls(mapped_names)
 

src/_pytest/mark/structures.py

@@ -143,9 +143,9 @@ class Mark(object):
     #: name of the mark
     name = attr.ib(type=str)
     #: positional arguments of the mark decorator
-    args = attr.ib()  # type: List[object]
+    args = attr.ib()  # List[object]
     #: keyword arguments of the mark decorator
-    kwargs = attr.ib()  # type: Dict[str, object]
+    kwargs = attr.ib()  # Dict[str, object]
 
     def combined_with(self, other):
         """

src/_pytest/monkeypatch.py

@@ -181,6 +181,8 @@ class MonkeyPatch(object):
         attribute is missing.
         """
         __tracebackhide__ = True
+        import inspect
+
         if name is notset:
             if not isinstance(target, six.string_types):
                 raise TypeError(
@@ -194,7 +196,11 @@ class MonkeyPatch(object):
             if raising:
                 raise AttributeError(name)
         else:
-            self._setattr.append((target, name, getattr(target, name, notset)))
+            oldval = getattr(target, name, notset)
+            # Avoid class descriptors like staticmethod/classmethod.
+            if inspect.isclass(target):
+                oldval = target.__dict__.get(name, notset)
+            self._setattr.append((target, name, oldval))
             delattr(target, name)
 
     def setitem(self, dic, name, value):

src/_pytest/nose.py

@@ -5,6 +5,8 @@ from __future__ import print_function
 
 import sys
 
+import six
+
 from _pytest import python
 from _pytest import runner
 from _pytest import unittest
@@ -24,7 +26,7 @@ def pytest_runtest_makereport(item, call):
     if call.excinfo and call.excinfo.errisinstance(get_skip_exceptions()):
         # let's substitute the excinfo with a pytest.skip one
         call2 = runner.CallInfo.from_call(
-            lambda: runner.skip(str(call.excinfo.value)), call.when
+            lambda: runner.skip(six.text_type(call.excinfo.value)), call.when
         )
         call.excinfo = call2.excinfo
 

src/_pytest/pytester.py

@@ -4,6 +4,7 @@ from __future__ import division
 from __future__ import print_function
 
 import codecs
+import distutils.spawn
 import gc
 import os
 import platform
@@ -20,6 +21,7 @@ import six
 
 import pytest
 from _pytest._code import Source
+from _pytest._io.saferepr import saferepr
 from _pytest.assertion.rewrite import AssertionRewritingHook
 from _pytest.capture import MultiCapture
 from _pytest.capture import SysCapture
@@ -79,7 +81,7 @@ class LsofFdLeakChecker(object):
 
     def _exec_lsof(self):
         pid = os.getpid()
-        return py.process.cmdexec("lsof -Ffn0 -p %d" % pid)
+        return subprocess.check_output(("lsof", "-Ffn0", "-p", str(pid))).decode()
 
     def _parse_lsof_output(self, out):
         def isopen(line):
@@ -106,11 +108,8 @@ class LsofFdLeakChecker(object):
 
     def matching_platform(self):
         try:
-            py.process.cmdexec("lsof -v")
+            subprocess.check_output(("lsof", "-v"))
-        except (py.process.cmdexec.Error, UnicodeDecodeError):
+        except (OSError, subprocess.CalledProcessError):
-            # cmdexec may raise UnicodeDecodeError on Windows systems with
-            # locale other than English:
-            # https://bitbucket.org/pytest-dev/py/issues/66
             return False
         else:
             return True
@@ -152,7 +151,7 @@ def getexecutable(name, cache={}):
     try:
         return cache[name]
     except KeyError:
-        executable = py.path.local.sysfind(name)
+        executable = distutils.spawn.find_executable(name)
         if executable:
             import subprocess
 
@@ -306,13 +305,10 @@ class HookRecorder(object):
         """return a testreport whose dotted import path matches"""
         values = []
         for rep in self.getreports(names=names):
-            try:
+            if not when and rep.when != "call" and rep.passed:
-                if not when and rep.when != "call" and rep.passed:
+                # setup/teardown passing reports - let's ignore those
-                    # setup/teardown passing reports - let's ignore those
+                continue
-                    continue
+            if when and rep.when != when:
-            except AttributeError:
-                pass
-            if when and getattr(rep, "when", None) != when:
                 continue
             if not inamepart or inamepart in rep.nodeid.split("::"):
                 values.append(rep)
@@ -339,7 +335,7 @@ class HookRecorder(object):
         failed = []
         for rep in self.getreports("pytest_collectreport pytest_runtest_logreport"):
             if rep.passed:
-                if getattr(rep, "when", None) == "call":
+                if rep.when == "call":
                     passed.append(rep)
             elif rep.skipped:
                 skipped.append(rep)
@@ -1225,9 +1221,7 @@ def getdecoded(out):
     try:
         return out.decode("utf-8")
     except UnicodeDecodeError:
-        return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % (
+        return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % (saferepr(out),)
-            py.io.saferepr(out),
-        )
 
 
 class LineComp(object):

src/_pytest/python.py

@@ -1029,7 +1029,7 @@ class Metafunc(fixtures.FuncargnamesCompatAttr):
         :rtype: List[str]
         :return: the list of ids for each argname given
         """
-        from py.io import saferepr
+        from _pytest._io.saferepr import saferepr
 
         idfn = None
         if callable(ids):

src/_pytest/reports.py

@@ -19,6 +19,8 @@ def getslaveinfoline(node):
 
 
 class BaseReport(object):
+    when = None
+
     def __init__(self, **kw):
         self.__dict__.update(kw)
 
@@ -159,6 +161,8 @@ class TestReport(BaseReport):
 
 
 class CollectReport(BaseReport):
+    when = "collect"
+
     def __init__(self, nodeid, outcome, longrepr, result, sections=(), **extra):
         self.nodeid = nodeid
         self.outcome = outcome

src/_pytest/runner.py

@@ -204,7 +204,7 @@ class CallInfo(object):
     """ Result/Exception info a function invocation. """
 
     _result = attr.ib()
-    # type: Optional[ExceptionInfo]
+    # Optional[ExceptionInfo]
     excinfo = attr.ib()
     start = attr.ib()
     stop = attr.ib()

src/_pytest/skipping.py

@@ -180,9 +180,9 @@ def pytest_runtest_makereport(item, call):
 def pytest_report_teststatus(report):
     if hasattr(report, "wasxfail"):
         if report.skipped:
-            return "xfailed", "x", "xfail"
+            return "xfailed", "x", "XFAIL"
         elif report.passed:
-            return "xpassed", "X", ("XPASS", {"yellow": True})
+            return "xpassed", "X", "XPASS"
 
 
 # called by the terminalreporter instance/plugin
@@ -191,11 +191,6 @@ def pytest_report_teststatus(report):
 def pytest_terminal_summary(terminalreporter):
     tr = terminalreporter
     if not tr.reportchars:
-        # for name in "xfailed skipped failed xpassed":
-        #    if not tr.stats.get(name, 0):
-        #        tr.write_line("HINT: use '-r' option to see extra "
-        #              "summary info about tests")
-        #        break
         return
 
     lines = []
@@ -209,21 +204,23 @@ def pytest_terminal_summary(terminalreporter):
             tr._tw.line(line)
 
 
-def show_simple(terminalreporter, lines, stat, format):
+def show_simple(terminalreporter, lines, stat):
     failed = terminalreporter.stats.get(stat)
     if failed:
         for rep in failed:
+            verbose_word = _get_report_str(terminalreporter, rep)
             pos = terminalreporter.config.cwd_relative_nodeid(rep.nodeid)
-            lines.append(format % (pos,))
+            lines.append("%s %s" % (verbose_word, pos))
 
 
 def show_xfailed(terminalreporter, lines):
     xfailed = terminalreporter.stats.get("xfailed")
     if xfailed:
         for rep in xfailed:
+            verbose_word = _get_report_str(terminalreporter, rep)
             pos = terminalreporter.config.cwd_relative_nodeid(rep.nodeid)
             reason = rep.wasxfail
-            lines.append("XFAIL %s" % (pos,))
+            lines.append("%s %s" % (verbose_word, pos))
             if reason:
                 lines.append("  " + str(reason))
 
@@ -232,9 +229,10 @@ def show_xpassed(terminalreporter, lines):
     xpassed = terminalreporter.stats.get("xpassed")
     if xpassed:
         for rep in xpassed:
+            verbose_word = _get_report_str(terminalreporter, rep)
             pos = terminalreporter.config.cwd_relative_nodeid(rep.nodeid)
             reason = rep.wasxfail
-            lines.append("XPASS %s %s" % (pos, reason))
+            lines.append("%s %s %s" % (verbose_word, pos, reason))
 
 
 def folded_skips(skipped):
@@ -246,8 +244,11 @@ def folded_skips(skipped):
         # folding reports with global pytestmark variable
         # this is workaround, because for now we cannot identify the scope of a skip marker
         # TODO: revisit after marks scope would be fixed
-        when = getattr(event, "when", None)
+        if (
-        if when == "setup" and "skip" in keywords and "pytestmark" not in keywords:
+            event.when == "setup"
+            and "skip" in keywords
+            and "pytestmark" not in keywords
+        ):
             key = (key[0], None, key[2])
         d.setdefault(key, []).append(event)
     values = []
@@ -260,39 +261,42 @@ def show_skipped(terminalreporter, lines):
     tr = terminalreporter
     skipped = tr.stats.get("skipped", [])
     if skipped:
-        # if not tr.hasopt('skipped'):
+        verbose_word = _get_report_str(terminalreporter, report=skipped[0])
-        #    tr.write_line(
-        #        "%d skipped tests, specify -rs for more info" %
-        #        len(skipped))
-        #    return
         fskips = folded_skips(skipped)
         if fskips:
-            # tr.write_sep("_", "skipped test summary")
             for num, fspath, lineno, reason in fskips:
                 if reason.startswith("Skipped: "):
                     reason = reason[9:]
                 if lineno is not None:
                     lines.append(
-                        "SKIP [%d] %s:%d: %s" % (num, fspath, lineno + 1, reason)
+                        "%s [%d] %s:%d: %s"
+                        % (verbose_word, num, fspath, lineno + 1, reason)
                     )
                 else:
-                    lines.append("SKIP [%d] %s: %s" % (num, fspath, reason))
+                    lines.append("%s [%d] %s: %s" % (verbose_word, num, fspath, reason))
 
 
-def shower(stat, format):
+def shower(stat):
     def show_(terminalreporter, lines):
-        return show_simple(terminalreporter, lines, stat, format)
+        return show_simple(terminalreporter, lines, stat)
 
     return show_
 
 
+def _get_report_str(terminalreporter, report):
+    _category, _short, verbose = terminalreporter.config.hook.pytest_report_teststatus(
+        report=report, config=terminalreporter.config
+    )
+    return verbose
+
+
 REPORTCHAR_ACTIONS = {
     "x": show_xfailed,
     "X": show_xpassed,
-    "f": shower("failed", "FAIL %s"),
+    "f": shower("failed"),
-    "F": shower("failed", "FAIL %s"),
+    "F": shower("failed"),
     "s": show_skipped,
     "S": show_skipped,
-    "p": shower("passed", "PASSED %s"),
+    "p": shower("passed"),
-    "E": shower("error", "ERROR %s"),
+    "E": shower("error"),
 }

src/_pytest/terminal.py

@@ -7,7 +7,7 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
-import itertools
+import collections
 import platform
 import sys
 import time
@@ -376,8 +376,11 @@ class TerminalReporter(object):
             return
         running_xdist = hasattr(rep, "node")
         if markup is None:
-            if rep.passed:
+            was_xfail = hasattr(report, "wasxfail")
+            if rep.passed and not was_xfail:
                 markup = {"green": True}
+            elif rep.passed and was_xfail:
+                markup = {"yellow": True}
             elif rep.failed:
                 markup = {"red": True}
             elif rep.skipped:
@@ -728,33 +731,33 @@ class TerminalReporter(object):
 
             final = hasattr(self, "_already_displayed_warnings")
             if final:
-                warnings = all_warnings[self._already_displayed_warnings :]
+                warning_reports = all_warnings[self._already_displayed_warnings :]
             else:
-                warnings = all_warnings
+                warning_reports = all_warnings
-            self._already_displayed_warnings = len(warnings)
+            self._already_displayed_warnings = len(warning_reports)
-            if not warnings:
+            if not warning_reports:
                 return
 
-            grouped = itertools.groupby(
+            reports_grouped_by_message = collections.OrderedDict()
-                warnings, key=lambda wr: wr.get_location(self.config)
+            for wr in warning_reports:
-            )
+                reports_grouped_by_message.setdefault(wr.message, []).append(wr)
 
             title = "warnings summary (final)" if final else "warnings summary"
             self.write_sep("=", title, yellow=True, bold=False)
-            for location, warning_records in grouped:
+            for message, warning_reports in reports_grouped_by_message.items():
-                # legacy warnings show their location explicitly, while standard warnings look better without
+                has_any_location = False
-                # it because the location is already formatted into the message
+                for w in warning_reports:
-                warning_records = list(warning_records)
+                    location = w.get_location(self.config)
-                if location:
-                    self._tw.line(str(location))
-                for w in warning_records:
                     if location:
-                        lines = w.message.splitlines()
+                        self._tw.line(str(location))
-                        indented = "\n".join("  " + x for x in lines)
+                        has_any_location = True
-                        message = indented.rstrip()
+                if has_any_location:
-                    else:
+                    lines = message.splitlines()
-                        message = w.message.rstrip()
+                    indented = "\n".join("  " + x for x in lines)
-                    self._tw.line(message)
+                    message = indented.rstrip()
+                else:
+                    message = message.rstrip()
+                self._tw.line(message)
                 self._tw.line()
             self._tw.line("-- Docs: https://docs.pytest.org/en/latest/warnings.html")
 
@@ -824,8 +827,7 @@ class TerminalReporter(object):
             self.write_sep("=", "ERRORS")
             for rep in self.stats["error"]:
                 msg = self._getfailureheadline(rep)
-                if not hasattr(rep, "when"):
+                if rep.when == "collect":
-                    # collect
                     msg = "ERROR collecting " + msg
                 elif rep.when == "setup":
                     msg = "ERROR at setup of " + msg

testing/acceptance_test.py

@@ -804,8 +804,8 @@ class TestInvocationVariants(object):
         result = testdir.runpytest("-rf")
         lines = result.stdout.str().splitlines()
         for line in lines:
-            if line.startswith("FAIL "):
+            if line.startswith(("FAIL ", "FAILED ")):
-                testid = line[5:].strip()
+                _fail, _sep, testid = line.partition(" ")
                 break
         result = testdir.runpytest(testid, "-rf")
         result.stdout.fnmatch_lines([line, "*1 failed*"])

testing/example_scripts/marks/marks_considered_keywords/test_marks_as_keywords.py

@@ -0,0 +1,6 @@
+import pytest
+
+
+@pytest.mark.foo
+def test_mark():
+    pass

testing/example_scripts/warnings/test_group_warnings_by_message.py

@@ -0,0 +1,16 @@
+import warnings
+
+import pytest
+
+
+def func():
+    warnings.warn(UserWarning("foo"))
+
+
+@pytest.mark.parametrize("i", range(5))
+def test_foo(i):
+    func()
+
+
+def test_bar():
+    func()

testing/io/test_saferepr.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+from _pytest._io.saferepr import saferepr
+
+
+def test_simple_repr():
+    assert saferepr(1) == "1"
+    assert saferepr(None) == "None"
+
+
+def test_maxsize():
+    s = saferepr("x" * 50, maxsize=25)
+    assert len(s) == 25
+    expected = repr("x" * 10 + "..." + "x" * 10)
+    assert s == expected
+
+
+def test_maxsize_error_on_instance():
+    class A:
+        def __repr__():
+            raise ValueError("...")
+
+    s = saferepr(("*" * 50, A()), maxsize=25)
+    assert len(s) == 25
+    assert s[0] == "(" and s[-1] == ")"
+
+
+def test_exceptions():
+    class BrokenRepr:
+        def __init__(self, ex):
+            self.ex = ex
+
+        def __repr__(self):
+            raise self.ex
+
+    class BrokenReprException(Exception):
+        __str__ = None
+        __repr__ = None
+
+    assert "Exception" in saferepr(BrokenRepr(Exception("broken")))
+    s = saferepr(BrokenReprException("really broken"))
+    assert "TypeError" in s
+    assert "TypeError" in saferepr(BrokenRepr("string"))
+
+    s2 = saferepr(BrokenRepr(BrokenReprException("omg even worse")))
+    assert "NameError" not in s2
+    assert "unknown" in s2
+
+
+def test_big_repr():
+    from _pytest._io.saferepr import SafeRepr
+
+    assert len(saferepr(range(1000))) <= len("[" + SafeRepr().maxlist * "1000" + "]")
+
+
+def test_repr_on_newstyle():
+    class Function(object):
+        def __repr__(self):
+            return "<%s>" % (self.name)
+
+    assert saferepr(Function())
+
+
+def test_unicode():
+    val = u"£€"
+    reprval = u"'£€'"
+    assert saferepr(val) == reprval

testing/python/collect.py

@@ -960,7 +960,7 @@ class TestTracebackCutting(object):
 
     def test_filter_traceback_generated_code(self):
         """test that filter_traceback() works with the fact that
-        py.code.Code.path attribute might return an str object.
+        _pytest._code.code.Code.path attribute might return an str object.
         In this case, one of the entries on the traceback was produced by
         dynamically generated code.
         See: https://bitbucket.org/pytest-dev/py/issues/71
@@ -981,7 +981,7 @@ class TestTracebackCutting(object):
 
     def test_filter_traceback_path_no_longer_valid(self, testdir):
         """test that filter_traceback() works with the fact that
-        py.code.Code.path attribute might return an str object.
+        _pytest._code.code.Code.path attribute might return an str object.
         In this case, one of the files in the traceback no longer exists.
         This fixes #1133.
         """

testing/test_assertion.py

@@ -7,7 +7,6 @@ import sys
 import textwrap
 
 import attr
-import py
 import six
 
 import _pytest.assertion as plugin
@@ -455,10 +454,13 @@ class TestAssert_reprcompare(object):
         assert len(expl) > 1
 
     def test_Sequence(self):
-        col = py.builtin._tryimport("collections.abc", "collections", "sys")
+        if sys.version_info >= (3, 3):
-        if not hasattr(col, "MutableSequence"):
+            import collections.abc as collections_abc
+        else:
+            import collections as collections_abc
+        if not hasattr(collections_abc, "MutableSequence"):
             pytest.skip("cannot import MutableSequence")
-        MutableSequence = col.MutableSequence
+        MutableSequence = collections_abc.MutableSequence
 
         class TestSequence(MutableSequence):  # works with a Sequence subclass
             def __init__(self, iterable):

testing/test_capture.py

@@ -4,8 +4,10 @@ from __future__ import division
 from __future__ import print_function
 
 import contextlib
+import io
 import os
 import pickle
+import subprocess
 import sys
 import textwrap
 from io import UnsupportedOperation
@@ -850,15 +852,6 @@ class TestCaptureIO(object):
         assert f.getvalue() == "foo\r\n"
 
 
-def test_bytes_io():
-    f = py.io.BytesIO()
-    f.write(b"hello")
-    with pytest.raises(TypeError):
-        f.write(u"hello")
-    s = f.getvalue()
-    assert s == b"hello"
-
-
 def test_dontreadfrominput():
     from _pytest.capture import DontReadFromInput
 
@@ -933,18 +926,18 @@ def test_dupfile(tmpfile):
 
 
 def test_dupfile_on_bytesio():
-    io = py.io.BytesIO()
+    bio = io.BytesIO()
-    f = capture.safe_text_dupfile(io, "wb")
+    f = capture.safe_text_dupfile(bio, "wb")
     f.write("hello")
-    assert io.getvalue() == b"hello"
+    assert bio.getvalue() == b"hello"
     assert "BytesIO object" in f.name
 
 
 def test_dupfile_on_textio():
-    io = py.io.TextIO()
+    tio = py.io.TextIO()
-    f = capture.safe_text_dupfile(io, "wb")
+    f = capture.safe_text_dupfile(tio, "wb")
     f.write("hello")
-    assert io.getvalue() == "hello"
+    assert tio.getvalue() == "hello"
     assert not hasattr(f, "name")
 
 
@@ -952,12 +945,12 @@ def test_dupfile_on_textio():
 def lsof_check():
     pid = os.getpid()
     try:
-        out = py.process.cmdexec("lsof -p %d" % pid)
+        out = subprocess.check_output(("lsof", "-p", str(pid))).decode()
-    except (py.process.cmdexec.Error, UnicodeDecodeError):
+    except (OSError, subprocess.CalledProcessError, UnicodeDecodeError):
         # about UnicodeDecodeError, see note on pytester
         pytest.skip("could not run 'lsof'")
     yield
-    out2 = py.process.cmdexec("lsof -p %d" % pid)
+    out2 = subprocess.check_output(("lsof", "-p", str(pid))).decode()
     len1 = len([x for x in out.split("\n") if "REG" in x])
     len2 = len([x for x in out2.split("\n") if "REG" in x])
     assert len2 < len1 + 3, out2

testing/test_mark.py

@@ -292,6 +292,13 @@ def test_keyword_option_custom(spec, testdir):
     assert list(passed) == list(passed_result)
 
 
+def test_keyword_option_considers_mark(testdir):
+    testdir.copy_example("marks/marks_considered_keywords")
+    rec = testdir.inline_run("-k", "foo")
+    passed = rec.listoutcomes()[0]
+    assert len(passed) == 1
+
+
 @pytest.mark.parametrize(
     "spec",
     [

testing/test_monkeypatch.py

@@ -391,6 +391,33 @@ def test_issue156_undo_staticmethod(Sample):
     assert Sample.hello()
 
 
+def test_undo_class_descriptors_delattr():
+    class SampleParent(object):
+        @classmethod
+        def hello(_cls):
+            pass
+
+        @staticmethod
+        def world():
+            pass
+
+    class SampleChild(SampleParent):
+        pass
+
+    monkeypatch = MonkeyPatch()
+
+    original_hello = SampleChild.hello
+    original_world = SampleChild.world
+    monkeypatch.delattr(SampleParent, "hello")
+    monkeypatch.delattr(SampleParent, "world")
+    assert getattr(SampleParent, "hello", None) is None
+    assert getattr(SampleParent, "world", None) is None
+
+    monkeypatch.undo()
+    assert original_hello == SampleChild.hello
+    assert original_world == SampleChild.world
+
+
 def test_issue1338_name_resolving():
     pytest.importorskip("requests")
     monkeypatch = MonkeyPatch()

testing/test_nose.py

@@ -1,3 +1,4 @@
+# encoding: utf-8
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -366,3 +367,17 @@ def test_nottest_class_decorator(testdir):
     assert not reprec.getfailedcollections()
     calls = reprec.getreports("pytest_runtest_logreport")
     assert not calls
+
+
+def test_skip_test_with_unicode(testdir):
+    testdir.makepyfile(
+        """
+        # encoding: utf-8
+        import unittest
+        class TestClass():
+            def test_io(self):
+                raise unittest.SkipTest(u'😊')
+    """
+    )
+    result = testdir.runpytest()
+    result.stdout.fnmatch_lines("* 1 skipped *")

testing/test_parseopt.py

@@ -3,6 +3,7 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
+import distutils.spawn
 import os
 import sys
 
@@ -296,7 +297,7 @@ class TestParser(object):
 
 
 def test_argcomplete(testdir, monkeypatch):
-    if not py.path.local.sysfind("bash"):
+    if not distutils.spawn.find_executable("bash"):
         pytest.skip("bash not available")
     script = str(testdir.tmpdir.join("test_argcomplete"))
     pytest_bin = sys.argv[0]

testing/test_skipping.py

@@ -770,6 +770,7 @@ def test_skip_reasons_folding():
 
     # ev3 might be a collection report
     ev3 = X()
+    ev3.when = "collect"
     ev3.longrepr = longrepr
     ev3.skipped = True
 
@@ -1202,6 +1203,6 @@ def test_summary_list_after_errors(testdir):
         [
             "=* FAILURES *=",
             "*= short test summary info =*",
-            "FAIL test_summary_list_after_errors.py::test_fail",
+            "FAILED test_summary_list_after_errors.py::test_fail",
         ]
     )

testing/test_terminal.py

@@ -614,7 +614,7 @@ class TestTerminalFunctional(object):
                 "*test_verbose_reporting.py::test_fail *FAIL*",
                 "*test_verbose_reporting.py::test_pass *PASS*",
                 "*test_verbose_reporting.py::TestClass::test_skip *SKIP*",
-                "*test_verbose_reporting.py::test_gen *xfail*",
+                "*test_verbose_reporting.py::test_gen *XFAIL*",
             ]
         )
         assert result.ret == 1

testing/test_tmpdir.py

@@ -121,6 +121,22 @@ def test_tmpdir_always_is_realpath(testdir):
     assert not result.ret
 
 
+def test_tmp_path_always_is_realpath(testdir, monkeypatch):
+    # for reasoning see: test_tmpdir_always_is_realpath test-case
+    realtemp = testdir.tmpdir.mkdir("myrealtemp")
+    linktemp = testdir.tmpdir.join("symlinktemp")
+    attempt_symlink_to(linktemp, str(realtemp))
+    monkeypatch.setenv("PYTEST_DEBUG_TEMPROOT", str(linktemp))
+    testdir.makepyfile(
+        """
+        def test_1(tmp_path):
+            assert tmp_path.resolve() == tmp_path
+    """
+    )
+    reprec = testdir.inline_run()
+    reprec.assertoutcome(passed=1)
+
+
 def test_tmpdir_too_long_on_parametrization(testdir):
     testdir.makepyfile(
         """

testing/test_warnings.py

@@ -693,3 +693,22 @@ def test_warnings_checker_twice():
         warnings.warn("Message A", UserWarning)
     with expectation:
         warnings.warn("Message B", UserWarning)
+
+
+@pytest.mark.filterwarnings("always")
+def test_group_warnings_by_message(testdir):
+    testdir.copy_example("warnings/test_group_warnings_by_message.py")
+    result = testdir.runpytest()
+    result.stdout.fnmatch_lines(
+        [
+            "test_group_warnings_by_message.py::test_foo[0]",
+            "test_group_warnings_by_message.py::test_foo[1]",
+            "test_group_warnings_by_message.py::test_foo[2]",
+            "test_group_warnings_by_message.py::test_foo[3]",
+            "test_group_warnings_by_message.py::test_foo[4]",
+            "test_group_warnings_by_message.py::test_bar",
+        ]
+    )
+    warning_code = 'warnings.warn(UserWarning("foo"))'
+    assert warning_code in result.stdout.str()
+    assert result.stdout.str().count(warning_code) == 1