From c7eb53317b4535cabd50d817964ca7e4644cdcf9 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 17 Jun 2018 16:23:44 +0200 Subject: [PATCH 01/11] port cache plugin internals to pathlib warning logging got broken by detanglement from config --- setup.py | 4 +- src/_pytest/cacheprovider.py | 101 ++++++++++++++++------------------ src/_pytest/compat.py | 3 + testing/test_cacheprovider.py | 4 +- 4 files changed, 57 insertions(+), 55 deletions(-) diff --git a/setup.py b/setup.py index e799d53bf..ed5a96d5e 100644 --- a/setup.py +++ b/setup.py @@ -73,15 +73,17 @@ def main(): environment_marker_support_level = get_environment_marker_support_level() if environment_marker_support_level >= 2: install_requires.append('funcsigs;python_version<"3.0"') + install_requires.append('pathlib2;python_version<"3.0"') install_requires.append('colorama;sys_platform=="win32"') elif environment_marker_support_level == 1: - extras_require[':python_version<"3.0"'] = ["funcsigs"] + extras_require[':python_version<"3.0"'] = ["funcsigs", "pathlib2"] extras_require[':sys_platform=="win32"'] = ["colorama"] else: if sys.platform == "win32": install_requires.append("colorama") if sys.version_info < (3, 0): install_requires.append("funcsigs") + install_requires.append("pathlib2") setup( name="pytest", diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index f15398383..5b4dbc381 100755 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -5,40 +5,41 @@ the name cache was not chosen to ensure pluggy automatically ignores the external pytest-cache """ from __future__ import absolute_import, division, print_function - from collections import OrderedDict import py import six +import attr import pytest import json -import os from os.path import sep as _sep, altsep as _altsep from textwrap import dedent +from . import paths +import logging + + +log = logging.getLogger(__name__) + + +@attr.s class Cache(object): - def __init__(self, config): - self.config = config - self._cachedir = Cache.cache_dir_from_config(config) - self.trace = config.trace.root.get("cache") - if config.getoption("cacheclear"): - self.trace("clearing cachedir") - if self._cachedir.check(): - self._cachedir.remove() - self._cachedir.mkdir() + @classmethod + def for_config(cls, config): + cachedir = cls.cache_dir_from_config(config) + if config.getoption("cacheclear") and cachedir.exists(): + shutil.rmtree(str(cachedir)) + cachedir.mkdir() + return cls(cachedir) + + _cachedir = attr.ib(repr=False) @staticmethod def cache_dir_from_config(config): - cache_dir = config.getini("cache_dir") - cache_dir = os.path.expanduser(cache_dir) - cache_dir = os.path.expandvars(cache_dir) - if os.path.isabs(cache_dir): - return py.path.local(cache_dir) - else: - return config.rootdir.join(cache_dir) + return paths.resolve_from_str(config.getini("cache_dir"), config.rootdir) def makedir(self, name): """ return a directory path object with the given name. If the @@ -52,10 +53,12 @@ class Cache(object): """ if _sep in name or _altsep is not None and _altsep in name: raise ValueError("name is not allowed to contain path separators") - return self._cachedir.ensure_dir("d", name) + res = self._cachedir.joinpath("d", name) + res.mkdir(exist_ok=True, parents=True) + return py.path.local(res) def _getvaluepath(self, key): - return self._cachedir.join("v", *key.split("/")) + return self._cachedir.joinpath("v", *key.split("/")) def get(self, key, default): """ return cached value for the given key. If no value @@ -69,13 +72,11 @@ class Cache(object): """ path = self._getvaluepath(key) - if path.check(): - try: - with path.open("r") as f: - return json.load(f) - except ValueError: - self.trace("cache-invalid at %s" % (path,)) - return default + try: + with path.open("r") as f: + return json.load(f) + except (ValueError, IOError): + return default def set(self, key, value): """ save value for the given key. @@ -88,21 +89,16 @@ class Cache(object): """ path = self._getvaluepath(key) try: - path.dirpath().ensure_dir() - except (py.error.EEXIST, py.error.EACCES): - self.config.warn( - code="I9", message="could not create cache path %s" % (path,) - ) + path.parent.mkdir(exist_ok=True, parents=True) + except IOError: + log.warning("could not create cache path %s", path) return try: f = path.open("w") except py.error.ENOTDIR: - self.config.warn( - code="I9", message="cache could not write path %s" % (path,) - ) + log.warning("cache could not write path %s", path) else: with f: - self.trace("cache-write %s: %r" % (key, value)) json.dump(value, f, indent=2, sort_keys=True) self._ensure_readme() @@ -297,7 +293,7 @@ def pytest_cmdline_main(config): @pytest.hookimpl(tryfirst=True) def pytest_configure(config): - config.cache = Cache(config) + config.cache = Cache.for_config(config) config.pluginmanager.register(LFPlugin(config), "lfplugin") config.pluginmanager.register(NFPlugin(config), "nfplugin") @@ -320,41 +316,40 @@ def cache(request): def pytest_report_header(config): if config.option.verbose: - relpath = py.path.local().bestrelpath(config.cache._cachedir) - return "cachedir: %s" % relpath + relpath = config.cache._cachedir.relative_to(config.rootdir) + return "cachedir: {}".format(relpath) def cacheshow(config, session): - from pprint import pprint + from pprint import pformat tw = py.io.TerminalWriter() tw.line("cachedir: " + str(config.cache._cachedir)) - if not config.cache._cachedir.check(): + if not config.cache._cachedir.is_dir(): tw.line("cache is empty") return 0 dummy = object() basedir = config.cache._cachedir - vdir = basedir.join("v") + vdir = basedir.joinpath("v") tw.sep("-", "cache values") - for valpath in sorted(vdir.visit(lambda x: x.isfile())): - key = valpath.relto(vdir).replace(valpath.sep, "/") + for valpath in sorted(x for x in vdir.rglob("*") if x.is_file()): + key = "/".join(valpath.relative_to(vdir).parts) val = config.cache.get(key, dummy) if val is dummy: tw.line("%s contains unreadable content, " "will be ignored" % key) else: tw.line("%s contains:" % key) - stream = py.io.TextIO() - pprint(val, stream=stream) - for line in stream.getvalue().splitlines(): + for line in pformat(val).splitlines(): tw.line(" " + line) - ddir = basedir.join("d") - if ddir.isdir() and ddir.listdir(): + ddir = basedir.joinpath("d") + if ddir.is_dir(): + contents = sorted(ddir.rglob("*")) tw.sep("-", "cache directories") - for p in sorted(basedir.join("d").visit()): + for p in contents: # if p.check(dir=1): # print("%s/" % p.relto(basedir)) - if p.isfile(): - key = p.relto(basedir) - tw.line("%s is a file of length %d" % (key, p.size())) + if p.is_file(): + key = p.relative_to(basedir) + tw.line("{} is a file of length {:d}".format(key, p.stat().st_size)) return 0 diff --git a/src/_pytest/compat.py b/src/_pytest/compat.py index 7abd3d53f..78a23523b 100644 --- a/src/_pytest/compat.py +++ b/src/_pytest/compat.py @@ -29,8 +29,11 @@ _PY2 = not _PY3 if _PY3: from inspect import signature, Parameter as Parameter + from pathlib import Path else: from funcsigs import signature, Parameter as Parameter + from pathlib2 import Path +Path NoneType = type(None) diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index 4a7c25aa0..68ed33910 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -1,5 +1,7 @@ from __future__ import absolute_import, division, print_function + import sys + import py import _pytest import pytest @@ -26,7 +28,7 @@ class TestNewAPI(object): cache = config.cache pytest.raises(TypeError, lambda: cache.set("key/name", cache)) config.cache.set("key/name", 0) - config.cache._getvaluepath("key/name").write("123invalid") + config.cache._getvaluepath("key/name").write_bytes(b"123invalid") val = config.cache.get("key/name", -2) assert val == -2 From 23581d44bd94a54eeb10f43a5a317420db02b269 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Mon, 18 Jun 2018 14:00:19 +0200 Subject: [PATCH 02/11] add missed file --- src/_pytest/paths.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/_pytest/paths.py diff --git a/src/_pytest/paths.py b/src/_pytest/paths.py new file mode 100644 index 000000000..7c0dc1ec1 --- /dev/null +++ b/src/_pytest/paths.py @@ -0,0 +1,13 @@ +from .compat import Path +from os.path import expanduser, expandvars, isabs + + +def resolve_from_str(input, root): + assert not isinstance(input, Path), "would break on py2" + root = Path(root) + input = expanduser(input) + input = expandvars(input) + if isabs(input): + return Path(input) + else: + return root.joinpath(input) From fb992a0c81e02e9a298a7aa38d46001229a14571 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Mon, 18 Jun 2018 14:01:06 +0200 Subject: [PATCH 03/11] reorder attr.ib specs --- src/_pytest/cacheprovider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index 5b4dbc381..b141d73a5 100755 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -27,6 +27,8 @@ log = logging.getLogger(__name__) @attr.s class Cache(object): + _cachedir = attr.ib(repr=False) + @classmethod def for_config(cls, config): cachedir = cls.cache_dir_from_config(config) @@ -35,8 +37,6 @@ class Cache(object): cachedir.mkdir() return cls(cachedir) - _cachedir = attr.ib(repr=False) - @staticmethod def cache_dir_from_config(config): return paths.resolve_from_str(config.getini("cache_dir"), config.rootdir) From ab80e0fba087b099445c1d57b661cfed1638096b Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Mon, 18 Jun 2018 14:05:14 +0200 Subject: [PATCH 04/11] sort compat flake8 mess correctly --- src/_pytest/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_pytest/compat.py b/src/_pytest/compat.py index 78a23523b..ec647bb44 100644 --- a/src/_pytest/compat.py +++ b/src/_pytest/compat.py @@ -22,6 +22,7 @@ except ImportError: # pragma: no cover # Only available in Python 3.4+ or as a backport enum = None +__all__ = ["Path"] _PY3 = sys.version_info > (3, 0) _PY2 = not _PY3 @@ -33,7 +34,6 @@ if _PY3: else: from funcsigs import signature, Parameter as Parameter from pathlib2 import Path -Path NoneType = type(None) From 1226cdab47335bb823b2fee092b94b1c37b0a62a Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 19 Jun 2018 10:28:36 +0200 Subject: [PATCH 05/11] fix warnings and json dumping of cacheprovider --- src/_pytest/cacheprovider.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index b141d73a5..5b262fddc 100755 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -17,17 +17,14 @@ from os.path import sep as _sep, altsep as _altsep from textwrap import dedent from . import paths - -import logging - - -log = logging.getLogger(__name__) +from .compat import _PY2 as PY2 @attr.s class Cache(object): _cachedir = attr.ib(repr=False) + _warn = attr.ib(repr=False) @classmethod def for_config(cls, config): @@ -35,12 +32,15 @@ class Cache(object): if config.getoption("cacheclear") and cachedir.exists(): shutil.rmtree(str(cachedir)) cachedir.mkdir() - return cls(cachedir) + return cls(cachedir, config.warn) @staticmethod def cache_dir_from_config(config): return paths.resolve_from_str(config.getini("cache_dir"), config.rootdir) + def warn(self, fmt, **args): + self._warn(code="I9", message=fmt.format(**args) if args else fmt) + def makedir(self, name): """ return a directory path object with the given name. If the directory does not yet exist, it will be created. You can use it @@ -75,7 +75,7 @@ class Cache(object): try: with path.open("r") as f: return json.load(f) - except (ValueError, IOError): + except (ValueError, IOError, OSError): return default def set(self, key, value): @@ -90,13 +90,13 @@ class Cache(object): path = self._getvaluepath(key) try: path.parent.mkdir(exist_ok=True, parents=True) - except IOError: - log.warning("could not create cache path %s", path) + except (IOError, OSError): + self.warn("could not create cache path {path}", path=path) return try: - f = path.open("w") - except py.error.ENOTDIR: - log.warning("cache could not write path %s", path) + f = path.open("wb" if PY2 else "w") + except (IOError, OSError): + self.warn("cache could not write path {path}", path=path) else: with f: json.dump(value, f, indent=2, sort_keys=True) From abbf73ad1a3b5d8d4522df2ca2d3f0c83ea8e349 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 19 Jun 2018 19:45:20 +0200 Subject: [PATCH 06/11] use pathlib2 up to python3.4 - damn the stdlib --- setup.py | 6 ++++-- src/_pytest/compat.py | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index ed5a96d5e..710228658 100644 --- a/setup.py +++ b/setup.py @@ -73,16 +73,18 @@ def main(): environment_marker_support_level = get_environment_marker_support_level() if environment_marker_support_level >= 2: install_requires.append('funcsigs;python_version<"3.0"') - install_requires.append('pathlib2;python_version<"3.0"') + install_requires.append('pathlib2;python_version<"3.5"') install_requires.append('colorama;sys_platform=="win32"') elif environment_marker_support_level == 1: - extras_require[':python_version<"3.0"'] = ["funcsigs", "pathlib2"] + extras_require[':python_version<"3.0"'] = ["funcsigs"] + extras_require[':python_version<"3.5"'] = ["pathlib2"] extras_require[':sys_platform=="win32"'] = ["colorama"] else: if sys.platform == "win32": install_requires.append("colorama") if sys.version_info < (3, 0): install_requires.append("funcsigs") + if sys.version_info < (3, 5): install_requires.append("pathlib2") setup( diff --git a/src/_pytest/compat.py b/src/_pytest/compat.py index ec647bb44..01d46222d 100644 --- a/src/_pytest/compat.py +++ b/src/_pytest/compat.py @@ -30,11 +30,8 @@ _PY2 = not _PY3 if _PY3: from inspect import signature, Parameter as Parameter - from pathlib import Path else: from funcsigs import signature, Parameter as Parameter - from pathlib2 import Path - NoneType = type(None) NOTSET = object() @@ -43,6 +40,12 @@ PY35 = sys.version_info[:2] >= (3, 5) PY36 = sys.version_info[:2] >= (3, 6) MODULE_NOT_FOUND_ERROR = "ModuleNotFoundError" if PY36 else "ImportError" +if PY35: + from pathlib import Path +else: + from pathlib2 import Path + + if _PY3: from collections.abc import MutableMapping as MappingMixin # noqa from collections.abc import Mapping, Sequence # noqa From 603df1ea1c638c0b704b5d135c03b00384cce0f9 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 19 Jun 2018 23:15:13 +0200 Subject: [PATCH 07/11] whops, its supported starting with python 3.6, not python 3.5 --- setup.py | 6 +++--- src/_pytest/compat.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 710228658..9c703cc8e 100644 --- a/setup.py +++ b/setup.py @@ -73,18 +73,18 @@ def main(): environment_marker_support_level = get_environment_marker_support_level() if environment_marker_support_level >= 2: install_requires.append('funcsigs;python_version<"3.0"') - install_requires.append('pathlib2;python_version<"3.5"') + install_requires.append('pathlib2;python_version<"3.6"') install_requires.append('colorama;sys_platform=="win32"') elif environment_marker_support_level == 1: extras_require[':python_version<"3.0"'] = ["funcsigs"] - extras_require[':python_version<"3.5"'] = ["pathlib2"] + extras_require[':python_version<"3.6"'] = ["pathlib2"] extras_require[':sys_platform=="win32"'] = ["colorama"] else: if sys.platform == "win32": install_requires.append("colorama") if sys.version_info < (3, 0): install_requires.append("funcsigs") - if sys.version_info < (3, 5): + if sys.version_info < (3, 6): install_requires.append("pathlib2") setup( diff --git a/src/_pytest/compat.py b/src/_pytest/compat.py index 01d46222d..4620f2be9 100644 --- a/src/_pytest/compat.py +++ b/src/_pytest/compat.py @@ -40,7 +40,7 @@ PY35 = sys.version_info[:2] >= (3, 5) PY36 = sys.version_info[:2] >= (3, 6) MODULE_NOT_FOUND_ERROR = "ModuleNotFoundError" if PY36 else "ImportError" -if PY35: +if PY36: from pathlib import Path else: from pathlib2 import Path From ee30bf45c99f1fd8f4ba9bc26a2b77b3a6aae5c6 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 23 Jun 2018 00:03:10 +0200 Subject: [PATCH 08/11] rebase onto readme addition --- src/_pytest/cacheprovider.py | 37 +++++++++++++++++------------------ testing/test_cacheprovider.py | 4 ++-- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index 5b262fddc..0d877f14b 100755 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -14,11 +14,22 @@ import attr import pytest import json from os.path import sep as _sep, altsep as _altsep -from textwrap import dedent +import shutil from . import paths from .compat import _PY2 as PY2 +README_CONTENT = u"""\ +# pytest cache directory # + +This directory contains data from the pytest's cache plugin, +which provides the `--lf` and `--ff` options, as well as the `cache` fixture. + +**Do not** commit this to version control. + +See [the docs](https://docs.pytest.org/en/latest/cache.html) for more information. +""" + @attr.s class Cache(object): @@ -104,22 +115,10 @@ class Cache(object): def _ensure_readme(self): - content_readme = dedent( - """\ - # pytest cache directory # - - This directory contains data from the pytest's cache plugin, - which provides the `--lf` and `--ff` options, as well as the `cache` fixture. - - **Do not** commit this to version control. - - See [the docs](https://docs.pytest.org/en/latest/cache.html) for more information. - """ - ) - if self._cachedir.check(dir=True): - readme_path = self._cachedir.join("README.md") - if not readme_path.check(file=True): - readme_path.write(content_readme) + if self._cachedir.is_dir(): + readme_path = self._cachedir / "README.md" + if not readme_path.is_file(): + readme_path.write_text(README_CONTENT) class LFPlugin(object): @@ -330,7 +329,7 @@ def cacheshow(config, session): return 0 dummy = object() basedir = config.cache._cachedir - vdir = basedir.joinpath("v") + vdir = basedir / "v" tw.sep("-", "cache values") for valpath in sorted(x for x in vdir.rglob("*") if x.is_file()): key = "/".join(valpath.relative_to(vdir).parts) @@ -342,7 +341,7 @@ def cacheshow(config, session): for line in pformat(val).splitlines(): tw.line(" " + line) - ddir = basedir.joinpath("d") + ddir = basedir / "d" if ddir.is_dir(): contents = sorted(ddir.rglob("*")) tw.sep("-", "cache directories") diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index 68ed33910..9190aecdd 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -826,8 +826,8 @@ class TestReadme(object): def check_readme(self, testdir): config = testdir.parseconfigure() - readme = config.cache._cachedir.join("README.md") - return readme.isfile() + readme = config.cache._cachedir.joinpath("README.md") + return readme.is_file() def test_readme_passed(self, testdir): testdir.makepyfile( From c4c666cbc43e66475618d8fe94cd9dd167140a2d Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 23 Jun 2018 00:07:57 +0200 Subject: [PATCH 09/11] use Pathlib instead of path splitting --- src/_pytest/cacheprovider.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index 0d877f14b..607a3b194 100755 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -13,11 +13,10 @@ import attr import pytest import json -from os.path import sep as _sep, altsep as _altsep import shutil from . import paths -from .compat import _PY2 as PY2 +from .compat import _PY2 as PY2, Path README_CONTENT = u"""\ # pytest cache directory # @@ -62,14 +61,15 @@ class Cache(object): Make sure the name contains your plugin or application identifiers to prevent clashes with other cache users. """ - if _sep in name or _altsep is not None and _altsep in name: + name = Path(name) + if len(name.parts) > 1: raise ValueError("name is not allowed to contain path separators") res = self._cachedir.joinpath("d", name) res.mkdir(exist_ok=True, parents=True) return py.path.local(res) def _getvaluepath(self, key): - return self._cachedir.joinpath("v", *key.split("/")) + return self._cachedir.joinpath("v", Path(key)) def get(self, key, default): """ return cached value for the given key. If no value From 95f00de0df0447cf39af2b90aa1ec34e4e1ec163 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 23 Jun 2018 00:14:06 +0200 Subject: [PATCH 10/11] use paths for config.cache.get key --- src/_pytest/cacheprovider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index 607a3b194..8ca88937f 100755 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -332,7 +332,7 @@ def cacheshow(config, session): vdir = basedir / "v" tw.sep("-", "cache values") for valpath in sorted(x for x in vdir.rglob("*") if x.is_file()): - key = "/".join(valpath.relative_to(vdir).parts) + key = valpath.relative_to(vdir) val = config.cache.get(key, dummy) if val is dummy: tw.line("%s contains unreadable content, " "will be ignored" % key) From 5a156b364519658daf0230b446d97967c3d0e4b9 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 24 Jun 2018 22:54:26 +0200 Subject: [PATCH 11/11] disable pypy on windows until scandir works for it --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index b808fa6d9..7d4138b84 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,7 +14,7 @@ environment: - TOXENV: "py34" - TOXENV: "py35" - TOXENV: "py36" - - TOXENV: "pypy" +# - TOXENV: "pypy" reenable when we are able to provide a scandir wheel or build scandir - TOXENV: "py27-pexpect" - TOXENV: "py27-xdist" - TOXENV: "py27-trial"