Merge pull request #5063 from asottile/importlib_metadata_v2

Switch to importlib-metadata

changelog/5063.feature.rst

@@ -0,0 +1 @@
+Switch from ``pkg_resources`` to ``importlib-metadata`` for entrypoint detection for improved performance and import time.

setup.py

@@ -5,7 +5,7 @@ from setuptools import setup
 INSTALL_REQUIRES = [
     "py>=1.5.0",
     "six>=1.10.0",
-    "setuptools",
+    "packaging",
     "attrs>=17.4.0",
     'more-itertools>=4.0.0,<6.0.0;python_version<="2.7"',
     'more-itertools>=4.0.0;python_version>"2.7"',
@@ -13,7 +13,8 @@ INSTALL_REQUIRES = [
     'funcsigs>=1.0;python_version<"3.0"',
     'pathlib2>=2.2.0;python_version<"3.6"',
     'colorama;sys_platform=="win32"',
-    "pluggy>=0.9,!=0.10,<1.0",
+    "pluggy>=0.12,<1.0",
+    "importlib-metadata>=0.12",
     "wcwidth",
 ]
 

src/_pytest/assertion/rewrite.py

@@ -64,7 +64,6 @@ class AssertionRewritingHook(object):
         self.session = None
         self.modules = {}
         self._rewritten_names = set()
-        self._register_with_pkg_resources()
         self._must_rewrite = set()
         # flag to guard against trying to rewrite a pyc file while we are already writing another pyc file,
         # which might result in infinite recursion (#3506)
@@ -315,24 +314,6 @@ class AssertionRewritingHook(object):
         tp = desc[2]
         return tp == imp.PKG_DIRECTORY
 
-    @classmethod
-    def _register_with_pkg_resources(cls):
-        """
-        Ensure package resources can be loaded from this loader. May be called
-        multiple times, as the operation is idempotent.
-        """
-        try:
-            import pkg_resources
-
-            # access an attribute in case a deferred importer is present
-            pkg_resources.__name__
-        except ImportError:
-            return
-
-        # Since pytest tests are always located in the file system, the
-        #  DefaultProvider is appropriate.
-        pkg_resources.register_loader_type(cls, pkg_resources.DefaultProvider)
-
     def get_data(self, pathname):
         """Optional PEP302 get_data API.
         """

src/_pytest/config/__init__.py

@@ -12,8 +12,10 @@ import sys
 import types
 import warnings
 
+import importlib_metadata
 import py
 import six
+from packaging.version import Version
 from pluggy import HookimplMarker
 from pluggy import HookspecMarker
 from pluggy import PluginManager
@@ -787,25 +789,17 @@ class Config(object):
         modules or packages in the distribution package for
         all pytest plugins.
         """
-        import pkg_resources
-
         self.pluginmanager.rewrite_hook = hook
 
         if os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"):
             # We don't autoload from setuptools entry points, no need to continue.
             return
 
-        # 'RECORD' available for plugins installed normally (pip install)
-        # 'SOURCES.txt' available for plugins installed in dev mode (pip install -e)
-        # for installed plugins 'SOURCES.txt' returns an empty list, and vice-versa
-        # so it shouldn't be an issue
-        metadata_files = "RECORD", "SOURCES.txt"
-
         package_files = (
-            entry.split(",")[0]
+            str(file)
-            for entrypoint in pkg_resources.iter_entry_points("pytest11")
+            for dist in importlib_metadata.distributions()
-            for metadata in metadata_files
+            if any(ep.group == "pytest11" for ep in dist.entry_points)
-            for entry in entrypoint.dist._get_metadata(metadata)
+            for file in dist.files
         )
 
         for name in _iter_rewritable_modules(package_files):
@@ -874,11 +868,10 @@ class Config(object):
 
     def _checkversion(self):
         import pytest
-        from pkg_resources import parse_version
 
         minver = self.inicfg.get("minversion", None)
         if minver:
-            if parse_version(minver) > parse_version(pytest.__version__):
+            if Version(minver) > Version(pytest.__version__):
                 raise pytest.UsageError(
                     "%s:%d: requires pytest-%s, actual pytest-%s'"
                     % (

src/_pytest/outcomes.py

@@ -8,6 +8,8 @@ from __future__ import print_function
 
 import sys
 
+from packaging.version import Version
+
 
 class OutcomeException(BaseException):
     """ OutcomeException and its subclass instances indicate and
@@ -175,15 +177,7 @@ def importorskip(modname, minversion=None, reason=None):
         return mod
     verattr = getattr(mod, "__version__", None)
     if minversion is not None:
-        try:
+        if verattr is None or Version(verattr) < Version(minversion):
-            from pkg_resources import parse_version as pv
-        except ImportError:
-            raise Skipped(
-                "we have a required version for %r but can not import "
-                "pkg_resources to parse version strings." % (modname,),
-                allow_module_level=True,
-            )
-        if verattr is None or pv(verattr) < pv(minversion):
             raise Skipped(
                 "module %r has __version__ %r, required is: %r"
                 % (modname, verattr, minversion),

testing/acceptance_test.py

@@ -9,6 +9,7 @@ import textwrap
 import types
 
 import attr
+import importlib_metadata
 import py
 import six
 
@@ -111,8 +112,6 @@ class TestGeneralUsage(object):
 
     @pytest.mark.parametrize("load_cov_early", [True, False])
     def test_early_load_setuptools_name(self, testdir, monkeypatch, load_cov_early):
-        pkg_resources = pytest.importorskip("pkg_resources")
-
         testdir.makepyfile(mytestplugin1_module="")
         testdir.makepyfile(mytestplugin2_module="")
         testdir.makepyfile(mycov_module="")
@@ -124,38 +123,28 @@ class TestGeneralUsage(object):
         class DummyEntryPoint(object):
             name = attr.ib()
             module = attr.ib()
-            version = "1.0"
+            group = "pytest11"
-
-            @property
-            def project_name(self):
-                return self.name
 
             def load(self):
                 __import__(self.module)
                 loaded.append(self.name)
                 return sys.modules[self.module]
 
-            @property
-            def dist(self):
-                return self
-
-            def _get_metadata(self, *args):
-                return []
-
         entry_points = [
             DummyEntryPoint("myplugin1", "mytestplugin1_module"),
             DummyEntryPoint("myplugin2", "mytestplugin2_module"),
             DummyEntryPoint("mycov", "mycov_module"),
         ]
 
-        def my_iter(group, name=None):
+        @attr.s
-            assert group == "pytest11"
+        class DummyDist(object):
-            for ep in entry_points:
+            entry_points = attr.ib()
-                if name is not None and ep.name != name:
+            files = ()
-                    continue
-                yield ep
 
-        monkeypatch.setattr(pkg_resources, "iter_entry_points", my_iter)
+        def my_dists():
+            return (DummyDist(entry_points),)
+
+        monkeypatch.setattr(importlib_metadata, "distributions", my_dists)
         params = ("-p", "mycov") if load_cov_early else ()
         testdir.runpytest_inprocess(*params)
         if load_cov_early:

testing/test_assertion.py

@@ -137,12 +137,12 @@ class TestImportHookInstallation(object):
     def test_pytest_plugins_rewrite_module_names_correctly(self, testdir):
         """Test that we match files correctly when they are marked for rewriting (#2939)."""
         contents = {
-            "conftest.py": """
+            "conftest.py": """\
                 pytest_plugins = "ham"
             """,
             "ham.py": "",
             "hamster.py": "",
-            "test_foo.py": """
+            "test_foo.py": """\
                 def test_foo(pytestconfig):
                     assert pytestconfig.pluginmanager.rewrite_hook.find_module('ham') is not None
                     assert pytestconfig.pluginmanager.rewrite_hook.find_module('hamster') is None
@@ -153,14 +153,13 @@ class TestImportHookInstallation(object):
         assert result.ret == 0
 
     @pytest.mark.parametrize("mode", ["plain", "rewrite"])
-    @pytest.mark.parametrize("plugin_state", ["development", "installed"])
+    def test_installed_plugin_rewrite(self, testdir, mode, monkeypatch):
-    def test_installed_plugin_rewrite(self, testdir, mode, plugin_state, monkeypatch):
         monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False)
         # Make sure the hook is installed early enough so that plugins
         # installed via setuptools are rewritten.
         testdir.tmpdir.join("hampkg").ensure(dir=1)
         contents = {
-            "hampkg/__init__.py": """
+            "hampkg/__init__.py": """\
                 import pytest
 
                 @pytest.fixture
@@ -169,7 +168,7 @@ class TestImportHookInstallation(object):
                         assert values.pop(0) == value
                     return check
             """,
-            "spamplugin.py": """
+            "spamplugin.py": """\
             import pytest
             from hampkg import check_first2
 
@@ -179,46 +178,31 @@ class TestImportHookInstallation(object):
                     assert values.pop(0) == value
                 return check
             """,
-            "mainwrapper.py": """
+            "mainwrapper.py": """\
-            import pytest, pkg_resources
+            import pytest, importlib_metadata
-
-            plugin_state = "{plugin_state}"
-
-            class DummyDistInfo(object):
-                project_name = 'spam'
-                version = '1.0'
-
-                def _get_metadata(self, name):
-                    # 'RECORD' meta-data only available in installed plugins
-                    if name == 'RECORD' and plugin_state == "installed":
-                        return ['spamplugin.py,sha256=abc,123',
-                                'hampkg/__init__.py,sha256=abc,123']
-                    # 'SOURCES.txt' meta-data only available for plugins in development mode
-                    elif name == 'SOURCES.txt' and plugin_state == "development":
-                        return ['spamplugin.py',
-                                'hampkg/__init__.py']
-                    return []
 
             class DummyEntryPoint(object):
                 name = 'spam'
                 module_name = 'spam.py'
-                attrs = ()
+                group = 'pytest11'
-                extras = None
-                dist = DummyDistInfo()
 
-                def load(self, require=True, *args, **kwargs):
+                def load(self):
                     import spamplugin
                     return spamplugin
 
-            def iter_entry_points(group, name=None):
+            class DummyDistInfo(object):
-                yield DummyEntryPoint()
+                version = '1.0'
+                files = ('spamplugin.py', 'hampkg/__init__.py')
+                entry_points = (DummyEntryPoint(),)
+                metadata = {'name': 'foo'}
 
-            pkg_resources.iter_entry_points = iter_entry_points
+            def distributions():
+                return (DummyDistInfo(),)
+
+            importlib_metadata.distributions = distributions
             pytest.main()
-            """.format(
+            """,
-                plugin_state=plugin_state
+            "test_foo.py": """\
-            ),
-            "test_foo.py": """
             def test(check_first):
                 check_first([10, 30], 30)
 

testing/test_config.py

@@ -5,7 +5,7 @@ from __future__ import print_function
 import sys
 import textwrap
 
-import attr
+import importlib_metadata
 
 import _pytest._code
 import pytest
@@ -531,32 +531,26 @@ def test_options_on_small_file_do_not_blow_up(testdir):
 
 
 def test_preparse_ordering_with_setuptools(testdir, monkeypatch):
-    pkg_resources = pytest.importorskip("pkg_resources")
     monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False)
 
-    def my_iter(group, name=None):
+    class EntryPoint(object):
-        assert group == "pytest11"
+        name = "mytestplugin"
+        group = "pytest11"
 
-        class Dist(object):
+        def load(self):
-            project_name = "spam"
+            class PseudoPlugin(object):
-            version = "1.0"
+                x = 42
 
-            def _get_metadata(self, name):
+            return PseudoPlugin()
-                return ["foo.txt,sha256=abc,123"]
 
-        class EntryPoint(object):
+    class Dist(object):
-            name = "mytestplugin"
+        files = ()
-            dist = Dist()
+        entry_points = (EntryPoint(),)
 
-            def load(self):
+    def my_dists():
-                class PseudoPlugin(object):
+        return (Dist,)
-                    x = 42
 
-                return PseudoPlugin()
+    monkeypatch.setattr(importlib_metadata, "distributions", my_dists)
-
-        return iter([EntryPoint()])
-
-    monkeypatch.setattr(pkg_resources, "iter_entry_points", my_iter)
     testdir.makeconftest(
         """
         pytest_plugins = "mytestplugin",
@@ -569,60 +563,50 @@ def test_preparse_ordering_with_setuptools(testdir, monkeypatch):
 
 
 def test_setuptools_importerror_issue1479(testdir, monkeypatch):
-    pkg_resources = pytest.importorskip("pkg_resources")
     monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False)
 
-    def my_iter(group, name=None):
+    class DummyEntryPoint(object):
-        assert group == "pytest11"
+        name = "mytestplugin"
+        group = "pytest11"
 
-        class Dist(object):
+        def load(self):
-            project_name = "spam"
+            raise ImportError("Don't hide me!")
-            version = "1.0"
 
-            def _get_metadata(self, name):
+    class Distribution(object):
-                return ["foo.txt,sha256=abc,123"]
+        version = "1.0"
+        files = ("foo.txt",)
+        entry_points = (DummyEntryPoint(),)
 
-        class EntryPoint(object):
+    def distributions():
-            name = "mytestplugin"
+        return (Distribution(),)
-            dist = Dist()
 
-            def load(self):
+    monkeypatch.setattr(importlib_metadata, "distributions", distributions)
-                raise ImportError("Don't hide me!")
-
-        return iter([EntryPoint()])
-
-    monkeypatch.setattr(pkg_resources, "iter_entry_points", my_iter)
     with pytest.raises(ImportError):
         testdir.parseconfig()
 
 
 @pytest.mark.parametrize("block_it", [True, False])
 def test_plugin_preparse_prevents_setuptools_loading(testdir, monkeypatch, block_it):
-    pkg_resources = pytest.importorskip("pkg_resources")
     monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False)
 
     plugin_module_placeholder = object()
 
-    def my_iter(group, name=None):
+    class DummyEntryPoint(object):
-        assert group == "pytest11"
+        name = "mytestplugin"
+        group = "pytest11"
 
-        class Dist(object):
+        def load(self):
-            project_name = "spam"
+            return plugin_module_placeholder
-            version = "1.0"
 
-            def _get_metadata(self, name):
+    class Distribution(object):
-                return ["foo.txt,sha256=abc,123"]
+        version = "1.0"
+        files = ("foo.txt",)
+        entry_points = (DummyEntryPoint(),)
 
-        class EntryPoint(object):
+    def distributions():
-            name = "mytestplugin"
+        return (Distribution(),)
-            dist = Dist()
 
-            def load(self):
+    monkeypatch.setattr(importlib_metadata, "distributions", distributions)
-                return plugin_module_placeholder
-
-        return iter([EntryPoint()])
-
-    monkeypatch.setattr(pkg_resources, "iter_entry_points", my_iter)
     args = ("-p", "no:mytestplugin") if block_it else ()
     config = testdir.parseconfig(*args)
     config.pluginmanager.import_plugin("mytestplugin")
@@ -639,37 +623,26 @@ def test_plugin_preparse_prevents_setuptools_loading(testdir, monkeypatch, block
     "parse_args,should_load", [(("-p", "mytestplugin"), True), ((), False)]
 )
 def test_disable_plugin_autoload(testdir, monkeypatch, parse_args, should_load):
-    pkg_resources = pytest.importorskip("pkg_resources")
-
-    def my_iter(group, name=None):
-        assert group == "pytest11"
-        assert name == "mytestplugin"
-        return iter([DummyEntryPoint()])
-
-    @attr.s
     class DummyEntryPoint(object):
-        name = "mytestplugin"
+        project_name = name = "mytestplugin"
+        group = "pytest11"
         version = "1.0"
 
-        @property
-        def project_name(self):
-            return self.name
-
         def load(self):
             return sys.modules[self.name]
 
-        @property
+    class Distribution(object):
-        def dist(self):
+        entry_points = (DummyEntryPoint(),)
-            return self
+        files = ()
-
-        def _get_metadata(self, *args):
-            return []
 
     class PseudoPlugin(object):
         x = 42
 
+    def distributions():
+        return (Distribution(),)
+
     monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1")
-    monkeypatch.setattr(pkg_resources, "iter_entry_points", my_iter)
+    monkeypatch.setattr(importlib_metadata, "distributions", distributions)
     monkeypatch.setitem(sys.modules, "mytestplugin", PseudoPlugin())
     config = testdir.parseconfig(*parse_args)
     has_loaded = config.pluginmanager.get_plugin("mytestplugin") is not None

testing/test_entry_points.py

@@ -2,16 +2,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import pkg_resources
+import importlib_metadata
-
-import pytest
-
-
-@pytest.mark.parametrize("entrypoint", ["py.test", "pytest"])
-def test_entry_point_exist(entrypoint):
-    assert entrypoint in pkg_resources.get_entry_map("pytest")["console_scripts"]
 
 
 def test_pytest_entry_points_are_identical():
-    entryMap = pkg_resources.get_entry_map("pytest")["console_scripts"]
+    dist = importlib_metadata.distribution("pytest")
-    assert entryMap["pytest"].module_name == entryMap["py.test"].module_name
+    entry_map = {ep.name: ep for ep in dist.entry_points}
+    assert entry_map["pytest"].value == entry_map["py.test"].value