pluginmanager.consider_preparse: add exclude_only kwarg (#6443)

Plugins specified with ``-p`` are now loaded after internal plugins, which
results in their hooks being called *before* the internal ones.

This makes the ``-p`` behavior consistent with ``PYTEST_PLUGINS``.

* fix/adjust test_disable_plugin_autoload
* adjust test_plugin_loading_order

changelog/6443.breaking.rst

@@ -0,0 +1,3 @@
+Plugins specified with ``-p`` are now loaded after internal plugins, which results in their hooks being called *before* the internal ones.
+
+This makes the ``-p`` behavior consistent with ``PYTEST_PLUGINS``.

src/_pytest/config/__init__.py

@@ -193,7 +193,7 @@ def get_config(args=None, plugins=None):
 
     if args is not None:
         # Handle any "-p no:plugin" args.
-        pluginmanager.consider_preparse(args)
+        pluginmanager.consider_preparse(args, exclude_only=True)
 
     for spec in default_plugins:
         pluginmanager.import_plugin(spec)
@@ -499,7 +499,7 @@ class PytestPluginManager(PluginManager):
     #
     #
 
-    def consider_preparse(self, args):
+    def consider_preparse(self, args, *, exclude_only=False):
         i = 0
         n = len(args)
         while i < n:
@@ -516,6 +516,8 @@ class PytestPluginManager(PluginManager):
                     parg = opt[2:]
                 else:
                     continue
+                if exclude_only and not parg.startswith("no:"):
+                    continue
                 self.consider_pluginarg(parg)
 
     def consider_pluginarg(self, arg):
@@ -951,7 +953,7 @@ class Config:
 
         self._checkversion()
         self._consider_importhook(args)
-        self.pluginmanager.consider_preparse(args)
+        self.pluginmanager.consider_preparse(args, exclude_only=False)
         if not os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"):
             # Don't autoload from setuptools entry point. Only explicitly specified
             # plugins are going to be loaded.

testing/test_config.py

@@ -659,6 +659,13 @@ def test_disable_plugin_autoload(testdir, monkeypatch, parse_args, should_load):
     class PseudoPlugin:
         x = 42
 
+        attrs_used = []
+
+        def __getattr__(self, name):
+            assert name == "__loader__"
+            self.attrs_used.append(name)
+            return object()
+
     def distributions():
         return (Distribution(),)
 
@@ -668,6 +675,10 @@ def test_disable_plugin_autoload(testdir, monkeypatch, parse_args, should_load):
     config = testdir.parseconfig(*parse_args)
     has_loaded = config.pluginmanager.get_plugin("mytestplugin") is not None
     assert has_loaded == should_load
+    if should_load:
+        assert PseudoPlugin.attrs_used == ["__loader__"]
+    else:
+        assert PseudoPlugin.attrs_used == []
 
 
 def test_plugin_loading_order(testdir):
@@ -676,7 +687,7 @@ def test_plugin_loading_order(testdir):
         """
         def test_terminal_plugin(request):
             import myplugin
-            assert myplugin.terminal_plugin == [True, True]
+            assert myplugin.terminal_plugin == [False, True]
         """,
         **{
             "myplugin": """

testing/test_warnings.py

@@ -679,10 +679,7 @@ class TestStackLevel:
         # with stacklevel=2 the warning should originate from
         # config.PytestPluginManager.import_plugin is thrown by a skipped plugin
 
-        # During config parsing the the pluginargs are checked in a while loop
-        # that as a result of the argument count runs import_plugin twice, hence
-        # two identical warnings are captured (is this intentional?).
-        assert len(capwarn.captured) == 2
+        assert len(capwarn.captured) == 1
         warning, location = capwarn.captured.pop()
         file, _, func = location