Smoke tests for assorted plugins (#7721)

Co-authored-by: Bruno Oliveira <nicoddemus@gmail.com>
Co-authored-by: Thomas Grainger <tagrain@gmail.com>
Co-authored-by: Kyle Altendorf <sda@fstab.net>

.github/workflows/main.yml

@@ -41,6 +41,7 @@ jobs:
 
           "docs",
           "doctesting",
+          "plugins",
         ]
 
         include:
@@ -111,6 +112,11 @@ jobs:
             tox_env: "py38-xdist"
             use_coverage: true
 
+          - name: "plugins"
+            python: "3.7"
+            os: ubuntu-latest
+            tox_env: "plugins"
+
           - name: "docs"
             python: "3.7"
             os: ubuntu-latest

pyproject.toml

@@ -13,7 +13,7 @@ write_to = "src/_pytest/_version.py"
 [tool.pytest.ini_options]
 minversion = "2.0"
 addopts = "-rfEX -p pytester --strict-markers"
-python_files = ["test_*.py", "*_test.py", "testing/*/*.py"]
+python_files = ["test_*.py", "*_test.py", "testing/python/*.py"]
 python_classes = ["Test", "Acceptance"]
 python_functions = ["test"]
 # NOTE: "doc" is not included here, but gets tested explicitly via "doctesting".

src/_pytest/config/__init__.py

@@ -1167,6 +1167,10 @@ class Config:
             self.pluginmanager.load_setuptools_entrypoints("pytest11")
         self.pluginmanager.consider_env()
 
+        self.known_args_namespace = self._parser.parse_known_args(
+            args, namespace=copy.copy(self.known_args_namespace)
+        )
+
         self._validate_plugins()
         self._warn_about_skipped_plugins()
 

src/_pytest/python.py

@@ -164,9 +164,10 @@ def async_warn_and_skip(nodeid: str) -> None:
     msg += (
         "You need to install a suitable plugin for your async framework, for example:\n"
     )
+    msg += "  - anyio\n"
     msg += "  - pytest-asyncio\n"
-    msg += "  - pytest-trio\n"
     msg += "  - pytest-tornasync\n"
+    msg += "  - pytest-trio\n"
     msg += "  - pytest-twisted"
     warnings.warn(PytestUnhandledCoroutineWarning(msg.format(nodeid)))
     skip(msg="async def function and no async plugin installed (see warnings)")

testing/plugins_integration/.gitignore

@@ -0,0 +1,2 @@
+*.html
+assets/

testing/plugins_integration/README.rst

@@ -0,0 +1,13 @@
+This folder contains tests and support files for smoke testing popular plugins against the current pytest version.
+
+The objective is to gauge if any intentional or unintentional changes in pytest break plugins.
+
+As a rule of thumb, we should add plugins here:
+
+1. That are used at large. This might be subjective in some cases, but if answer is yes to
+   the question: *if a new release of pytest causes pytest-X to break, will this break a ton of test suites out there?*.
+2. That don't have large external dependencies: such as external services.
+
+Besides adding the plugin as dependency, we should also add a quick test which uses some
+minimal part of the plugin, a smoke test. Also consider reusing one of the existing tests if that's
+possible.

testing/plugins_integration/bdd_wallet.feature

@@ -0,0 +1,9 @@
+Feature: Buy things with apple
+
+    Scenario: Buy fruits
+        Given A wallet with 50
+
+        When I buy some apples for 1
+        And I buy some bananas for 2
+
+        Then I have 47 left

testing/plugins_integration/bdd_wallet.py

@@ -0,0 +1,39 @@
+from pytest_bdd import given
+from pytest_bdd import scenario
+from pytest_bdd import then
+from pytest_bdd import when
+
+import pytest
+
+
+@scenario("bdd_wallet.feature", "Buy fruits")
+def test_publish():
+    pass
+
+
+@pytest.fixture
+def wallet():
+    class Wallet:
+        amount = 0
+
+    return Wallet()
+
+
+@given("A wallet with 50")
+def fill_wallet(wallet):
+    wallet.amount = 50
+
+
+@when("I buy some apples for 1")
+def buy_apples(wallet):
+    wallet.amount -= 1
+
+
+@when("I buy some bananas for 2")
+def buy_bananas(wallet):
+    wallet.amount -= 2
+
+
+@then("I have 47 left")
+def check(wallet):
+    assert wallet.amount == 47

testing/plugins_integration/django_settings.py

@@ -0,0 +1 @@
+SECRET_KEY = "mysecret"

testing/plugins_integration/pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+addopts = --strict-markers
+filterwarnings =
+    error::pytest.PytestWarning

testing/plugins_integration/pytest_anyio_integration.py

@@ -0,0 +1,8 @@
+import anyio
+
+import pytest
+
+
+@pytest.mark.anyio
+async def test_sleep():
+    await anyio.sleep(0)

testing/plugins_integration/pytest_asyncio_integration.py

@@ -0,0 +1,8 @@
+import asyncio
+
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_sleep():
+    await asyncio.sleep(0)

testing/plugins_integration/pytest_mock_integration.py

@@ -0,0 +1,2 @@
+def test_mocker(mocker):
+    mocker.MagicMock()

testing/plugins_integration/pytest_trio_integration.py

@@ -0,0 +1,8 @@
+import trio
+
+import pytest
+
+
+@pytest.mark.trio
+async def test_sleep():
+    await trio.sleep(0)

testing/plugins_integration/pytest_twisted_integration.py

@@ -0,0 +1,18 @@
+import pytest_twisted
+from twisted.internet.task import deferLater
+
+
+def sleep():
+    import twisted.internet.reactor
+
+    return deferLater(clock=twisted.internet.reactor, delay=0)
+
+
+@pytest_twisted.inlineCallbacks
+def test_inlineCallbacks():
+    yield sleep()
+
+
+@pytest_twisted.ensureDeferred
+async def test_inlineCallbacks_async():
+    await sleep()

testing/plugins_integration/simple_integration.py

@@ -0,0 +1,10 @@
+import pytest
+
+
+def test_foo():
+    assert True
+
+
+@pytest.mark.parametrize("i", range(3))
+def test_bar(i):
+    assert True

testing/test_assertrewrite.py

@@ -990,7 +990,7 @@ class TestAssertionRewriteHookDetails:
                 e = OSError()
                 e.errno = 10
                 raise e
-                yield
+                yield  # type:ignore[unreachable]
 
             monkeypatch.setattr(
                 _pytest.assertion.rewrite, "atomic_write", atomic_write_failed
@@ -1597,7 +1597,7 @@ class TestPyCacheDir:
         if prefix:
             if sys.version_info < (3, 8):
                 pytest.skip("pycache_prefix not available in py<38")
-            monkeypatch.setattr(sys, "pycache_prefix", prefix)
+            monkeypatch.setattr(sys, "pycache_prefix", prefix)  # type:ignore
 
         assert get_cache_dir(Path(source)) == Path(expected)
 

testing/test_config.py

@@ -422,6 +422,25 @@ class TestParseIni:
         else:
             testdir.parseconfig()
 
+    def test_early_config_cmdline(self, testdir, monkeypatch):
+        """early_config contains options registered by third-party plugins.
+
+        This is a regression involving pytest-cov (and possibly others) introduced in #7700.
+        """
+        testdir.makepyfile(
+            myplugin="""
+            def pytest_addoption(parser):
+                parser.addoption('--foo', default=None, dest='foo')
+
+            def pytest_load_initial_conftests(early_config, parser, args):
+                assert early_config.known_args_namespace.foo == "1"
+            """
+        )
+        monkeypatch.setenv("PYTEST_PLUGINS", "myplugin")
+        testdir.syspathinsert()
+        result = testdir.runpytest("--foo=1")
+        result.stdout.fnmatch_lines("* no tests ran in *")
+
 
 class TestConfigCmdlineParsing:
     def test_parsing_again_fails(self, testdir):

tox.ini

@@ -13,6 +13,7 @@ envlist =
     pypy3
     py37-{pexpect,xdist,unittestextras,numpy,pluggymaster}
     doctesting
+    plugins
     py37-freeze
     docs
     docs-checklinks
@@ -114,6 +115,39 @@ commands =
     rm -rf {envdir}/.pytest_cache
     make regen
 
+[testenv:plugins]
+pip_pre=true
+changedir = testing/plugins_integration
+deps =
+    anyio[curio,trio]
+    django
+    pytest-asyncio
+    pytest-bdd
+    pytest-cov
+    pytest-django
+    pytest-flakes
+    pytest-html
+    pytest-mock
+    pytest-sugar
+    pytest-trio
+    pytest-twisted
+    twisted
+    pytest-xvfb
+setenv =
+    PYTHONPATH=.
+commands =
+    pip check
+    pytest bdd_wallet.py
+    pytest --cov=. simple_integration.py
+    pytest --ds=django_settings simple_integration.py
+    pytest --html=simple.html simple_integration.py
+    pytest pytest_anyio_integration.py
+    pytest pytest_asyncio_integration.py
+    pytest pytest_mock_integration.py
+    pytest pytest_trio_integration.py
+    pytest pytest_twisted_integration.py
+    pytest simple_integration.py --force-sugar --flakes
+
 [testenv:py37-freeze]
 changedir = testing/freeze
 deps =