From b031a7cecfe66367b0767300f454669b6234508f Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Sat, 19 Sep 2020 19:56:52 +0100 Subject: [PATCH] Smoke tests for assorted plugins (#7721) Co-authored-by: Bruno Oliveira Co-authored-by: Thomas Grainger Co-authored-by: Kyle Altendorf --- .github/workflows/main.yml | 6 +++ pyproject.toml | 2 +- src/_pytest/config/__init__.py | 4 ++ src/_pytest/python.py | 3 +- testing/plugins_integration/.gitignore | 2 + testing/plugins_integration/README.rst | 13 +++++++ .../plugins_integration/bdd_wallet.feature | 9 +++++ testing/plugins_integration/bdd_wallet.py | 39 +++++++++++++++++++ .../plugins_integration/django_settings.py | 1 + testing/plugins_integration/pytest.ini | 4 ++ .../pytest_anyio_integration.py | 8 ++++ .../pytest_asyncio_integration.py | 8 ++++ .../pytest_mock_integration.py | 2 + .../pytest_trio_integration.py | 8 ++++ .../pytest_twisted_integration.py | 18 +++++++++ .../plugins_integration/simple_integration.py | 10 +++++ testing/test_assertrewrite.py | 4 +- testing/test_config.py | 19 +++++++++ tox.ini | 34 ++++++++++++++++ 19 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 testing/plugins_integration/.gitignore create mode 100644 testing/plugins_integration/README.rst create mode 100644 testing/plugins_integration/bdd_wallet.feature create mode 100644 testing/plugins_integration/bdd_wallet.py create mode 100644 testing/plugins_integration/django_settings.py create mode 100644 testing/plugins_integration/pytest.ini create mode 100644 testing/plugins_integration/pytest_anyio_integration.py create mode 100644 testing/plugins_integration/pytest_asyncio_integration.py create mode 100644 testing/plugins_integration/pytest_mock_integration.py create mode 100644 testing/plugins_integration/pytest_trio_integration.py create mode 100644 testing/plugins_integration/pytest_twisted_integration.py create mode 100644 testing/plugins_integration/simple_integration.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 29d7cd1fe..9ef1a08b9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,6 +41,7 @@ jobs: "docs", "doctesting", + "plugins", ] include: @@ -111,6 +112,11 @@ jobs: tox_env: "py38-xdist" use_coverage: true + - name: "plugins" + python: "3.7" + os: ubuntu-latest + tox_env: "plugins" + - name: "docs" python: "3.7" os: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index 3d3d232f4..aee467fcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ write_to = "src/_pytest/_version.py" [tool.pytest.ini_options] minversion = "2.0" addopts = "-rfEX -p pytester --strict-markers" -python_files = ["test_*.py", "*_test.py", "testing/*/*.py"] +python_files = ["test_*.py", "*_test.py", "testing/python/*.py"] python_classes = ["Test", "Acceptance"] python_functions = ["test"] # NOTE: "doc" is not included here, but gets tested explicitly via "doctesting". diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 0f25b76a6..088ec765e 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1167,6 +1167,10 @@ class Config: self.pluginmanager.load_setuptools_entrypoints("pytest11") self.pluginmanager.consider_env() + self.known_args_namespace = self._parser.parse_known_args( + args, namespace=copy.copy(self.known_args_namespace) + ) + self._validate_plugins() self._warn_about_skipped_plugins() diff --git a/src/_pytest/python.py b/src/_pytest/python.py index eeccb4755..ea584f364 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -164,9 +164,10 @@ def async_warn_and_skip(nodeid: str) -> None: msg += ( "You need to install a suitable plugin for your async framework, for example:\n" ) + msg += " - anyio\n" msg += " - pytest-asyncio\n" - msg += " - pytest-trio\n" msg += " - pytest-tornasync\n" + msg += " - pytest-trio\n" msg += " - pytest-twisted" warnings.warn(PytestUnhandledCoroutineWarning(msg.format(nodeid))) skip(msg="async def function and no async plugin installed (see warnings)") diff --git a/testing/plugins_integration/.gitignore b/testing/plugins_integration/.gitignore new file mode 100644 index 000000000..d934447a0 --- /dev/null +++ b/testing/plugins_integration/.gitignore @@ -0,0 +1,2 @@ +*.html +assets/ diff --git a/testing/plugins_integration/README.rst b/testing/plugins_integration/README.rst new file mode 100644 index 000000000..8f027c3bd --- /dev/null +++ b/testing/plugins_integration/README.rst @@ -0,0 +1,13 @@ +This folder contains tests and support files for smoke testing popular plugins against the current pytest version. + +The objective is to gauge if any intentional or unintentional changes in pytest break plugins. + +As a rule of thumb, we should add plugins here: + +1. That are used at large. This might be subjective in some cases, but if answer is yes to + the question: *if a new release of pytest causes pytest-X to break, will this break a ton of test suites out there?*. +2. That don't have large external dependencies: such as external services. + +Besides adding the plugin as dependency, we should also add a quick test which uses some +minimal part of the plugin, a smoke test. Also consider reusing one of the existing tests if that's +possible. diff --git a/testing/plugins_integration/bdd_wallet.feature b/testing/plugins_integration/bdd_wallet.feature new file mode 100644 index 000000000..e404c4948 --- /dev/null +++ b/testing/plugins_integration/bdd_wallet.feature @@ -0,0 +1,9 @@ +Feature: Buy things with apple + + Scenario: Buy fruits + Given A wallet with 50 + + When I buy some apples for 1 + And I buy some bananas for 2 + + Then I have 47 left diff --git a/testing/plugins_integration/bdd_wallet.py b/testing/plugins_integration/bdd_wallet.py new file mode 100644 index 000000000..35927ea58 --- /dev/null +++ b/testing/plugins_integration/bdd_wallet.py @@ -0,0 +1,39 @@ +from pytest_bdd import given +from pytest_bdd import scenario +from pytest_bdd import then +from pytest_bdd import when + +import pytest + + +@scenario("bdd_wallet.feature", "Buy fruits") +def test_publish(): + pass + + +@pytest.fixture +def wallet(): + class Wallet: + amount = 0 + + return Wallet() + + +@given("A wallet with 50") +def fill_wallet(wallet): + wallet.amount = 50 + + +@when("I buy some apples for 1") +def buy_apples(wallet): + wallet.amount -= 1 + + +@when("I buy some bananas for 2") +def buy_bananas(wallet): + wallet.amount -= 2 + + +@then("I have 47 left") +def check(wallet): + assert wallet.amount == 47 diff --git a/testing/plugins_integration/django_settings.py b/testing/plugins_integration/django_settings.py new file mode 100644 index 000000000..0715f4765 --- /dev/null +++ b/testing/plugins_integration/django_settings.py @@ -0,0 +1 @@ +SECRET_KEY = "mysecret" diff --git a/testing/plugins_integration/pytest.ini b/testing/plugins_integration/pytest.ini new file mode 100644 index 000000000..f6c77b0de --- /dev/null +++ b/testing/plugins_integration/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +addopts = --strict-markers +filterwarnings = + error::pytest.PytestWarning diff --git a/testing/plugins_integration/pytest_anyio_integration.py b/testing/plugins_integration/pytest_anyio_integration.py new file mode 100644 index 000000000..65c2f5936 --- /dev/null +++ b/testing/plugins_integration/pytest_anyio_integration.py @@ -0,0 +1,8 @@ +import anyio + +import pytest + + +@pytest.mark.anyio +async def test_sleep(): + await anyio.sleep(0) diff --git a/testing/plugins_integration/pytest_asyncio_integration.py b/testing/plugins_integration/pytest_asyncio_integration.py new file mode 100644 index 000000000..5d2a3facc --- /dev/null +++ b/testing/plugins_integration/pytest_asyncio_integration.py @@ -0,0 +1,8 @@ +import asyncio + +import pytest + + +@pytest.mark.asyncio +async def test_sleep(): + await asyncio.sleep(0) diff --git a/testing/plugins_integration/pytest_mock_integration.py b/testing/plugins_integration/pytest_mock_integration.py new file mode 100644 index 000000000..740469d00 --- /dev/null +++ b/testing/plugins_integration/pytest_mock_integration.py @@ -0,0 +1,2 @@ +def test_mocker(mocker): + mocker.MagicMock() diff --git a/testing/plugins_integration/pytest_trio_integration.py b/testing/plugins_integration/pytest_trio_integration.py new file mode 100644 index 000000000..199f7850b --- /dev/null +++ b/testing/plugins_integration/pytest_trio_integration.py @@ -0,0 +1,8 @@ +import trio + +import pytest + + +@pytest.mark.trio +async def test_sleep(): + await trio.sleep(0) diff --git a/testing/plugins_integration/pytest_twisted_integration.py b/testing/plugins_integration/pytest_twisted_integration.py new file mode 100644 index 000000000..94748d036 --- /dev/null +++ b/testing/plugins_integration/pytest_twisted_integration.py @@ -0,0 +1,18 @@ +import pytest_twisted +from twisted.internet.task import deferLater + + +def sleep(): + import twisted.internet.reactor + + return deferLater(clock=twisted.internet.reactor, delay=0) + + +@pytest_twisted.inlineCallbacks +def test_inlineCallbacks(): + yield sleep() + + +@pytest_twisted.ensureDeferred +async def test_inlineCallbacks_async(): + await sleep() diff --git a/testing/plugins_integration/simple_integration.py b/testing/plugins_integration/simple_integration.py new file mode 100644 index 000000000..20b2fc4b5 --- /dev/null +++ b/testing/plugins_integration/simple_integration.py @@ -0,0 +1,10 @@ +import pytest + + +def test_foo(): + assert True + + +@pytest.mark.parametrize("i", range(3)) +def test_bar(i): + assert True diff --git a/testing/test_assertrewrite.py b/testing/test_assertrewrite.py index 63a6fdd12..ad3089a23 100644 --- a/testing/test_assertrewrite.py +++ b/testing/test_assertrewrite.py @@ -990,7 +990,7 @@ class TestAssertionRewriteHookDetails: e = OSError() e.errno = 10 raise e - yield + yield # type:ignore[unreachable] monkeypatch.setattr( _pytest.assertion.rewrite, "atomic_write", atomic_write_failed @@ -1597,7 +1597,7 @@ class TestPyCacheDir: if prefix: if sys.version_info < (3, 8): pytest.skip("pycache_prefix not available in py<38") - monkeypatch.setattr(sys, "pycache_prefix", prefix) + monkeypatch.setattr(sys, "pycache_prefix", prefix) # type:ignore assert get_cache_dir(Path(source)) == Path(expected) diff --git a/testing/test_config.py b/testing/test_config.py index aa84a3cf5..0cfd11fd5 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -422,6 +422,25 @@ class TestParseIni: else: testdir.parseconfig() + def test_early_config_cmdline(self, testdir, monkeypatch): + """early_config contains options registered by third-party plugins. + + This is a regression involving pytest-cov (and possibly others) introduced in #7700. + """ + testdir.makepyfile( + myplugin=""" + def pytest_addoption(parser): + parser.addoption('--foo', default=None, dest='foo') + + def pytest_load_initial_conftests(early_config, parser, args): + assert early_config.known_args_namespace.foo == "1" + """ + ) + monkeypatch.setenv("PYTEST_PLUGINS", "myplugin") + testdir.syspathinsert() + result = testdir.runpytest("--foo=1") + result.stdout.fnmatch_lines("* no tests ran in *") + class TestConfigCmdlineParsing: def test_parsing_again_fails(self, testdir): diff --git a/tox.ini b/tox.ini index 30aeb27be..642076dd4 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,7 @@ envlist = pypy3 py37-{pexpect,xdist,unittestextras,numpy,pluggymaster} doctesting + plugins py37-freeze docs docs-checklinks @@ -114,6 +115,39 @@ commands = rm -rf {envdir}/.pytest_cache make regen +[testenv:plugins] +pip_pre=true +changedir = testing/plugins_integration +deps = + anyio[curio,trio] + django + pytest-asyncio + pytest-bdd + pytest-cov + pytest-django + pytest-flakes + pytest-html + pytest-mock + pytest-sugar + pytest-trio + pytest-twisted + twisted + pytest-xvfb +setenv = + PYTHONPATH=. +commands = + pip check + pytest bdd_wallet.py + pytest --cov=. simple_integration.py + pytest --ds=django_settings simple_integration.py + pytest --html=simple.html simple_integration.py + pytest pytest_anyio_integration.py + pytest pytest_asyncio_integration.py + pytest pytest_mock_integration.py + pytest pytest_trio_integration.py + pytest pytest_twisted_integration.py + pytest simple_integration.py --force-sugar --flakes + [testenv:py37-freeze] changedir = testing/freeze deps =