From 6cddeb8cb3779480ea8c57a62fdf109b1bfe2271 Mon Sep 17 00:00:00 2001 From: Simon K Date: Fri, 30 Oct 2020 19:13:06 +0000 Subject: [PATCH] #7938 - [Plugin: Stepwise][Enhancements] Refactoring, smarter registration & --sw-skip functionality (#7939) * adding --sw-skip shorthand for stepwise skip * be explicit rather than implicit with default args for stepwise * add constant for sw cache dir; only register plugin if necessary rather check check activity always; * use str format; remove unused args in hooks * assert cache upfront, allow stepwise to have a reference to the cache * type hinting lf, skip, move literal strings into module constants * convert parametrized option into a list * add a sessionfinish hook for stepwise to keep backwards behaviour the same * add changelog for #7938 * Improve performance of stepwise modifyitems & address PR feedback * add test for stepwise deselected based on performance enhancements * Apply suggestions from code review * delete from items, account for edge case where failed_index = 0 Co-authored-by: Bruno Oliveira --- changelog/7938.improvement.rst | 1 + src/_pytest/stepwise.py | 77 ++++++++++++++++------------------ testing/test_stepwise.py | 27 ++++++++---- 3 files changed, 58 insertions(+), 47 deletions(-) create mode 100644 changelog/7938.improvement.rst diff --git a/changelog/7938.improvement.rst b/changelog/7938.improvement.rst new file mode 100644 index 000000000..ffe612d0d --- /dev/null +++ b/changelog/7938.improvement.rst @@ -0,0 +1 @@ +New ``--sw-skip`` argument which is a shorthand for ``--stepwise-skip``. diff --git a/src/_pytest/stepwise.py b/src/_pytest/stepwise.py index 97eae18fd..197577c79 100644 --- a/src/_pytest/stepwise.py +++ b/src/_pytest/stepwise.py @@ -1,5 +1,6 @@ from typing import List from typing import Optional +from typing import TYPE_CHECKING import pytest from _pytest import nodes @@ -8,6 +9,11 @@ from _pytest.config.argparsing import Parser from _pytest.main import Session from _pytest.reports import TestReport +if TYPE_CHECKING: + from _pytest.cacheprovider import Cache + +STEPWISE_CACHE_DIR = "cache/stepwise" + def pytest_addoption(parser: Parser) -> None: group = parser.getgroup("general") @@ -15,12 +21,15 @@ def pytest_addoption(parser: Parser) -> None: "--sw", "--stepwise", action="store_true", + default=False, dest="stepwise", help="exit on test failure and continue from last failing test next time", ) group.addoption( + "--sw-skip", "--stepwise-skip", action="store_true", + default=False, dest="stepwise_skip", help="ignore the first failing test but stop on the next failing test", ) @@ -28,63 +37,56 @@ def pytest_addoption(parser: Parser) -> None: @pytest.hookimpl def pytest_configure(config: Config) -> None: - config.pluginmanager.register(StepwisePlugin(config), "stepwiseplugin") + # We should always have a cache as cache provider plugin uses tryfirst=True + if config.getoption("stepwise"): + config.pluginmanager.register(StepwisePlugin(config), "stepwiseplugin") + + +def pytest_sessionfinish(session: Session) -> None: + if not session.config.getoption("stepwise"): + assert session.config.cache is not None + # Clear the list of failing tests if the plugin is not active. + session.config.cache.set(STEPWISE_CACHE_DIR, []) class StepwisePlugin: def __init__(self, config: Config) -> None: self.config = config - self.active = config.getvalue("stepwise") self.session: Optional[Session] = None self.report_status = "" - - if self.active: - assert config.cache is not None - self.lastfailed = config.cache.get("cache/stepwise", None) - self.skip = config.getvalue("stepwise_skip") + assert config.cache is not None + self.cache: Cache = config.cache + self.lastfailed: Optional[str] = self.cache.get(STEPWISE_CACHE_DIR, None) + self.skip: bool = config.getoption("stepwise_skip") def pytest_sessionstart(self, session: Session) -> None: self.session = session def pytest_collection_modifyitems( - self, session: Session, config: Config, items: List[nodes.Item] + self, config: Config, items: List[nodes.Item] ) -> None: - if not self.active: - return if not self.lastfailed: self.report_status = "no previously failed tests, not skipping." return - already_passed = [] - found = False - - # Make a list of all tests that have been run before the last failing one. - for item in items: + # check all item nodes until we find a match on last failed + failed_index = None + for index, item in enumerate(items): if item.nodeid == self.lastfailed: - found = True + failed_index = index break - else: - already_passed.append(item) # If the previously failed test was not found among the test items, # do not skip any tests. - if not found: + if failed_index is None: self.report_status = "previously failed test not found, not skipping." - already_passed = [] else: - self.report_status = "skipping {} already passed items.".format( - len(already_passed) - ) - - for item in already_passed: - items.remove(item) - - config.hook.pytest_deselected(items=already_passed) + self.report_status = f"skipping {failed_index} already passed items." + deselected = items[:failed_index] + del items[:failed_index] + config.hook.pytest_deselected(items=deselected) def pytest_runtest_logreport(self, report: TestReport) -> None: - if not self.active: - return - if report.failed: if self.skip: # Remove test from the failed ones (if it exists) and unset the skip option @@ -109,14 +111,9 @@ class StepwisePlugin: self.lastfailed = None def pytest_report_collectionfinish(self) -> Optional[str]: - if self.active and self.config.getoption("verbose") >= 0 and self.report_status: - return "stepwise: %s" % self.report_status + if self.config.getoption("verbose") >= 0 and self.report_status: + return f"stepwise: {self.report_status}" return None - def pytest_sessionfinish(self, session: Session) -> None: - assert self.config.cache is not None - if self.active: - self.config.cache.set("cache/stepwise", self.lastfailed) - else: - # Clear the list of failing tests if the plugin is not active. - self.config.cache.set("cache/stepwise", []) + def pytest_sessionfinish(self) -> None: + self.cache.set(STEPWISE_CACHE_DIR, self.lastfailed) diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py index df66d798b..bf8d94f1d 100644 --- a/testing/test_stepwise.py +++ b/testing/test_stepwise.py @@ -93,6 +93,23 @@ def test_run_without_stepwise(stepwise_testdir): result.stdout.fnmatch_lines(["*test_success_after_fail PASSED*"]) +def test_stepwise_output_summary(testdir): + testdir.makepyfile( + """ + import pytest + @pytest.mark.parametrize("expected", [True, True, True, True, False]) + def test_data(expected): + assert expected + """ + ) + result = testdir.runpytest("-v", "--stepwise") + result.stdout.fnmatch_lines(["stepwise: no previously failed tests, not skipping."]) + result = testdir.runpytest("-v", "--stepwise") + result.stdout.fnmatch_lines( + ["stepwise: skipping 4 already passed items.", "*1 failed, 4 deselected*"] + ) + + def test_fail_and_continue_with_stepwise(stepwise_testdir): # Run the tests with a failing second test. result = stepwise_testdir.runpytest( @@ -117,14 +134,10 @@ def test_fail_and_continue_with_stepwise(stepwise_testdir): assert "test_success_after_fail PASSED" in stdout -def test_run_with_skip_option(stepwise_testdir): +@pytest.mark.parametrize("stepwise_skip", ["--stepwise-skip", "--sw-skip"]) +def test_run_with_skip_option(stepwise_testdir, stepwise_skip): result = stepwise_testdir.runpytest( - "-v", - "--strict-markers", - "--stepwise", - "--stepwise-skip", - "--fail", - "--fail-last", + "-v", "--strict-markers", "--stepwise", stepwise_skip, "--fail", "--fail-last", ) assert _strip_resource_warnings(result.stderr.lines) == []