p53209761
/
test_ok2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#7938 - [Plugin: Stepwise][Enhancements] Refactoring, smarter registration & --sw-skip functionality (#7939) 

* adding --sw-skip shorthand for stepwise skip

* be explicit rather than implicit with default args for stepwise

* add constant for sw cache dir; only register plugin if necessary rather check check activity always;

* use str format; remove unused args in hooks

* assert cache upfront, allow stepwise to have a reference to the cache

* type hinting lf, skip, move literal strings into module constants

* convert parametrized option into a list

* add a sessionfinish hook for stepwise to keep backwards behaviour the same

* add changelog for #7938

* Improve performance of stepwise modifyitems & address PR feedback

* add test for stepwise deselected based on performance enhancements

* Apply suggestions from code review

* delete from items, account for edge case where failed_index = 0

Co-authored-by: Bruno Oliveira <nicoddemus@gmail.com>
This commit is contained in:
Simon K 2020-10-30 19:13:06 +00:00 committed by GitHub
parent 0cd190f037
commit 6cddeb8cb3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 58 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog/7938.improvement.rst Normal file
View File

@ -0,0 +1 @@
New ``--sw-skip`` argument which is a shorthand for ``--stepwise-skip``.

77
src/_pytest/stepwise.py
View File

@ -1,5 +1,6 @@
from typing import List
from typing import Optional
from typing import TYPE_CHECKING
import pytest
from _pytest import nodes
@ -8,6 +9,11 @@ from _pytest.config.argparsing import Parser
from _pytest.main import Session
from _pytest.reports import TestReport
if TYPE_CHECKING:
from _pytest.cacheprovider import Cache
STEPWISE_CACHE_DIR = "cache/stepwise"
def pytest_addoption(parser: Parser) -> None:
group = parser.getgroup("general")
@ -15,12 +21,15 @@ def pytest_addoption(parser: Parser) -> None:
"--sw",
"--stepwise",
action="store_true",
default=False,
dest="stepwise",
help="exit on test failure and continue from last failing test next time",
)
group.addoption(
"--sw-skip",
"--stepwise-skip",
action="store_true",
default=False,
dest="stepwise_skip",
help="ignore the first failing test but stop on the next failing test",
)
@ -28,63 +37,56 @@ def pytest_addoption(parser: Parser) -> None:
@pytest.hookimpl
def pytest_configure(config: Config) -> None:
config.pluginmanager.register(StepwisePlugin(config), "stepwiseplugin")
# We should always have a cache as cache provider plugin uses tryfirst=True
if config.getoption("stepwise"):
config.pluginmanager.register(StepwisePlugin(config), "stepwiseplugin")
def pytest_sessionfinish(session: Session) -> None:
if not session.config.getoption("stepwise"):
assert session.config.cache is not None
# Clear the list of failing tests if the plugin is not active.
session.config.cache.set(STEPWISE_CACHE_DIR, [])
class StepwisePlugin:
def __init__(self, config: Config) -> None:
self.config = config
self.active = config.getvalue("stepwise")
self.session: Optional[Session] = None
self.report_status = ""
if self.active:
assert config.cache is not None
self.lastfailed = config.cache.get("cache/stepwise", None)
self.skip = config.getvalue("stepwise_skip")
assert config.cache is not None
self.cache: Cache = config.cache
self.lastfailed: Optional[str] = self.cache.get(STEPWISE_CACHE_DIR, None)
self.skip: bool = config.getoption("stepwise_skip")
def pytest_sessionstart(self, session: Session) -> None:
self.session = session
def pytest_collection_modifyitems(
self, session: Session, config: Config, items: List[nodes.Item]
self, config: Config, items: List[nodes.Item]
) -> None:
if not self.active:
return
if not self.lastfailed:
self.report_status = "no previously failed tests, not skipping."
return
already_passed = []
found = False
# Make a list of all tests that have been run before the last failing one.
for item in items:
# check all item nodes until we find a match on last failed
failed_index = None
for index, item in enumerate(items):
if item.nodeid == self.lastfailed:
found = True
failed_index = index
break
else:
already_passed.append(item)
# If the previously failed test was not found among the test items,
# do not skip any tests.
if not found:
if failed_index is None:
self.report_status = "previously failed test not found, not skipping."
already_passed = []
else:
self.report_status = "skipping {} already passed items.".format(
len(already_passed)
)
for item in already_passed:
items.remove(item)
config.hook.pytest_deselected(items=already_passed)
self.report_status = f"skipping {failed_index} already passed items."
deselected = items[:failed_index]
del items[:failed_index]
config.hook.pytest_deselected(items=deselected)
def pytest_runtest_logreport(self, report: TestReport) -> None:
if not self.active:
return
if report.failed:
if self.skip:
# Remove test from the failed ones (if it exists) and unset the skip option
@ -109,14 +111,9 @@ class StepwisePlugin:
self.lastfailed = None
def pytest_report_collectionfinish(self) -> Optional[str]:
if self.active and self.config.getoption("verbose") >= 0 and self.report_status:
return "stepwise: %s" % self.report_status
if self.config.getoption("verbose") >= 0 and self.report_status:
return f"stepwise: {self.report_status}"
return None
def pytest_sessionfinish(self, session: Session) -> None:
assert self.config.cache is not None
if self.active:
self.config.cache.set("cache/stepwise", self.lastfailed)
else:
# Clear the list of failing tests if the plugin is not active.
self.config.cache.set("cache/stepwise", [])
def pytest_sessionfinish(self) -> None:
self.cache.set(STEPWISE_CACHE_DIR, self.lastfailed)

27
testing/test_stepwise.py
View File

@ -93,6 +93,23 @@ def test_run_without_stepwise(stepwise_testdir):
result.stdout.fnmatch_lines(["*test_success_after_fail PASSED*"])
def test_stepwise_output_summary(testdir):
testdir.makepyfile(
"""
import pytest
@pytest.mark.parametrize("expected", [True, True, True, True, False])
def test_data(expected):
assert expected
"""
)
result = testdir.runpytest("-v", "--stepwise")
result.stdout.fnmatch_lines(["stepwise: no previously failed tests, not skipping."])
result = testdir.runpytest("-v", "--stepwise")
result.stdout.fnmatch_lines(
["stepwise: skipping 4 already passed items.", "*1 failed, 4 deselected*"]
)
def test_fail_and_continue_with_stepwise(stepwise_testdir):
# Run the tests with a failing second test.
result = stepwise_testdir.runpytest(
@ -117,14 +134,10 @@ def test_fail_and_continue_with_stepwise(stepwise_testdir):
assert "test_success_after_fail PASSED" in stdout
def test_run_with_skip_option(stepwise_testdir):
@pytest.mark.parametrize("stepwise_skip", ["--stepwise-skip", "--sw-skip"])
def test_run_with_skip_option(stepwise_testdir, stepwise_skip):
result = stepwise_testdir.runpytest(
"-v",
"--strict-markers",
"--stepwise",
"--stepwise-skip",
"--fail",
"--fail-last",
"-v", "--strict-markers", "--stepwise", stepwise_skip, "--fail", "--fail-last",
)
assert _strip_resource_warnings(result.stderr.lines) == []