From 62810f61b22341bc2b0a09da30252dfb0f57e027 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 26 Jul 2017 21:06:08 -0300 Subject: [PATCH 1/3] Make cache plugin always remember failed tests --- _pytest/cacheprovider.py | 26 ++++++++----------- changelog/2621.feature | 2 ++ testing/test_cache.py | 55 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 15 deletions(-) create mode 100644 changelog/2621.feature diff --git a/_pytest/cacheprovider.py b/_pytest/cacheprovider.py index 5f2c6b062..14fd86f6b 100755 --- a/_pytest/cacheprovider.py +++ b/_pytest/cacheprovider.py @@ -105,27 +105,22 @@ class LFPlugin: self.config = config active_keys = 'lf', 'failedfirst' self.active = any(config.getvalue(key) for key in active_keys) - if self.active: - self.lastfailed = config.cache.get("cache/lastfailed", {}) - else: - self.lastfailed = {} + self.lastfailed = config.cache.get("cache/lastfailed", {}) def pytest_report_header(self): if self.active: if not self.lastfailed: mode = "run all (no recorded failures)" else: - mode = "rerun last %d failures%s" % ( - len(self.lastfailed), + mode = "rerun previous failures%s" % ( " first" if self.config.getvalue("failedfirst") else "") return "run-last-failure: %s" % mode def pytest_runtest_logreport(self, report): - if report.failed and "xfail" not in report.keywords: + if report.passed and report.when == 'call': + self.lastfailed.pop(report.nodeid, None) + elif report.failed: self.lastfailed[report.nodeid] = True - elif not report.failed: - if report.when == "call": - self.lastfailed.pop(report.nodeid, None) def pytest_collectreport(self, report): passed = report.outcome in ('passed', 'skipped') @@ -147,11 +142,11 @@ class LFPlugin: previously_failed.append(item) else: previously_passed.append(item) - if not previously_failed and previously_passed: + if not previously_failed: # running a subset of all tests with recorded failures outside # of the set of tests currently executing - pass - elif self.config.getvalue("lf"): + return + if self.config.getvalue("lf"): items[:] = previously_failed config.hook.pytest_deselected(items=previously_passed) else: @@ -161,8 +156,9 @@ class LFPlugin: config = self.config if config.getvalue("cacheshow") or hasattr(config, "slaveinput"): return - prev_failed = config.cache.get("cache/lastfailed", None) is not None - if (session.testscollected and prev_failed) or self.lastfailed: + + saved_lastfailed = config.cache.get("cache/lastfailed", {}) + if saved_lastfailed != self.lastfailed: config.cache.set("cache/lastfailed", self.lastfailed) diff --git a/changelog/2621.feature b/changelog/2621.feature new file mode 100644 index 000000000..19ca96355 --- /dev/null +++ b/changelog/2621.feature @@ -0,0 +1,2 @@ +``--last-failed`` now remembers forever when a test has failed and only forgets it if it passes again. This makes it +easy to fix a test suite by selectively running files and fixing tests incrementally. diff --git a/testing/test_cache.py b/testing/test_cache.py index 36059ec29..04ce6671c 100755 --- a/testing/test_cache.py +++ b/testing/test_cache.py @@ -437,3 +437,58 @@ class TestLastFailed(object): testdir.makepyfile(test_errored='def test_error():\n assert False') testdir.runpytest('-q', '--lf') assert os.path.exists('.cache') + + def get_cached_last_failed(self, testdir): + config = testdir.parseconfigure() + return sorted(config.cache.get("cache/lastfailed", {})) + + def test_cache_cumulative(self, testdir): + """ + Test workflow where user fixes errors gradually file by file using --lf. + """ + # 1. initial run + test_bar = testdir.makepyfile(test_bar=""" + def test_bar_1(): + pass + def test_bar_2(): + assert 0 + """) + test_foo = testdir.makepyfile(test_foo=""" + def test_foo_3(): + pass + def test_foo_4(): + assert 0 + """) + testdir.runpytest() + assert self.get_cached_last_failed(testdir) == ['test_bar.py::test_bar_2', 'test_foo.py::test_foo_4'] + + # 2. fix test_bar_2, run only test_bar.py + testdir.makepyfile(test_bar=""" + def test_bar_1(): + pass + def test_bar_2(): + pass + """) + result = testdir.runpytest(test_bar) + result.stdout.fnmatch_lines('*2 passed*') + # ensure cache does not forget that test_foo_4 failed once before + assert self.get_cached_last_failed(testdir) == ['test_foo.py::test_foo_4'] + + result = testdir.runpytest('--last-failed') + result.stdout.fnmatch_lines('*1 failed, 3 deselected*') + assert self.get_cached_last_failed(testdir) == ['test_foo.py::test_foo_4'] + + # 3. fix test_foo_4, run only test_foo.py + test_foo = testdir.makepyfile(test_foo=""" + def test_foo_3(): + pass + def test_foo_4(): + pass + """) + result = testdir.runpytest(test_foo, '--last-failed') + result.stdout.fnmatch_lines('*1 passed, 1 deselected*') + assert self.get_cached_last_failed(testdir) == [] + + result = testdir.runpytest('--last-failed') + result.stdout.fnmatch_lines('*4 passed*') + assert self.get_cached_last_failed(testdir) == [] From 22212c4d61823e767de2b34da362960e4882113f Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 27 Jul 2017 10:56:04 -0300 Subject: [PATCH 2/3] Add xfail specific tests --- testing/test_cache.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/testing/test_cache.py b/testing/test_cache.py index 04ce6671c..1b2c1499e 100755 --- a/testing/test_cache.py +++ b/testing/test_cache.py @@ -438,6 +438,28 @@ class TestLastFailed(object): testdir.runpytest('-q', '--lf') assert os.path.exists('.cache') + def test_xfail_not_considered_failure(self, testdir): + testdir.makepyfile(''' + import pytest + @pytest.mark.xfail + def test(): + assert 0 + ''') + result = testdir.runpytest() + result.stdout.fnmatch_lines('*1 xfailed*') + assert self.get_cached_last_failed(testdir) == [] + + def test_xfail_strict_considered_failure(self, testdir): + testdir.makepyfile(''' + import pytest + @pytest.mark.xfail(strict=True) + def test(): + pass + ''') + result = testdir.runpytest() + result.stdout.fnmatch_lines('*1 failed*') + assert self.get_cached_last_failed(testdir) == ['test_xfail_strict_considered_failure.py::test'] + def get_cached_last_failed(self, testdir): config = testdir.parseconfigure() return sorted(config.cache.get("cache/lastfailed", {})) From eb1bd3449ecdb47e6b3502be39aaf699e3b8f09a Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 27 Jul 2017 18:43:04 -0300 Subject: [PATCH 3/3] xfail and skipped tests are removed from the "last-failed" cache This accommodates the case where a failing test is marked as skipped/failed later --- _pytest/cacheprovider.py | 2 +- testing/test_cache.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/_pytest/cacheprovider.py b/_pytest/cacheprovider.py index 14fd86f6b..6374453ec 100755 --- a/_pytest/cacheprovider.py +++ b/_pytest/cacheprovider.py @@ -117,7 +117,7 @@ class LFPlugin: return "run-last-failure: %s" % mode def pytest_runtest_logreport(self, report): - if report.passed and report.when == 'call': + if (report.when == 'call' and report.passed) or report.skipped: self.lastfailed.pop(report.nodeid, None) elif report.failed: self.lastfailed[report.nodeid] = True diff --git a/testing/test_cache.py b/testing/test_cache.py index 1b2c1499e..da86a202c 100755 --- a/testing/test_cache.py +++ b/testing/test_cache.py @@ -460,6 +460,28 @@ class TestLastFailed(object): result.stdout.fnmatch_lines('*1 failed*') assert self.get_cached_last_failed(testdir) == ['test_xfail_strict_considered_failure.py::test'] + @pytest.mark.parametrize('mark', ['mark.xfail', 'mark.skip']) + def test_failed_changed_to_xfail_or_skip(self, testdir, mark): + testdir.makepyfile(''' + import pytest + def test(): + assert 0 + ''') + result = testdir.runpytest() + assert self.get_cached_last_failed(testdir) == ['test_failed_changed_to_xfail_or_skip.py::test'] + assert result.ret == 1 + + testdir.makepyfile(''' + import pytest + @pytest.{mark} + def test(): + assert 0 + '''.format(mark=mark)) + result = testdir.runpytest() + assert result.ret == 0 + assert self.get_cached_last_failed(testdir) == [] + assert result.ret == 0 + def get_cached_last_failed(self, testdir): config = testdir.parseconfigure() return sorted(config.cache.get("cache/lastfailed", {}))