From bf3b26b3f72d46bf64a2a6c64da03f93f24b47af Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 29 May 2019 22:43:24 +0200 Subject: [PATCH 1/8] Fix regression with --lf and non-selected failures --- src/_pytest/cacheprovider.py | 1 + testing/test_cacheprovider.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index 3eaf7de28..b194f80b0 100755 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -178,6 +178,7 @@ class LFPlugin(object): """ if ( self.active + and self._previously_failed_count and self.config.getoption("lf") and path.isfile() and self.lastfailed diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index 5024701f1..da6a3da49 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -832,6 +832,26 @@ class TestLastFailed(object): ] ) + def test_lastfailed_with_unknown_failure(self, testdir): + testdir.makepyfile( + **{ + "pkg1/test_1.py": """def test_1(): assert 0""", + "pkg1/test_2.py": """def test_2(): pass""", + } + ) + result = testdir.runpytest() + result.stdout.fnmatch_lines(["collected 2 items", "* 1 failed, 1 passed in *"]) + + py.path.local("pkg1/test_1.py").remove() + result = testdir.runpytest("--lf") + result.stdout.fnmatch_lines( + [ + "collected 1 item", + "run-last-failure: 1 known failures not in selected tests", + "* 1 passed in *", + ] + ) + class TestNewFirst(object): def test_newfirst_usecase(self, testdir): From ceb4f3f701e330221f13a7ee66a2199b9b5e82e6 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 29 May 2019 23:12:45 +0200 Subject: [PATCH 2/8] fixup! Fix regression with --lf and non-selected failures --- changelog/5333.bugfix.rst | 1 + src/_pytest/cacheprovider.py | 19 ++++++++----------- testing/test_cacheprovider.py | 24 +++++++++++++++++++++++- 3 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 changelog/5333.bugfix.rst diff --git a/changelog/5333.bugfix.rst b/changelog/5333.bugfix.rst new file mode 100644 index 000000000..ac1d79585 --- /dev/null +++ b/changelog/5333.bugfix.rst @@ -0,0 +1 @@ +Fix regression with ``--lf`` not re-running all tests with known failures from non-selected tests. diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index b194f80b0..b13f5e819 100755 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -168,6 +168,7 @@ class LFPlugin(object): if result is None: rootpath = Path(self.config.rootdir) result = {rootpath / nodeid.split("::")[0] for nodeid in self.lastfailed} + result = {x for x in result if x.exists()} self._last_failed_paths = result return result @@ -176,17 +177,13 @@ class LFPlugin(object): Ignore this file path if we are in --lf mode and it is not in the list of previously failed files. """ - if ( - self.active - and self._previously_failed_count - and self.config.getoption("lf") - and path.isfile() - and self.lastfailed - ): - skip_it = Path(path) not in self.last_failed_paths() - if skip_it: - self._skipped_files += 1 - return skip_it + if self.active and self.config.getoption("lf") and path.isfile(): + last_failed_paths = self.last_failed_paths() + if last_failed_paths: + skip_it = Path(path) not in self.last_failed_paths() + if skip_it: + self._skipped_files += 1 + return skip_it def pytest_report_collectionfinish(self): if self.active and self.config.getoption("verbose") >= 0: diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index da6a3da49..ed205d0f8 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -832,7 +832,7 @@ class TestLastFailed(object): ] ) - def test_lastfailed_with_unknown_failure(self, testdir): + def test_lastfailed_with_known_failures_not_being_selected(self, testdir): testdir.makepyfile( **{ "pkg1/test_1.py": """def test_1(): assert 0""", @@ -852,6 +852,28 @@ class TestLastFailed(object): ] ) + # Recreate file with known failure. + testdir.makepyfile(**{"pkg1/test_1.py": """def test_1(): assert 0"""}) + result = testdir.runpytest("--lf") + result.stdout.fnmatch_lines( + [ + "collected 1 item", + "run-last-failure: rerun previous 1 failure (skipped 1 file)", + "* 1 failed in *", + ] + ) + + # Remove/rename test. + testdir.makepyfile(**{"pkg1/test_1.py": """def test_renamed(): assert 0"""}) + result = testdir.runpytest("--lf") + result.stdout.fnmatch_lines( + [ + "collected 1 item", + "run-last-failure: 1 known failures not in selected tests", + "* 1 failed in *", + ] + ) + class TestNewFirst(object): def test_newfirst_usecase(self, testdir): From 480dd9e6d6dd4869a7467ae17a7bea6e5367a3c4 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 29 May 2019 23:28:58 +0200 Subject: [PATCH 3/8] last_failed_paths: improve caching --- src/_pytest/cacheprovider.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index b13f5e819..106535467 100755 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -164,13 +164,14 @@ class LFPlugin(object): def last_failed_paths(self): """Returns a set with all Paths()s of the previously failed nodeids (cached). """ - result = getattr(self, "_last_failed_paths", None) - if result is None: + try: + return self._last_failed_paths + except AttributeError: rootpath = Path(self.config.rootdir) result = {rootpath / nodeid.split("::")[0] for nodeid in self.lastfailed} result = {x for x in result if x.exists()} self._last_failed_paths = result - return result + return result def pytest_ignore_collect(self, path): """ From ff80464b47d095906bd3d1c68c6afc68c105ca41 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 29 May 2019 23:32:52 +0200 Subject: [PATCH 4/8] last-failed: display skipped-files msg always --- src/_pytest/cacheprovider.py | 20 ++++++++------------ testing/test_cacheprovider.py | 2 +- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index 106535467..045248cb7 100755 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -233,19 +233,15 @@ class LFPlugin(object): items[:] = previously_failed + previously_passed noun = "failure" if self._previously_failed_count == 1 else "failures" - if self._skipped_files > 0: - files_noun = "file" if self._skipped_files == 1 else "files" - skipped_files_msg = " (skipped {files} {files_noun})".format( - files=self._skipped_files, files_noun=files_noun - ) - else: - skipped_files_msg = "" suffix = " first" if self.config.getoption("failedfirst") else "" - self._report_status = "rerun previous {count} {noun}{suffix}{skipped_files}".format( - count=self._previously_failed_count, - suffix=suffix, - noun=noun, - skipped_files=skipped_files_msg, + self._report_status = "rerun previous {count} {noun}{suffix}".format( + count=self._previously_failed_count, suffix=suffix, noun=noun + ) + + if self._skipped_files > 0: + files_noun = "file" if self._skipped_files == 1 else "files" + self._report_status += " (skipped {files} {files_noun})".format( + files=self._skipped_files, files_noun=files_noun ) else: self._report_status = "no previously failed tests, " diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index ed205d0f8..6de0b3129 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -869,7 +869,7 @@ class TestLastFailed(object): result.stdout.fnmatch_lines( [ "collected 1 item", - "run-last-failure: 1 known failures not in selected tests", + "run-last-failure: 1 known failures not in selected tests (skipped 1 file)", "* 1 failed in *", ] ) From 8e51563384d5f955e0c3db363f2b498e77d14c22 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Mon, 27 May 2019 03:00:32 +0200 Subject: [PATCH 5/8] tests: pdb: flush also on MacOS, but read() before Ref: https://github.com/pytest-dev/pytest/issues/2022 --- testing/test_pdb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/test_pdb.py b/testing/test_pdb.py index 007423a9e..119c2542d 100644 --- a/testing/test_pdb.py +++ b/testing/test_pdb.py @@ -4,7 +4,6 @@ from __future__ import division from __future__ import print_function import os -import platform import sys import six @@ -153,10 +152,11 @@ class TestPDB(object): @staticmethod def flush(child): - if platform.system() == "Darwin": - return if child.isalive(): + # Read if the test has not (e.g. test_pdb_unittest_skip). + child.read() child.wait() + assert not child.isalive() def test_pdb_unittest_postmortem(self, testdir): p1 = testdir.makepyfile( From fb12d2a612dc00dbc0a9f472b0cc6b305e00caf0 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Mon, 27 May 2019 03:27:37 +0200 Subject: [PATCH 6/8] test_enter_leave_pdb_hooks_are_called: remove child.sendeof() --- testing/test_pdb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testing/test_pdb.py b/testing/test_pdb.py index 119c2542d..0db3593d8 100644 --- a/testing/test_pdb.py +++ b/testing/test_pdb.py @@ -797,7 +797,6 @@ class TestPDB(object): rest = child.read().decode("utf8") assert "leave_pdb_hook" in rest assert "1 failed" in rest - child.sendeof() self.flush(child) def test_pdb_custom_cls(self, testdir, custom_pdb_calls): From 49c6aebbc7ab1a7fb7a58f6f3a61407175de0953 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Thu, 30 May 2019 03:10:42 +0200 Subject: [PATCH 7/8] ci: Travis: test with pexpect on macos Uses `PYTEST_COVERAGE=1` already, which will come in via dropping Python 2 soon anyway. --- .travis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 728048cce..3136dec1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,9 +20,6 @@ jobs: include: # OSX tests - first (in test stage), since they are the slower ones. - &test-macos - # NOTE: (tests with) pexpect appear to be buggy on Travis, - # at least with coverage. - # Log: https://travis-ci.org/pytest-dev/pytest/jobs/500358864 os: osx osx_image: xcode10.1 language: generic @@ -33,7 +30,7 @@ jobs: - python -V - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 27 - <<: *test-macos - env: TOXENV=py37-xdist + env: TOXENV=py37-pexpect,py37-xdist PYTEST_COVERAGE=1 before_install: - which python3 - python3 -V From 6896dbc5ca0b00ea7162ed3b31ebbea201c72fad Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Thu, 30 May 2019 03:52:41 +0200 Subject: [PATCH 8/8] tox: pexpect: use "-m uses_pexpect" --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a04f91ae5..0b1be0d33 100644 --- a/tox.ini +++ b/tox.ini @@ -40,7 +40,7 @@ setenv = lsof: _PYTEST_TOX_POSARGS_LSOF=--lsof pexpect: _PYTEST_TOX_PLATFORM=linux|darwin - pexpect: _PYTEST_TOX_POSARGS_PEXPECT=testing/test_pdb.py testing/test_terminal.py testing/test_unittest.py + pexpect: _PYTEST_TOX_POSARGS_PEXPECT=-m uses_pexpect twisted: _PYTEST_TOX_POSARGS_TWISTED=testing/test_unittest.py