from __future__ import absolute_import, division, print_function import sys import py import _pytest import pytest import os import shutil pytest_plugins = ("pytester",) class TestNewAPI(object): def test_config_cache_makedir(self, testdir): testdir.makeini("[pytest]") config = testdir.parseconfigure() with pytest.raises(ValueError): config.cache.makedir("key/name") p = config.cache.makedir("name") assert p.check() def test_config_cache_dataerror(self, testdir): testdir.makeini("[pytest]") config = testdir.parseconfigure() cache = config.cache pytest.raises(TypeError, lambda: cache.set("key/name", cache)) config.cache.set("key/name", 0) config.cache._getvaluepath("key/name").write_bytes(b"123invalid") val = config.cache.get("key/name", -2) assert val == -2 def test_cache_writefail_cachfile_silent(self, testdir): testdir.makeini("[pytest]") testdir.tmpdir.join(".pytest_cache").write("gone wrong") config = testdir.parseconfigure() cache = config.cache cache.set("test/broken", []) @pytest.mark.skipif(sys.platform.startswith("win"), reason="no chmod on windows") def test_cache_writefail_permissions(self, testdir): testdir.makeini("[pytest]") testdir.tmpdir.ensure_dir(".pytest_cache").chmod(0) config = testdir.parseconfigure() cache = config.cache cache.set("test/broken", []) @pytest.mark.skipif(sys.platform.startswith("win"), reason="no chmod on windows") def test_cache_failure_warns(self, testdir): testdir.tmpdir.ensure_dir(".pytest_cache").chmod(0) testdir.makepyfile( """ def test_error(): raise Exception """ ) result = testdir.runpytest("-rw") assert result.ret == 1 result.stdout.fnmatch_lines(["*could not create cache path*", "*2 warnings*"]) def test_config_cache(self, testdir): testdir.makeconftest( """ def pytest_configure(config): # see that we get cache information early on assert hasattr(config, "cache") """ ) testdir.makepyfile( """ def test_session(pytestconfig): assert hasattr(pytestconfig, "cache") """ ) result = testdir.runpytest() assert result.ret == 0 result.stdout.fnmatch_lines(["*1 passed*"]) def test_cachefuncarg(self, testdir): testdir.makepyfile( """ import pytest def test_cachefuncarg(cache): val = cache.get("some/thing", None) assert val is None cache.set("some/thing", [1]) pytest.raises(TypeError, lambda: cache.get("some/thing")) val = cache.get("some/thing", []) assert val == [1] """ ) result = testdir.runpytest() assert result.ret == 0 result.stdout.fnmatch_lines(["*1 passed*"]) def test_custom_rel_cache_dir(self, testdir): rel_cache_dir = os.path.join("custom_cache_dir", "subdir") testdir.makeini( """ [pytest] cache_dir = {cache_dir} """.format( cache_dir=rel_cache_dir ) ) testdir.makepyfile(test_errored="def test_error():\n assert False") testdir.runpytest() assert testdir.tmpdir.join(rel_cache_dir).isdir() def test_custom_abs_cache_dir(self, testdir, tmpdir_factory): tmp = str(tmpdir_factory.mktemp("tmp")) abs_cache_dir = os.path.join(tmp, "custom_cache_dir") testdir.makeini( """ [pytest] cache_dir = {cache_dir} """.format( cache_dir=abs_cache_dir ) ) testdir.makepyfile(test_errored="def test_error():\n assert False") testdir.runpytest() assert py.path.local(abs_cache_dir).isdir() def test_custom_cache_dir_with_env_var(self, testdir, monkeypatch): monkeypatch.setenv("env_var", "custom_cache_dir") testdir.makeini( """ [pytest] cache_dir = {cache_dir} """.format( cache_dir="$env_var" ) ) testdir.makepyfile(test_errored="def test_error():\n assert False") testdir.runpytest() assert testdir.tmpdir.join("custom_cache_dir").isdir() def test_cache_reportheader(testdir): testdir.makepyfile( """ def test_hello(): pass """ ) result = testdir.runpytest("-v") result.stdout.fnmatch_lines(["cachedir: .pytest_cache"]) def test_cache_reportheader_external_abspath(testdir, tmpdir_factory): external_cache = tmpdir_factory.mktemp( "test_cache_reportheader_external_abspath_abs" ) testdir.makepyfile( """ def test_hello(): pass """ ) testdir.makeini( """ [pytest] cache_dir = {abscache} """.format( abscache=external_cache ) ) result = testdir.runpytest("-v") result.stdout.fnmatch_lines( ["cachedir: {abscache}".format(abscache=external_cache)] ) def test_cache_show(testdir): result = testdir.runpytest("--cache-show") assert result.ret == 0 result.stdout.fnmatch_lines(["*cache is empty*"]) testdir.makeconftest( """ def pytest_configure(config): config.cache.set("my/name", [1,2,3]) config.cache.set("other/some", {1:2}) dp = config.cache.makedir("mydb") dp.ensure("hello") dp.ensure("world") """ ) result = testdir.runpytest() assert result.ret == 5 # no tests executed result = testdir.runpytest("--cache-show") result.stdout.fnmatch_lines_random( [ "*cachedir:*", "-*cache values*-", "*my/name contains:", " [1, 2, 3]", "*other/some contains*", " {*1*: 2}", "-*cache directories*-", "*mydb/hello*length 0*", "*mydb/world*length 0*", ] ) class TestLastFailed(object): def test_lastfailed_usecase(self, testdir, monkeypatch): monkeypatch.setenv("PYTHONDONTWRITEBYTECODE", 1) p = testdir.makepyfile( """ def test_1(): assert 0 def test_2(): assert 0 def test_3(): assert 1 """ ) result = testdir.runpytest() result.stdout.fnmatch_lines(["*2 failed*"]) p.write( _pytest._code.Source( """ def test_1(): assert 1 def test_2(): assert 1 def test_3(): assert 0 """ ) ) result = testdir.runpytest("--lf") result.stdout.fnmatch_lines(["*2 passed*1 desel*"]) result = testdir.runpytest("--lf") result.stdout.fnmatch_lines(["*1 failed*2 passed*"]) result = testdir.runpytest("--lf", "--cache-clear") result.stdout.fnmatch_lines(["*1 failed*2 passed*"]) # Run this again to make sure clear-cache is robust if os.path.isdir(".pytest_cache"): shutil.rmtree(".pytest_cache") result = testdir.runpytest("--lf", "--cache-clear") result.stdout.fnmatch_lines(["*1 failed*2 passed*"]) def test_failedfirst_order(self, testdir): testdir.tmpdir.join("test_a.py").write( _pytest._code.Source( """ def test_always_passes(): assert 1 """ ) ) testdir.tmpdir.join("test_b.py").write( _pytest._code.Source( """ def test_always_fails(): assert 0 """ ) ) result = testdir.runpytest() # Test order will be collection order; alphabetical result.stdout.fnmatch_lines(["test_a.py*", "test_b.py*"]) result = testdir.runpytest("--ff") # Test order will be failing tests firs result.stdout.fnmatch_lines(["test_b.py*", "test_a.py*"]) def test_lastfailed_failedfirst_order(self, testdir): testdir.makepyfile( **{ "test_a.py": """ def test_always_passes(): assert 1 """, "test_b.py": """ def test_always_fails(): assert 0 """, } ) result = testdir.runpytest() # Test order will be collection order; alphabetical result.stdout.fnmatch_lines(["test_a.py*", "test_b.py*"]) result = testdir.runpytest("--lf", "--ff") # Test order will be failing tests firs result.stdout.fnmatch_lines(["test_b.py*"]) assert "test_a.py" not in result.stdout.str() def test_lastfailed_difference_invocations(self, testdir, monkeypatch): monkeypatch.setenv("PYTHONDONTWRITEBYTECODE", 1) testdir.makepyfile( test_a=""" def test_a1(): assert 0 def test_a2(): assert 1 """, test_b=""" def test_b1(): assert 0 """, ) p = testdir.tmpdir.join("test_a.py") p2 = testdir.tmpdir.join("test_b.py") result = testdir.runpytest() result.stdout.fnmatch_lines(["*2 failed*"]) result = testdir.runpytest("--lf", p2) result.stdout.fnmatch_lines(["*1 failed*"]) p2.write( _pytest._code.Source( """ def test_b1(): assert 1 """ ) ) result = testdir.runpytest("--lf", p2) result.stdout.fnmatch_lines(["*1 passed*"]) result = testdir.runpytest("--lf", p) result.stdout.fnmatch_lines(["*1 failed*1 desel*"]) def test_lastfailed_usecase_splice(self, testdir, monkeypatch): monkeypatch.setenv("PYTHONDONTWRITEBYTECODE", 1) testdir.makepyfile( """ def test_1(): assert 0 """ ) p2 = testdir.tmpdir.join("test_something.py") p2.write( _pytest._code.Source( """ def test_2(): assert 0 """ ) ) result = testdir.runpytest() result.stdout.fnmatch_lines(["*2 failed*"]) result = testdir.runpytest("--lf", p2) result.stdout.fnmatch_lines(["*1 failed*"]) result = testdir.runpytest("--lf") result.stdout.fnmatch_lines(["*2 failed*"]) def test_lastfailed_xpass(self, testdir): testdir.inline_runsource( """ import pytest @pytest.mark.xfail def test_hello(): assert 1 """ ) config = testdir.parseconfigure() lastfailed = config.cache.get("cache/lastfailed", -1) assert lastfailed == -1 def test_non_serializable_parametrize(self, testdir): """Test that failed parametrized tests with unmarshable parameters don't break pytest-cache. """ testdir.makepyfile( r""" import pytest @pytest.mark.parametrize('val', [ b'\xac\x10\x02G', ]) def test_fail(val): assert False """ ) result = testdir.runpytest() result.stdout.fnmatch_lines("*1 failed in*") def test_terminal_report_lastfailed(self, testdir): test_a = testdir.makepyfile( test_a=""" def test_a1(): pass def test_a2(): pass """ ) test_b = testdir.makepyfile( test_b=""" def test_b1(): assert 0 def test_b2(): assert 0 """ ) result = testdir.runpytest() result.stdout.fnmatch_lines(["collected 4 items", "*2 failed, 2 passed in*"]) result = testdir.runpytest("--lf") result.stdout.fnmatch_lines( [ "collected 4 items / 2 deselected", "run-last-failure: rerun previous 2 failures", "*2 failed, 2 deselected in*", ] ) result = testdir.runpytest(test_a, "--lf") result.stdout.fnmatch_lines( [ "collected 2 items", "run-last-failure: run all (no recorded failures)", "*2 passed in*", ] ) result = testdir.runpytest(test_b, "--lf") result.stdout.fnmatch_lines( [ "collected 2 items", "run-last-failure: rerun previous 2 failures", "*2 failed in*", ] ) result = testdir.runpytest("test_b.py::test_b1", "--lf") result.stdout.fnmatch_lines( [ "collected 1 item", "run-last-failure: rerun previous 1 failure", "*1 failed in*", ] ) def test_terminal_report_failedfirst(self, testdir): testdir.makepyfile( test_a=""" def test_a1(): assert 0 def test_a2(): pass """ ) result = testdir.runpytest() result.stdout.fnmatch_lines(["collected 2 items", "*1 failed, 1 passed in*"]) result = testdir.runpytest("--ff") result.stdout.fnmatch_lines( [ "collected 2 items", "run-last-failure: rerun previous 1 failure first", "*1 failed, 1 passed in*", ] ) def test_lastfailed_collectfailure(self, testdir, monkeypatch): testdir.makepyfile( test_maybe=""" import os env = os.environ if '1' == env['FAILIMPORT']: raise ImportError('fail') def test_hello(): assert '0' == env['FAILTEST'] """ ) def rlf(fail_import, fail_run): monkeypatch.setenv("FAILIMPORT", fail_import) monkeypatch.setenv("FAILTEST", fail_run) testdir.runpytest("-q") config = testdir.parseconfigure() lastfailed = config.cache.get("cache/lastfailed", -1) return lastfailed lastfailed = rlf(fail_import=0, fail_run=0) assert lastfailed == -1 lastfailed = rlf(fail_import=1, fail_run=0) assert list(lastfailed) == ["test_maybe.py"] lastfailed = rlf(fail_import=0, fail_run=1) assert list(lastfailed) == ["test_maybe.py::test_hello"] def test_lastfailed_failure_subset(self, testdir, monkeypatch): testdir.makepyfile( test_maybe=""" import os env = os.environ if '1' == env['FAILIMPORT']: raise ImportError('fail') def test_hello(): assert '0' == env['FAILTEST'] """ ) testdir.makepyfile( test_maybe2=""" import os env = os.environ if '1' == env['FAILIMPORT']: raise ImportError('fail') def test_hello(): assert '0' == env['FAILTEST'] def test_pass(): pass """ ) def rlf(fail_import, fail_run, args=()): monkeypatch.setenv("FAILIMPORT", fail_import) monkeypatch.setenv("FAILTEST", fail_run) result = testdir.runpytest("-q", "--lf", *args) config = testdir.parseconfigure() lastfailed = config.cache.get("cache/lastfailed", -1) return result, lastfailed result, lastfailed = rlf(fail_import=0, fail_run=0) assert lastfailed == -1 result.stdout.fnmatch_lines(["*3 passed*"]) result, lastfailed = rlf(fail_import=1, fail_run=0) assert sorted(list(lastfailed)) == ["test_maybe.py", "test_maybe2.py"] result, lastfailed = rlf(fail_import=0, fail_run=0, args=("test_maybe2.py",)) assert list(lastfailed) == ["test_maybe.py"] # edge case of test selection - even if we remember failures # from other tests we still need to run all tests if no test # matches the failures result, lastfailed = rlf(fail_import=0, fail_run=0, args=("test_maybe2.py",)) assert list(lastfailed) == ["test_maybe.py"] result.stdout.fnmatch_lines(["*2 passed*"]) def test_lastfailed_creates_cache_when_needed(self, testdir): # Issue #1342 testdir.makepyfile(test_empty="") testdir.runpytest("-q", "--lf") assert not os.path.exists(".pytest_cache/v/cache/lastfailed") testdir.makepyfile(test_successful="def test_success():\n assert True") testdir.runpytest("-q", "--lf") assert not os.path.exists(".pytest_cache/v/cache/lastfailed") testdir.makepyfile(test_errored="def test_error():\n assert False") testdir.runpytest("-q", "--lf") assert os.path.exists(".pytest_cache/v/cache/lastfailed") def test_xfail_not_considered_failure(self, testdir): testdir.makepyfile( """ import pytest @pytest.mark.xfail def test(): assert 0 """ ) result = testdir.runpytest() result.stdout.fnmatch_lines("*1 xfailed*") assert self.get_cached_last_failed(testdir) == [] def test_xfail_strict_considered_failure(self, testdir): testdir.makepyfile( """ import pytest @pytest.mark.xfail(strict=True) def test(): pass """ ) result = testdir.runpytest() result.stdout.fnmatch_lines("*1 failed*") assert self.get_cached_last_failed(testdir) == [ "test_xfail_strict_considered_failure.py::test" ] @pytest.mark.parametrize("mark", ["mark.xfail", "mark.skip"]) def test_failed_changed_to_xfail_or_skip(self, testdir, mark): testdir.makepyfile( """ import pytest def test(): assert 0 """ ) result = testdir.runpytest() assert self.get_cached_last_failed(testdir) == [ "test_failed_changed_to_xfail_or_skip.py::test" ] assert result.ret == 1 testdir.makepyfile( """ import pytest @pytest.{mark} def test(): assert 0 """.format( mark=mark ) ) result = testdir.runpytest() assert result.ret == 0 assert self.get_cached_last_failed(testdir) == [] assert result.ret == 0 def get_cached_last_failed(self, testdir): config = testdir.parseconfigure() return sorted(config.cache.get("cache/lastfailed", {})) def test_cache_cumulative(self, testdir): """ Test workflow where user fixes errors gradually file by file using --lf. """ # 1. initial run test_bar = testdir.makepyfile( test_bar=""" def test_bar_1(): pass def test_bar_2(): assert 0 """ ) test_foo = testdir.makepyfile( test_foo=""" def test_foo_3(): pass def test_foo_4(): assert 0 """ ) testdir.runpytest() assert self.get_cached_last_failed(testdir) == [ "test_bar.py::test_bar_2", "test_foo.py::test_foo_4", ] # 2. fix test_bar_2, run only test_bar.py testdir.makepyfile( test_bar=""" def test_bar_1(): pass def test_bar_2(): pass """ ) result = testdir.runpytest(test_bar) result.stdout.fnmatch_lines("*2 passed*") # ensure cache does not forget that test_foo_4 failed once before assert self.get_cached_last_failed(testdir) == ["test_foo.py::test_foo_4"] result = testdir.runpytest("--last-failed") result.stdout.fnmatch_lines("*1 failed, 3 deselected*") assert self.get_cached_last_failed(testdir) == ["test_foo.py::test_foo_4"] # 3. fix test_foo_4, run only test_foo.py test_foo = testdir.makepyfile( test_foo=""" def test_foo_3(): pass def test_foo_4(): pass """ ) result = testdir.runpytest(test_foo, "--last-failed") result.stdout.fnmatch_lines("*1 passed, 1 deselected*") assert self.get_cached_last_failed(testdir) == [] result = testdir.runpytest("--last-failed") result.stdout.fnmatch_lines("*4 passed*") assert self.get_cached_last_failed(testdir) == [] def test_lastfailed_no_failures_behavior_all_passed(self, testdir): testdir.makepyfile( """ def test_1(): assert True def test_2(): assert True """ ) result = testdir.runpytest() result.stdout.fnmatch_lines(["*2 passed*"]) result = testdir.runpytest("--lf") result.stdout.fnmatch_lines(["*2 passed*"]) result = testdir.runpytest("--lf", "--lfnf", "all") result.stdout.fnmatch_lines(["*2 passed*"]) result = testdir.runpytest("--lf", "--lfnf", "none") result.stdout.fnmatch_lines(["*2 desel*"]) def test_lastfailed_no_failures_behavior_empty_cache(self, testdir): testdir.makepyfile( """ def test_1(): assert True def test_2(): assert False """ ) result = testdir.runpytest("--lf", "--cache-clear") result.stdout.fnmatch_lines(["*1 failed*1 passed*"]) result = testdir.runpytest("--lf", "--cache-clear", "--lfnf", "all") result.stdout.fnmatch_lines(["*1 failed*1 passed*"]) result = testdir.runpytest("--lf", "--cache-clear", "--lfnf", "none") result.stdout.fnmatch_lines(["*2 desel*"]) class TestNewFirst(object): def test_newfirst_usecase(self, testdir): testdir.makepyfile( **{ "test_1/test_1.py": """ def test_1(): assert 1 def test_2(): assert 1 def test_3(): assert 1 """, "test_2/test_2.py": """ def test_1(): assert 1 def test_2(): assert 1 def test_3(): assert 1 """, } ) testdir.tmpdir.join("test_1/test_1.py").setmtime(1) result = testdir.runpytest("-v") result.stdout.fnmatch_lines( [ "*test_1/test_1.py::test_1 PASSED*", "*test_1/test_1.py::test_2 PASSED*", "*test_1/test_1.py::test_3 PASSED*", "*test_2/test_2.py::test_1 PASSED*", "*test_2/test_2.py::test_2 PASSED*", "*test_2/test_2.py::test_3 PASSED*", ] ) result = testdir.runpytest("-v", "--nf") result.stdout.fnmatch_lines( [ "*test_2/test_2.py::test_1 PASSED*", "*test_2/test_2.py::test_2 PASSED*", "*test_2/test_2.py::test_3 PASSED*", "*test_1/test_1.py::test_1 PASSED*", "*test_1/test_1.py::test_2 PASSED*", "*test_1/test_1.py::test_3 PASSED*", ] ) testdir.tmpdir.join("test_1/test_1.py").write( "def test_1(): assert 1\n" "def test_2(): assert 1\n" "def test_3(): assert 1\n" "def test_4(): assert 1\n" ) testdir.tmpdir.join("test_1/test_1.py").setmtime(1) result = testdir.runpytest("-v", "--nf") result.stdout.fnmatch_lines( [ "*test_1/test_1.py::test_4 PASSED*", "*test_2/test_2.py::test_1 PASSED*", "*test_2/test_2.py::test_2 PASSED*", "*test_2/test_2.py::test_3 PASSED*", "*test_1/test_1.py::test_1 PASSED*", "*test_1/test_1.py::test_2 PASSED*", "*test_1/test_1.py::test_3 PASSED*", ] ) def test_newfirst_parametrize(self, testdir): testdir.makepyfile( **{ "test_1/test_1.py": """ import pytest @pytest.mark.parametrize('num', [1, 2]) def test_1(num): assert num """, "test_2/test_2.py": """ import pytest @pytest.mark.parametrize('num', [1, 2]) def test_1(num): assert num """, } ) testdir.tmpdir.join("test_1/test_1.py").setmtime(1) result = testdir.runpytest("-v") result.stdout.fnmatch_lines( [ "*test_1/test_1.py::test_1[1*", "*test_1/test_1.py::test_1[2*", "*test_2/test_2.py::test_1[1*", "*test_2/test_2.py::test_1[2*", ] ) result = testdir.runpytest("-v", "--nf") result.stdout.fnmatch_lines( [ "*test_2/test_2.py::test_1[1*", "*test_2/test_2.py::test_1[2*", "*test_1/test_1.py::test_1[1*", "*test_1/test_1.py::test_1[2*", ] ) testdir.tmpdir.join("test_1/test_1.py").write( "import pytest\n" "@pytest.mark.parametrize('num', [1, 2, 3])\n" "def test_1(num): assert num\n" ) testdir.tmpdir.join("test_1/test_1.py").setmtime(1) result = testdir.runpytest("-v", "--nf") result.stdout.fnmatch_lines( [ "*test_1/test_1.py::test_1[3*", "*test_2/test_2.py::test_1[1*", "*test_2/test_2.py::test_1[2*", "*test_1/test_1.py::test_1[1*", "*test_1/test_1.py::test_1[2*", ] ) class TestReadme(object): def check_readme(self, testdir): config = testdir.parseconfigure() readme = config.cache._cachedir.joinpath("README.md") return readme.is_file() def test_readme_passed(self, testdir): testdir.makepyfile( """ def test_always_passes(): assert 1 """ ) testdir.runpytest() assert self.check_readme(testdir) is True def test_readme_failed(self, testdir): testdir.makepyfile( """ def test_always_passes(): assert 0 """ ) testdir.runpytest() assert self.check_readme(testdir) is True