test_ok2/testing/test_pytester.py

810 lines
25 KiB
Python

import os
import subprocess
import sys
import time
from typing import List
import py.path
import _pytest.pytester as pytester
import pytest
from _pytest.config import ExitCode
from _pytest.config import PytestPluginManager
from _pytest.pytester import CwdSnapshot
from _pytest.pytester import HookRecorder
from _pytest.pytester import LineMatcher
from _pytest.pytester import SysModulesSnapshot
from _pytest.pytester import SysPathsSnapshot
from _pytest.pytester import Testdir
def test_make_hook_recorder(testdir) -> None:
item = testdir.getitem("def test_func(): pass")
recorder = testdir.make_hook_recorder(item.config.pluginmanager)
assert not recorder.getfailures()
# (The silly condition is to fool mypy that the code below this is reachable)
if 1 + 1 == 2:
pytest.xfail("internal reportrecorder tests need refactoring")
class rep:
excinfo = None
passed = False
failed = True
skipped = False
when = "call"
recorder.hook.pytest_runtest_logreport(report=rep)
failures = recorder.getfailures()
assert failures == [rep]
failures = recorder.getfailures()
assert failures == [rep]
class rep2:
excinfo = None
passed = False
failed = False
skipped = True
when = "call"
rep2.passed = False
rep2.skipped = True
recorder.hook.pytest_runtest_logreport(report=rep2)
modcol = testdir.getmodulecol("")
rep3 = modcol.config.hook.pytest_make_collect_report(collector=modcol)
rep3.passed = False
rep3.failed = True
rep3.skipped = False
recorder.hook.pytest_collectreport(report=rep3)
passed, skipped, failed = recorder.listoutcomes()
assert not passed and skipped and failed
numpassed, numskipped, numfailed = recorder.countoutcomes()
assert numpassed == 0
assert numskipped == 1
assert numfailed == 1
assert len(recorder.getfailedcollections()) == 1
recorder.unregister()
recorder.clear()
recorder.hook.pytest_runtest_logreport(report=rep3)
pytest.raises(ValueError, recorder.getfailures)
def test_parseconfig(testdir) -> None:
config1 = testdir.parseconfig()
config2 = testdir.parseconfig()
assert config2 is not config1
def test_testdir_runs_with_plugin(testdir) -> None:
testdir.makepyfile(
"""
pytest_plugins = "pytester"
def test_hello(testdir):
assert 1
"""
)
result = testdir.runpytest()
result.assert_outcomes(passed=1)
def test_testdir_with_doctest(testdir):
"""Check that testdir can be used within doctests.
It used to use `request.function`, which is `None` with doctests."""
testdir.makepyfile(
**{
"sub/t-doctest.py": """
'''
>>> import os
>>> testdir = getfixture("testdir")
>>> str(testdir.makepyfile("content")).replace(os.sep, '/')
'.../basetemp/sub.t-doctest0/sub.py'
'''
""",
"sub/__init__.py": "",
}
)
result = testdir.runpytest(
"-p", "pytester", "--doctest-modules", "sub/t-doctest.py"
)
assert result.ret == 0
def test_runresult_assertion_on_xfail(testdir) -> None:
testdir.makepyfile(
"""
import pytest
pytest_plugins = "pytester"
@pytest.mark.xfail
def test_potato():
assert False
"""
)
result = testdir.runpytest()
result.assert_outcomes(xfailed=1)
assert result.ret == 0
def test_runresult_assertion_on_xpassed(testdir) -> None:
testdir.makepyfile(
"""
import pytest
pytest_plugins = "pytester"
@pytest.mark.xfail
def test_potato():
assert True
"""
)
result = testdir.runpytest()
result.assert_outcomes(xpassed=1)
assert result.ret == 0
def test_xpassed_with_strict_is_considered_a_failure(testdir) -> None:
testdir.makepyfile(
"""
import pytest
pytest_plugins = "pytester"
@pytest.mark.xfail(strict=True)
def test_potato():
assert True
"""
)
result = testdir.runpytest()
result.assert_outcomes(failed=1)
assert result.ret != 0
def make_holder():
class apiclass:
def pytest_xyz(self, arg):
"""X"""
def pytest_xyz_noarg(self):
"""X"""
apimod = type(os)("api")
def pytest_xyz(arg):
"""X"""
def pytest_xyz_noarg():
"""X"""
apimod.pytest_xyz = pytest_xyz # type: ignore
apimod.pytest_xyz_noarg = pytest_xyz_noarg # type: ignore
return apiclass, apimod
@pytest.mark.parametrize("holder", make_holder())
def test_hookrecorder_basic(holder) -> None:
pm = PytestPluginManager()
pm.add_hookspecs(holder)
rec = HookRecorder(pm)
pm.hook.pytest_xyz(arg=123)
call = rec.popcall("pytest_xyz")
assert call.arg == 123
assert call._name == "pytest_xyz"
pytest.raises(pytest.fail.Exception, rec.popcall, "abc")
pm.hook.pytest_xyz_noarg()
call = rec.popcall("pytest_xyz_noarg")
assert call._name == "pytest_xyz_noarg"
def test_makepyfile_unicode(testdir) -> None:
testdir.makepyfile(chr(0xFFFD))
def test_makepyfile_utf8(testdir) -> None:
"""Ensure makepyfile accepts utf-8 bytes as input (#2738)"""
utf8_contents = """
def setup_function(function):
mixed_encoding = 'São Paulo'
""".encode()
p = testdir.makepyfile(utf8_contents)
assert "mixed_encoding = 'São Paulo'".encode() in p.read("rb")
class TestInlineRunModulesCleanup:
def test_inline_run_test_module_not_cleaned_up(self, testdir) -> None:
test_mod = testdir.makepyfile("def test_foo(): assert True")
result = testdir.inline_run(str(test_mod))
assert result.ret == ExitCode.OK
# rewrite module, now test should fail if module was re-imported
test_mod.write("def test_foo(): assert False")
result2 = testdir.inline_run(str(test_mod))
assert result2.ret == ExitCode.TESTS_FAILED
def spy_factory(self):
class SysModulesSnapshotSpy:
instances = [] # type: List[SysModulesSnapshotSpy]
def __init__(self, preserve=None) -> None:
SysModulesSnapshotSpy.instances.append(self)
self._spy_restore_count = 0
self._spy_preserve = preserve
self.__snapshot = SysModulesSnapshot(preserve=preserve)
def restore(self):
self._spy_restore_count += 1
return self.__snapshot.restore()
return SysModulesSnapshotSpy
def test_inline_run_taking_and_restoring_a_sys_modules_snapshot(
self, testdir, monkeypatch
) -> None:
spy_factory = self.spy_factory()
monkeypatch.setattr(pytester, "SysModulesSnapshot", spy_factory)
testdir.syspathinsert()
original = dict(sys.modules)
testdir.makepyfile(import1="# you son of a silly person")
testdir.makepyfile(import2="# my hovercraft is full of eels")
test_mod = testdir.makepyfile(
"""
import import1
def test_foo(): import import2"""
)
testdir.inline_run(str(test_mod))
assert len(spy_factory.instances) == 1
spy = spy_factory.instances[0]
assert spy._spy_restore_count == 1
assert sys.modules == original
assert all(sys.modules[x] is original[x] for x in sys.modules)
def test_inline_run_sys_modules_snapshot_restore_preserving_modules(
self, testdir, monkeypatch
) -> None:
spy_factory = self.spy_factory()
monkeypatch.setattr(pytester, "SysModulesSnapshot", spy_factory)
test_mod = testdir.makepyfile("def test_foo(): pass")
testdir.inline_run(str(test_mod))
spy = spy_factory.instances[0]
assert not spy._spy_preserve("black_knight")
assert spy._spy_preserve("zope")
assert spy._spy_preserve("zope.interface")
assert spy._spy_preserve("zopelicious")
def test_external_test_module_imports_not_cleaned_up(self, testdir) -> None:
testdir.syspathinsert()
testdir.makepyfile(imported="data = 'you son of a silly person'")
import imported
test_mod = testdir.makepyfile(
"""
def test_foo():
import imported
imported.data = 42"""
)
testdir.inline_run(str(test_mod))
assert imported.data == 42
def test_assert_outcomes_after_pytest_error(testdir) -> None:
testdir.makepyfile("def test_foo(): assert True")
result = testdir.runpytest("--unexpected-argument")
with pytest.raises(ValueError, match="Pytest terminal summary report not found"):
result.assert_outcomes(passed=0)
def test_cwd_snapshot(testdir: Testdir) -> None:
tmpdir = testdir.tmpdir
foo = tmpdir.ensure("foo", dir=1)
bar = tmpdir.ensure("bar", dir=1)
foo.chdir()
snapshot = CwdSnapshot()
bar.chdir()
assert py.path.local() == bar
snapshot.restore()
assert py.path.local() == foo
class TestSysModulesSnapshot:
key = "my-test-module"
def test_remove_added(self) -> None:
original = dict(sys.modules)
assert self.key not in sys.modules
snapshot = SysModulesSnapshot()
sys.modules[self.key] = "something" # type: ignore
assert self.key in sys.modules
snapshot.restore()
assert sys.modules == original
def test_add_removed(self, monkeypatch) -> None:
assert self.key not in sys.modules
monkeypatch.setitem(sys.modules, self.key, "something")
assert self.key in sys.modules
original = dict(sys.modules)
snapshot = SysModulesSnapshot()
del sys.modules[self.key]
assert self.key not in sys.modules
snapshot.restore()
assert sys.modules == original
def test_restore_reloaded(self, monkeypatch) -> None:
assert self.key not in sys.modules
monkeypatch.setitem(sys.modules, self.key, "something")
assert self.key in sys.modules
original = dict(sys.modules)
snapshot = SysModulesSnapshot()
sys.modules[self.key] = "something else" # type: ignore
snapshot.restore()
assert sys.modules == original
def test_preserve_modules(self, monkeypatch) -> None:
key = [self.key + str(i) for i in range(3)]
assert not any(k in sys.modules for k in key)
for i, k in enumerate(key):
monkeypatch.setitem(sys.modules, k, "something" + str(i))
original = dict(sys.modules)
def preserve(name):
return name in (key[0], key[1], "some-other-key")
snapshot = SysModulesSnapshot(preserve=preserve)
sys.modules[key[0]] = original[key[0]] = "something else0" # type: ignore
sys.modules[key[1]] = original[key[1]] = "something else1" # type: ignore
sys.modules[key[2]] = "something else2" # type: ignore
snapshot.restore()
assert sys.modules == original
def test_preserve_container(self, monkeypatch) -> None:
original = dict(sys.modules)
assert self.key not in original
replacement = dict(sys.modules)
replacement[self.key] = "life of brian" # type: ignore
snapshot = SysModulesSnapshot()
monkeypatch.setattr(sys, "modules", replacement)
snapshot.restore()
assert sys.modules is replacement
assert sys.modules == original
@pytest.mark.parametrize("path_type", ("path", "meta_path"))
class TestSysPathsSnapshot:
other_path = {"path": "meta_path", "meta_path": "path"}
@staticmethod
def path(n: int) -> str:
return "my-dirty-little-secret-" + str(n)
def test_restore(self, monkeypatch, path_type) -> None:
other_path_type = self.other_path[path_type]
for i in range(10):
assert self.path(i) not in getattr(sys, path_type)
sys_path = [self.path(i) for i in range(6)]
monkeypatch.setattr(sys, path_type, sys_path)
original = list(sys_path)
original_other = list(getattr(sys, other_path_type))
snapshot = SysPathsSnapshot()
transformation = {"source": (0, 1, 2, 3, 4, 5), "target": (6, 2, 9, 7, 5, 8)}
assert sys_path == [self.path(x) for x in transformation["source"]]
sys_path[1] = self.path(6)
sys_path[3] = self.path(7)
sys_path.append(self.path(8))
del sys_path[4]
sys_path[3:3] = [self.path(9)]
del sys_path[0]
assert sys_path == [self.path(x) for x in transformation["target"]]
snapshot.restore()
assert getattr(sys, path_type) is sys_path
assert getattr(sys, path_type) == original
assert getattr(sys, other_path_type) == original_other
def test_preserve_container(self, monkeypatch, path_type) -> None:
other_path_type = self.other_path[path_type]
original_data = list(getattr(sys, path_type))
original_other = getattr(sys, other_path_type)
original_other_data = list(original_other)
new = [] # type: List[object]
snapshot = SysPathsSnapshot()
monkeypatch.setattr(sys, path_type, new)
snapshot.restore()
assert getattr(sys, path_type) is new
assert getattr(sys, path_type) == original_data
assert getattr(sys, other_path_type) is original_other
assert getattr(sys, other_path_type) == original_other_data
def test_testdir_subprocess(testdir) -> None:
testfile = testdir.makepyfile("def test_one(): pass")
assert testdir.runpytest_subprocess(testfile).ret == 0
def test_testdir_subprocess_via_runpytest_arg(testdir) -> None:
testfile = testdir.makepyfile(
"""
def test_testdir_subprocess(testdir):
import os
testfile = testdir.makepyfile(
\"""
import os
def test_one():
assert {} != os.getpid()
\""".format(os.getpid())
)
assert testdir.runpytest(testfile).ret == 0
"""
)
result = testdir.runpytest_subprocess(
"-p", "pytester", "--runpytest", "subprocess", testfile
)
assert result.ret == 0
def test_unicode_args(testdir) -> None:
result = testdir.runpytest("-k", "אבג")
assert result.ret == ExitCode.NO_TESTS_COLLECTED
def test_testdir_run_no_timeout(testdir) -> None:
testfile = testdir.makepyfile("def test_no_timeout(): pass")
assert testdir.runpytest_subprocess(testfile).ret == ExitCode.OK
def test_testdir_run_with_timeout(testdir) -> None:
testfile = testdir.makepyfile("def test_no_timeout(): pass")
timeout = 120
start = time.time()
result = testdir.runpytest_subprocess(testfile, timeout=timeout)
end = time.time()
duration = end - start
assert result.ret == ExitCode.OK
assert duration < timeout
def test_testdir_run_timeout_expires(testdir) -> None:
testfile = testdir.makepyfile(
"""
import time
def test_timeout():
time.sleep(10)"""
)
with pytest.raises(testdir.TimeoutExpired):
testdir.runpytest_subprocess(testfile, timeout=1)
def test_linematcher_with_nonlist() -> None:
"""Test LineMatcher with regard to passing in a set (accidentally)."""
from _pytest._code.source import Source
lm = LineMatcher([])
with pytest.raises(TypeError, match="invalid type for lines2: set"):
lm.fnmatch_lines(set()) # type: ignore[arg-type]
with pytest.raises(TypeError, match="invalid type for lines2: dict"):
lm.fnmatch_lines({}) # type: ignore[arg-type]
with pytest.raises(TypeError, match="invalid type for lines2: set"):
lm.re_match_lines(set()) # type: ignore[arg-type]
with pytest.raises(TypeError, match="invalid type for lines2: dict"):
lm.re_match_lines({}) # type: ignore[arg-type]
with pytest.raises(TypeError, match="invalid type for lines2: Source"):
lm.fnmatch_lines(Source()) # type: ignore[arg-type]
lm.fnmatch_lines([])
lm.fnmatch_lines(())
lm.fnmatch_lines("")
assert lm._getlines({}) == {} # type: ignore[arg-type,comparison-overlap]
assert lm._getlines(set()) == set() # type: ignore[arg-type,comparison-overlap]
assert lm._getlines(Source()) == []
assert lm._getlines(Source("pass\npass")) == ["pass", "pass"]
def test_linematcher_match_failure() -> None:
lm = LineMatcher(["foo", "foo", "bar"])
with pytest.raises(pytest.fail.Exception) as e:
lm.fnmatch_lines(["foo", "f*", "baz"])
assert e.value.msg is not None
assert e.value.msg.splitlines() == [
"exact match: 'foo'",
"fnmatch: 'f*'",
" with: 'foo'",
"nomatch: 'baz'",
" and: 'bar'",
"remains unmatched: 'baz'",
]
lm = LineMatcher(["foo", "foo", "bar"])
with pytest.raises(pytest.fail.Exception) as e:
lm.re_match_lines(["foo", "^f.*", "baz"])
assert e.value.msg is not None
assert e.value.msg.splitlines() == [
"exact match: 'foo'",
"re.match: '^f.*'",
" with: 'foo'",
" nomatch: 'baz'",
" and: 'bar'",
"remains unmatched: 'baz'",
]
def test_linematcher_consecutive():
lm = LineMatcher(["1", "", "2"])
with pytest.raises(pytest.fail.Exception) as excinfo:
lm.fnmatch_lines(["1", "2"], consecutive=True)
assert str(excinfo.value).splitlines() == [
"exact match: '1'",
"no consecutive match: '2'",
" with: ''",
]
lm.re_match_lines(["1", r"\d?", "2"], consecutive=True)
with pytest.raises(pytest.fail.Exception) as excinfo:
lm.re_match_lines(["1", r"\d", "2"], consecutive=True)
assert str(excinfo.value).splitlines() == [
"exact match: '1'",
r"no consecutive match: '\\d'",
" with: ''",
]
@pytest.mark.parametrize("function", ["no_fnmatch_line", "no_re_match_line"])
def test_linematcher_no_matching(function) -> None:
if function == "no_fnmatch_line":
good_pattern = "*.py OK*"
bad_pattern = "*X.py OK*"
else:
assert function == "no_re_match_line"
good_pattern = r".*py OK"
bad_pattern = r".*Xpy OK"
lm = LineMatcher(
[
"cachedir: .pytest_cache",
"collecting ... collected 1 item",
"",
"show_fixtures_per_test.py OK",
"=== elapsed 1s ===",
]
)
# check the function twice to ensure we don't accumulate the internal buffer
for i in range(2):
with pytest.raises(pytest.fail.Exception) as e:
func = getattr(lm, function)
func(good_pattern)
obtained = str(e.value).splitlines()
if function == "no_fnmatch_line":
assert obtained == [
"nomatch: '{}'".format(good_pattern),
" and: 'cachedir: .pytest_cache'",
" and: 'collecting ... collected 1 item'",
" and: ''",
"fnmatch: '{}'".format(good_pattern),
" with: 'show_fixtures_per_test.py OK'",
]
else:
assert obtained == [
" nomatch: '{}'".format(good_pattern),
" and: 'cachedir: .pytest_cache'",
" and: 'collecting ... collected 1 item'",
" and: ''",
"re.match: '{}'".format(good_pattern),
" with: 'show_fixtures_per_test.py OK'",
]
func = getattr(lm, function)
func(bad_pattern) # bad pattern does not match any line: passes
def test_linematcher_no_matching_after_match() -> None:
lm = LineMatcher(["1", "2", "3"])
lm.fnmatch_lines(["1", "3"])
with pytest.raises(pytest.fail.Exception) as e:
lm.no_fnmatch_line("*")
assert str(e.value).splitlines() == ["fnmatch: '*'", " with: '1'"]
def test_pytester_addopts_before_testdir(request, monkeypatch) -> None:
orig = os.environ.get("PYTEST_ADDOPTS", None)
monkeypatch.setenv("PYTEST_ADDOPTS", "--orig-unused")
testdir = request.getfixturevalue("testdir")
assert "PYTEST_ADDOPTS" not in os.environ
testdir.finalize()
assert os.environ.get("PYTEST_ADDOPTS") == "--orig-unused"
monkeypatch.undo()
assert os.environ.get("PYTEST_ADDOPTS") == orig
def test_run_stdin(testdir) -> None:
with pytest.raises(testdir.TimeoutExpired):
testdir.run(
sys.executable,
"-c",
"import sys, time; time.sleep(1); print(sys.stdin.read())",
stdin=subprocess.PIPE,
timeout=0.1,
)
with pytest.raises(testdir.TimeoutExpired):
result = testdir.run(
sys.executable,
"-c",
"import sys, time; time.sleep(1); print(sys.stdin.read())",
stdin=b"input\n2ndline",
timeout=0.1,
)
result = testdir.run(
sys.executable,
"-c",
"import sys; print(sys.stdin.read())",
stdin=b"input\n2ndline",
)
assert result.stdout.lines == ["input", "2ndline"]
assert result.stderr.str() == ""
assert result.ret == 0
def test_popen_stdin_pipe(testdir) -> None:
proc = testdir.popen(
[sys.executable, "-c", "import sys; print(sys.stdin.read())"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
)
stdin = b"input\n2ndline"
stdout, stderr = proc.communicate(input=stdin)
assert stdout.decode("utf8").splitlines() == ["input", "2ndline"]
assert stderr == b""
assert proc.returncode == 0
def test_popen_stdin_bytes(testdir) -> None:
proc = testdir.popen(
[sys.executable, "-c", "import sys; print(sys.stdin.read())"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=b"input\n2ndline",
)
stdout, stderr = proc.communicate()
assert stdout.decode("utf8").splitlines() == ["input", "2ndline"]
assert stderr == b""
assert proc.returncode == 0
def test_popen_default_stdin_stderr_and_stdin_None(testdir) -> None:
# stdout, stderr default to pipes,
# stdin can be None to not close the pipe, avoiding
# "ValueError: flush of closed file" with `communicate()`.
#
# Wraps the test to make it not hang when run with "-s".
p1 = testdir.makepyfile(
'''
import sys
def test_inner(testdir):
p1 = testdir.makepyfile(
"""
import sys
print(sys.stdin.read()) # empty
print('stdout')
sys.stderr.write('stderr')
"""
)
proc = testdir.popen([sys.executable, str(p1)], stdin=None)
stdout, stderr = proc.communicate(b"ignored")
assert stdout.splitlines() == [b"", b"stdout"]
assert stderr.splitlines() == [b"stderr"]
assert proc.returncode == 0
'''
)
result = testdir.runpytest("-p", "pytester", str(p1))
assert result.ret == 0
def test_spawn_uses_tmphome(testdir) -> None:
tmphome = str(testdir.tmpdir)
assert os.environ.get("HOME") == tmphome
testdir.monkeypatch.setenv("CUSTOMENV", "42")
p1 = testdir.makepyfile(
"""
import os
def test():
assert os.environ["HOME"] == {tmphome!r}
assert os.environ["CUSTOMENV"] == "42"
""".format(
tmphome=tmphome
)
)
child = testdir.spawn_pytest(str(p1))
out = child.read()
assert child.wait() == 0, out.decode("utf8")
def test_run_result_repr() -> None:
outlines = ["some", "normal", "output"]
errlines = ["some", "nasty", "errors", "happened"]
# known exit code
r = pytester.RunResult(1, outlines, errlines, duration=0.5)
assert (
repr(r) == "<RunResult ret=ExitCode.TESTS_FAILED len(stdout.lines)=3"
" len(stderr.lines)=4 duration=0.50s>"
)
# unknown exit code: just the number
r = pytester.RunResult(99, outlines, errlines, duration=0.5)
assert (
repr(r) == "<RunResult ret=99 len(stdout.lines)=3"
" len(stderr.lines)=4 duration=0.50s>"
)
def test_testdir_outcomes_with_multiple_errors(testdir):
p1 = testdir.makepyfile(
"""
import pytest
@pytest.fixture
def bad_fixture():
raise Exception("bad")
def test_error1(bad_fixture):
pass
def test_error2(bad_fixture):
pass
"""
)
result = testdir.runpytest(str(p1))
result.assert_outcomes(errors=2)
assert result.parseoutcomes() == {"errors": 2}
def test_parse_summary_line_always_plural():
"""Parsing summaries always returns plural nouns (#6505)"""
lines = [
"some output 1",
"some output 2",
"======= 1 failed, 1 passed, 1 warning, 1 error in 0.13s ====",
"done.",
]
assert pytester.RunResult.parse_summary_nouns(lines) == {
"errors": 1,
"failed": 1,
"passed": 1,
"warnings": 1,
}
lines = [
"some output 1",
"some output 2",
"======= 1 failed, 1 passed, 2 warnings, 2 errors in 0.13s ====",
"done.",
]
assert pytester.RunResult.parse_summary_nouns(lines) == {
"errors": 2,
"failed": 1,
"passed": 1,
"warnings": 2,
}
def test_makefile_joins_absolute_path(testdir: Testdir) -> None:
absfile = testdir.tmpdir / "absfile"
if sys.platform == "win32":
with pytest.raises(OSError):
testdir.makepyfile(**{str(absfile): ""})
else:
p1 = testdir.makepyfile(**{str(absfile): ""})
assert str(p1) == (testdir.tmpdir / absfile) + ".py"