import sys import textwrap import pytest from _pytest.pytester import Pytester from _pytest.runner import runtestprotocol from _pytest.skipping import evaluate_skip_marks from _pytest.skipping import evaluate_xfail_marks from _pytest.skipping import pytest_runtest_setup class TestEvaluation: def test_no_marker(self, pytester: Pytester) -> None: item = pytester.getitem("def test_func(): pass") skipped = evaluate_skip_marks(item) assert not skipped def test_marked_xfail_no_args(self, pytester: Pytester) -> None: item = pytester.getitem( """ import pytest @pytest.mark.xfail def test_func(): pass """ ) xfailed = evaluate_xfail_marks(item) assert xfailed assert xfailed.reason == "" assert xfailed.run def test_marked_skipif_no_args(self, pytester: Pytester) -> None: item = pytester.getitem( """ import pytest @pytest.mark.skipif def test_func(): pass """ ) skipped = evaluate_skip_marks(item) assert skipped assert skipped.reason == "" def test_marked_one_arg(self, pytester: Pytester) -> None: item = pytester.getitem( """ import pytest @pytest.mark.skipif("hasattr(os, 'sep')") def test_func(): pass """ ) skipped = evaluate_skip_marks(item) assert skipped assert skipped.reason == "condition: hasattr(os, 'sep')" def test_marked_one_arg_with_reason(self, pytester: Pytester) -> None: item = pytester.getitem( """ import pytest @pytest.mark.skipif("hasattr(os, 'sep')", attr=2, reason="hello world") def test_func(): pass """ ) skipped = evaluate_skip_marks(item) assert skipped assert skipped.reason == "hello world" def test_marked_one_arg_twice(self, pytester: Pytester) -> None: lines = [ """@pytest.mark.skipif("not hasattr(os, 'murks')")""", """@pytest.mark.skipif(condition="hasattr(os, 'murks')")""", ] for i in range(0, 2): item = pytester.getitem( """ import pytest %s %s def test_func(): pass """ % (lines[i], lines[(i + 1) % 2]) ) skipped = evaluate_skip_marks(item) assert skipped assert skipped.reason == "condition: not hasattr(os, 'murks')" def test_marked_one_arg_twice2(self, pytester: Pytester) -> None: item = pytester.getitem( """ import pytest @pytest.mark.skipif("hasattr(os, 'murks')") @pytest.mark.skipif("not hasattr(os, 'murks')") def test_func(): pass """ ) skipped = evaluate_skip_marks(item) assert skipped assert skipped.reason == "condition: not hasattr(os, 'murks')" def test_marked_skipif_with_boolean_without_reason( self, pytester: Pytester ) -> None: item = pytester.getitem( """ import pytest @pytest.mark.skipif(False) def test_func(): pass """ ) with pytest.raises(pytest.fail.Exception) as excinfo: evaluate_skip_marks(item) assert excinfo.value.msg is not None assert ( """Error evaluating 'skipif': you need to specify reason=STRING when using booleans as conditions.""" in excinfo.value.msg ) def test_marked_skipif_with_invalid_boolean(self, pytester: Pytester) -> None: item = pytester.getitem( """ import pytest class InvalidBool: def __bool__(self): raise TypeError("INVALID") @pytest.mark.skipif(InvalidBool(), reason="xxx") def test_func(): pass """ ) with pytest.raises(pytest.fail.Exception) as excinfo: evaluate_skip_marks(item) assert excinfo.value.msg is not None assert "Error evaluating 'skipif' condition as a boolean" in excinfo.value.msg assert "INVALID" in excinfo.value.msg def test_skipif_class(self, pytester: Pytester) -> None: (item,) = pytester.getitems( """ import pytest class TestClass(object): pytestmark = pytest.mark.skipif("config._hackxyz") def test_func(self): pass """ ) item.config._hackxyz = 3 # type: ignore[attr-defined] skipped = evaluate_skip_marks(item) assert skipped assert skipped.reason == "condition: config._hackxyz" def test_skipif_markeval_namespace(self, pytester: Pytester) -> None: pytester.makeconftest( """ import pytest def pytest_markeval_namespace(): return {"color": "green"} """ ) p = pytester.makepyfile( """ import pytest @pytest.mark.skipif("color == 'green'") def test_1(): assert True @pytest.mark.skipif("color == 'red'") def test_2(): assert True """ ) res = pytester.runpytest(p) assert res.ret == 0 res.stdout.fnmatch_lines(["*1 skipped*"]) res.stdout.fnmatch_lines(["*1 passed*"]) def test_skipif_markeval_namespace_multiple(self, pytester: Pytester) -> None: """Keys defined by ``pytest_markeval_namespace()`` in nested plugins override top-level ones.""" root = pytester.mkdir("root") root.joinpath("__init__.py").touch() root.joinpath("conftest.py").write_text( textwrap.dedent( """\ import pytest def pytest_markeval_namespace(): return {"arg": "root"} """ ), encoding="utf-8", ) root.joinpath("test_root.py").write_text( textwrap.dedent( """\ import pytest @pytest.mark.skipif("arg == 'root'") def test_root(): assert False """ ), encoding="utf-8", ) foo = root.joinpath("foo") foo.mkdir() foo.joinpath("__init__.py").touch() foo.joinpath("conftest.py").write_text( textwrap.dedent( """\ import pytest def pytest_markeval_namespace(): return {"arg": "foo"} """ ), encoding="utf-8", ) foo.joinpath("test_foo.py").write_text( textwrap.dedent( """\ import pytest @pytest.mark.skipif("arg == 'foo'") def test_foo(): assert False """ ), encoding="utf-8", ) bar = root.joinpath("bar") bar.mkdir() bar.joinpath("__init__.py").touch() bar.joinpath("conftest.py").write_text( textwrap.dedent( """\ import pytest def pytest_markeval_namespace(): return {"arg": "bar"} """ ), encoding="utf-8", ) bar.joinpath("test_bar.py").write_text( textwrap.dedent( """\ import pytest @pytest.mark.skipif("arg == 'bar'") def test_bar(): assert False """ ), encoding="utf-8", ) reprec = pytester.inline_run("-vs", "--capture=no") reprec.assertoutcome(skipped=3) def test_skipif_markeval_namespace_ValueError(self, pytester: Pytester) -> None: pytester.makeconftest( """ import pytest def pytest_markeval_namespace(): return True """ ) p = pytester.makepyfile( """ import pytest @pytest.mark.skipif("color == 'green'") def test_1(): assert True """ ) res = pytester.runpytest(p) assert res.ret == 1 res.stdout.fnmatch_lines( [ "*ValueError: pytest_markeval_namespace() needs to return a dict, got True*" ] ) class TestXFail: @pytest.mark.parametrize("strict", [True, False]) def test_xfail_simple(self, pytester: Pytester, strict: bool) -> None: item = pytester.getitem( """ import pytest @pytest.mark.xfail(strict=%s) def test_func(): assert 0 """ % strict ) reports = runtestprotocol(item, log=False) assert len(reports) == 3 callreport = reports[1] assert callreport.skipped assert callreport.wasxfail == "" def test_xfail_xpassed(self, pytester: Pytester) -> None: item = pytester.getitem( """ import pytest @pytest.mark.xfail(reason="this is an xfail") def test_func(): assert 1 """ ) reports = runtestprotocol(item, log=False) assert len(reports) == 3 callreport = reports[1] assert callreport.passed assert callreport.wasxfail == "this is an xfail" def test_xfail_using_platform(self, pytester: Pytester) -> None: """Verify that platform can be used with xfail statements.""" item = pytester.getitem( """ import pytest @pytest.mark.xfail("platform.platform() == platform.platform()") def test_func(): assert 0 """ ) reports = runtestprotocol(item, log=False) assert len(reports) == 3 callreport = reports[1] assert callreport.wasxfail def test_xfail_xpassed_strict(self, pytester: Pytester) -> None: item = pytester.getitem( """ import pytest @pytest.mark.xfail(strict=True, reason="nope") def test_func(): assert 1 """ ) reports = runtestprotocol(item, log=False) assert len(reports) == 3 callreport = reports[1] assert callreport.failed assert str(callreport.longrepr) == "[XPASS(strict)] nope" assert not hasattr(callreport, "wasxfail") def test_xfail_run_anyway(self, pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @pytest.mark.xfail def test_func(): assert 0 def test_func2(): pytest.xfail("hello") """ ) result = pytester.runpytest("--runxfail") result.stdout.fnmatch_lines( ["*def test_func():*", "*assert 0*", "*1 failed*1 pass*"] ) @pytest.mark.parametrize( "test_input,expected", [ ( ["-rs"], ["SKIPPED [1] test_sample.py:2: unconditional skip", "*1 skipped*"], ), ( ["-rs", "--runxfail"], ["SKIPPED [1] test_sample.py:2: unconditional skip", "*1 skipped*"], ), ], ) def test_xfail_run_with_skip_mark( self, pytester: Pytester, test_input, expected ) -> None: pytester.makepyfile( test_sample=""" import pytest @pytest.mark.skip def test_skip_location() -> None: assert 0 """ ) result = pytester.runpytest(*test_input) result.stdout.fnmatch_lines(expected) def test_xfail_evalfalse_but_fails(self, pytester: Pytester) -> None: item = pytester.getitem( """ import pytest @pytest.mark.xfail('False') def test_func(): assert 0 """ ) reports = runtestprotocol(item, log=False) callreport = reports[1] assert callreport.failed assert not hasattr(callreport, "wasxfail") assert "xfail" in callreport.keywords def test_xfail_not_report_default(self, pytester: Pytester) -> None: p = pytester.makepyfile( test_one=""" import pytest @pytest.mark.xfail def test_this(): assert 0 """ ) pytester.runpytest(p, "-v") # result.stdout.fnmatch_lines([ # "*HINT*use*-r*" # ]) def test_xfail_not_run_xfail_reporting(self, pytester: Pytester) -> None: p = pytester.makepyfile( test_one=""" import pytest @pytest.mark.xfail(run=False, reason="noway") def test_this(): assert 0 @pytest.mark.xfail("True", run=False) def test_this_true(): assert 0 @pytest.mark.xfail("False", run=False, reason="huh") def test_this_false(): assert 1 """ ) result = pytester.runpytest(p, "-rx") result.stdout.fnmatch_lines( [ "*test_one*test_this - reason: *NOTRUN* noway", "*test_one*test_this_true - reason: *NOTRUN* condition: True", "*1 passed*", ] ) def test_xfail_not_run_no_setup_run(self, pytester: Pytester) -> None: p = pytester.makepyfile( test_one=""" import pytest @pytest.mark.xfail(run=False, reason="hello") def test_this(): assert 0 def setup_module(mod): raise ValueError(42) """ ) result = pytester.runpytest(p, "-rx") result.stdout.fnmatch_lines(["*test_one*test_this*NOTRUN*hello", "*1 xfailed*"]) def test_xfail_xpass(self, pytester: Pytester) -> None: p = pytester.makepyfile( test_one=""" import pytest @pytest.mark.xfail def test_that(): assert 1 """ ) result = pytester.runpytest(p, "-rX") result.stdout.fnmatch_lines(["*XPASS*test_that*", "*1 xpassed*"]) assert result.ret == 0 def test_xfail_imperative(self, pytester: Pytester) -> None: p = pytester.makepyfile( """ import pytest def test_this(): pytest.xfail("hello") """ ) result = pytester.runpytest(p) result.stdout.fnmatch_lines(["*1 xfailed*"]) result = pytester.runpytest(p, "-rx") result.stdout.fnmatch_lines(["*XFAIL*test_this*reason:*hello*"]) result = pytester.runpytest(p, "--runxfail") result.stdout.fnmatch_lines(["*1 pass*"]) def test_xfail_imperative_in_setup_function(self, pytester: Pytester) -> None: p = pytester.makepyfile( """ import pytest def setup_function(function): pytest.xfail("hello") def test_this(): assert 0 """ ) result = pytester.runpytest(p) result.stdout.fnmatch_lines(["*1 xfailed*"]) result = pytester.runpytest(p, "-rx") result.stdout.fnmatch_lines(["*XFAIL*test_this*reason:*hello*"]) result = pytester.runpytest(p, "--runxfail") result.stdout.fnmatch_lines( """ *def test_this* *1 fail* """ ) def xtest_dynamic_xfail_set_during_setup(self, pytester: Pytester) -> None: p = pytester.makepyfile( """ import pytest def setup_function(function): pytest.mark.xfail(function) def test_this(): assert 0 def test_that(): assert 1 """ ) result = pytester.runpytest(p, "-rxX") result.stdout.fnmatch_lines(["*XFAIL*test_this*", "*XPASS*test_that*"]) def test_dynamic_xfail_no_run(self, pytester: Pytester) -> None: p = pytester.makepyfile( """ import pytest @pytest.fixture def arg(request): request.applymarker(pytest.mark.xfail(run=False)) def test_this(arg): assert 0 """ ) result = pytester.runpytest(p, "-rxX") result.stdout.fnmatch_lines(["*XFAIL*test_this*NOTRUN*"]) def test_dynamic_xfail_set_during_funcarg_setup(self, pytester: Pytester) -> None: p = pytester.makepyfile( """ import pytest @pytest.fixture def arg(request): request.applymarker(pytest.mark.xfail) def test_this2(arg): assert 0 """ ) result = pytester.runpytest(p) result.stdout.fnmatch_lines(["*1 xfailed*"]) def test_dynamic_xfail_set_during_runtest_failed(self, pytester: Pytester) -> None: # Issue #7486. p = pytester.makepyfile( """ import pytest def test_this(request): request.node.add_marker(pytest.mark.xfail(reason="xfail")) assert 0 """ ) result = pytester.runpytest(p) result.assert_outcomes(xfailed=1) def test_dynamic_xfail_set_during_runtest_passed_strict( self, pytester: Pytester ) -> None: # Issue #7486. p = pytester.makepyfile( """ import pytest def test_this(request): request.node.add_marker(pytest.mark.xfail(reason="xfail", strict=True)) """ ) result = pytester.runpytest(p) result.assert_outcomes(failed=1) @pytest.mark.parametrize( "expected, actual, matchline", [ ("TypeError", "TypeError", "*1 xfailed*"), ("(AttributeError, TypeError)", "TypeError", "*1 xfailed*"), ("TypeError", "IndexError", "*1 failed*"), ("(AttributeError, TypeError)", "IndexError", "*1 failed*"), ], ) def test_xfail_raises( self, expected, actual, matchline, pytester: Pytester ) -> None: p = pytester.makepyfile( """ import pytest @pytest.mark.xfail(raises=%s) def test_raises(): raise %s() """ % (expected, actual) ) result = pytester.runpytest(p) result.stdout.fnmatch_lines([matchline]) def test_strict_sanity(self, pytester: Pytester) -> None: """Sanity check for xfail(strict=True): a failing test should behave exactly like a normal xfail.""" p = pytester.makepyfile( """ import pytest @pytest.mark.xfail(reason='unsupported feature', strict=True) def test_foo(): assert 0 """ ) result = pytester.runpytest(p, "-rxX") result.stdout.fnmatch_lines(["*XFAIL*unsupported feature*"]) assert result.ret == 0 @pytest.mark.parametrize("strict", [True, False]) def test_strict_xfail(self, pytester: Pytester, strict: bool) -> None: p = pytester.makepyfile( """ import pytest @pytest.mark.xfail(reason='unsupported feature', strict=%s) def test_foo(): with open('foo_executed', 'w', encoding='utf-8'): pass # make sure test executes """ % strict ) result = pytester.runpytest(p, "-rxX") if strict: result.stdout.fnmatch_lines( ["*test_foo*", "*XPASS(strict)*unsupported feature*"] ) else: result.stdout.fnmatch_lines( [ "*test_strict_xfail*", "XPASS test_strict_xfail.py::test_foo unsupported feature", ] ) assert result.ret == (1 if strict else 0) assert pytester.path.joinpath("foo_executed").exists() @pytest.mark.parametrize("strict", [True, False]) def test_strict_xfail_condition(self, pytester: Pytester, strict: bool) -> None: p = pytester.makepyfile( """ import pytest @pytest.mark.xfail(False, reason='unsupported feature', strict=%s) def test_foo(): pass """ % strict ) result = pytester.runpytest(p, "-rxX") result.stdout.fnmatch_lines(["*1 passed*"]) assert result.ret == 0 @pytest.mark.parametrize("strict", [True, False]) def test_xfail_condition_keyword(self, pytester: Pytester, strict: bool) -> None: p = pytester.makepyfile( """ import pytest @pytest.mark.xfail(condition=False, reason='unsupported feature', strict=%s) def test_foo(): pass """ % strict ) result = pytester.runpytest(p, "-rxX") result.stdout.fnmatch_lines(["*1 passed*"]) assert result.ret == 0 @pytest.mark.parametrize("strict_val", ["true", "false"]) def test_strict_xfail_default_from_file( self, pytester: Pytester, strict_val ) -> None: pytester.makeini( """ [pytest] xfail_strict = %s """ % strict_val ) p = pytester.makepyfile( """ import pytest @pytest.mark.xfail(reason='unsupported feature') def test_foo(): pass """ ) result = pytester.runpytest(p, "-rxX") strict = strict_val == "true" result.stdout.fnmatch_lines(["*1 failed*" if strict else "*1 xpassed*"]) assert result.ret == (1 if strict else 0) def test_xfail_markeval_namespace(self, pytester: Pytester) -> None: pytester.makeconftest( """ import pytest def pytest_markeval_namespace(): return {"color": "green"} """ ) p = pytester.makepyfile( """ import pytest @pytest.mark.xfail("color == 'green'") def test_1(): assert False @pytest.mark.xfail("color == 'red'") def test_2(): assert False """ ) res = pytester.runpytest(p) assert res.ret == 1 res.stdout.fnmatch_lines(["*1 failed*"]) res.stdout.fnmatch_lines(["*1 xfailed*"]) class TestXFailwithSetupTeardown: def test_failing_setup_issue9(self, pytester: Pytester) -> None: pytester.makepyfile( """ import pytest def setup_function(func): assert 0 @pytest.mark.xfail def test_func(): pass """ ) result = pytester.runpytest() result.stdout.fnmatch_lines(["*1 xfail*"]) def test_failing_teardown_issue9(self, pytester: Pytester) -> None: pytester.makepyfile( """ import pytest def teardown_function(func): assert 0 @pytest.mark.xfail def test_func(): pass """ ) result = pytester.runpytest() result.stdout.fnmatch_lines(["*1 xfail*"]) class TestSkip: def test_skip_class(self, pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @pytest.mark.skip class TestSomething(object): def test_foo(self): pass def test_bar(self): pass def test_baz(): pass """ ) rec = pytester.inline_run() rec.assertoutcome(skipped=2, passed=1) def test_skips_on_false_string(self, pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @pytest.mark.skip('False') def test_foo(): pass """ ) rec = pytester.inline_run() rec.assertoutcome(skipped=1) def test_arg_as_reason(self, pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @pytest.mark.skip('testing stuff') def test_bar(): pass """ ) result = pytester.runpytest("-rs") result.stdout.fnmatch_lines(["*testing stuff*", "*1 skipped*"]) def test_skip_no_reason(self, pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @pytest.mark.skip def test_foo(): pass """ ) result = pytester.runpytest("-rs") result.stdout.fnmatch_lines(["*unconditional skip*", "*1 skipped*"]) def test_skip_with_reason(self, pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @pytest.mark.skip(reason="for lolz") def test_bar(): pass """ ) result = pytester.runpytest("-rs") result.stdout.fnmatch_lines(["*for lolz*", "*1 skipped*"]) def test_only_skips_marked_test(self, pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @pytest.mark.skip def test_foo(): pass @pytest.mark.skip(reason="nothing in particular") def test_bar(): pass def test_baz(): assert True """ ) result = pytester.runpytest("-rs") result.stdout.fnmatch_lines(["*nothing in particular*", "*1 passed*2 skipped*"]) def test_strict_and_skip(self, pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @pytest.mark.skip def test_hello(): pass """ ) result = pytester.runpytest("-rs", "--strict-markers") result.stdout.fnmatch_lines(["*unconditional skip*", "*1 skipped*"]) def test_wrong_skip_usage(self, pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @pytest.mark.skip(False, reason="I thought this was skipif") def test_hello(): pass """ ) result = pytester.runpytest() result.stdout.fnmatch_lines( [ "*TypeError: *__init__() got multiple values for argument 'reason'" " - maybe you meant pytest.mark.skipif?" ] ) class TestSkipif: def test_skipif_conditional(self, pytester: Pytester) -> None: item = pytester.getitem( """ import pytest @pytest.mark.skipif("hasattr(os, 'sep')") def test_func(): pass """ ) x = pytest.raises(pytest.skip.Exception, lambda: pytest_runtest_setup(item)) assert x.value.msg == "condition: hasattr(os, 'sep')" @pytest.mark.parametrize( "params", ["\"hasattr(sys, 'platform')\"", 'True, reason="invalid platform"'] ) def test_skipif_reporting(self, pytester: Pytester, params) -> None: p = pytester.makepyfile( test_foo=""" import pytest @pytest.mark.skipif(%(params)s) def test_that(): assert 0 """ % dict(params=params) ) result = pytester.runpytest(p, "-s", "-rs") result.stdout.fnmatch_lines(["*SKIP*1*test_foo.py*platform*", "*1 skipped*"]) assert result.ret == 0 def test_skipif_using_platform(self, pytester: Pytester) -> None: item = pytester.getitem( """ import pytest @pytest.mark.skipif("platform.platform() == platform.platform()") def test_func(): pass """ ) pytest.raises(pytest.skip.Exception, lambda: pytest_runtest_setup(item)) @pytest.mark.parametrize( "marker, msg1, msg2", [("skipif", "SKIP", "skipped"), ("xfail", "XPASS", "xpassed")], ) def test_skipif_reporting_multiple( self, pytester: Pytester, marker, msg1, msg2 ) -> None: pytester.makepyfile( test_foo=""" import pytest @pytest.mark.{marker}(False, reason='first_condition') @pytest.mark.{marker}(True, reason='second_condition') def test_foobar(): assert 1 """.format( marker=marker ) ) result = pytester.runpytest("-s", "-rsxX") result.stdout.fnmatch_lines( [f"*{msg1}*test_foo.py*second_condition*", f"*1 {msg2}*"] ) assert result.ret == 0 def test_skip_not_report_default(pytester: Pytester) -> None: p = pytester.makepyfile( test_one=""" import pytest def test_this(): pytest.skip("hello") """ ) result = pytester.runpytest(p, "-v") result.stdout.fnmatch_lines( [ # "*HINT*use*-r*", "*1 skipped*" ] ) def test_skipif_class(pytester: Pytester) -> None: p = pytester.makepyfile( """ import pytest class TestClass(object): pytestmark = pytest.mark.skipif("True") def test_that(self): assert 0 def test_though(self): assert 0 """ ) result = pytester.runpytest(p) result.stdout.fnmatch_lines(["*2 skipped*"]) def test_skipped_reasons_functional(pytester: Pytester) -> None: pytester.makepyfile( test_one=""" import pytest from helpers import doskip def setup_function(func): # LINE 4 doskip("setup function") def test_func(): pass class TestClass: def test_method(self): doskip("test method") @pytest.mark.skip("via_decorator") # LINE 14 def test_deco(self): assert 0 """, helpers=""" import pytest, sys def doskip(reason): assert sys._getframe().f_lineno == 3 pytest.skip(reason) # LINE 4 """, ) result = pytester.runpytest("-rs") result.stdout.fnmatch_lines_random( [ "SKIPPED [[]1[]] test_one.py:7: setup function", "SKIPPED [[]1[]] helpers.py:4: test method", "SKIPPED [[]1[]] test_one.py:14: via_decorator", ] ) assert result.ret == 0 def test_skipped_folding(pytester: Pytester) -> None: pytester.makepyfile( test_one=""" import pytest pytestmark = pytest.mark.skip("Folding") def setup_function(func): pass def test_func(): pass class TestClass(object): def test_method(self): pass """ ) result = pytester.runpytest("-rs") result.stdout.fnmatch_lines(["*SKIP*2*test_one.py: Folding"]) assert result.ret == 0 def test_reportchars(pytester: Pytester) -> None: pytester.makepyfile( """ import pytest def test_1(): assert 0 @pytest.mark.xfail def test_2(): assert 0 @pytest.mark.xfail def test_3(): pass def test_4(): pytest.skip("four") """ ) result = pytester.runpytest("-rfxXs") result.stdout.fnmatch_lines( ["FAIL*test_1*", "XFAIL*test_2*", "XPASS*test_3*", "SKIP*four*"] ) def test_reportchars_error(pytester: Pytester) -> None: pytester.makepyfile( conftest=""" def pytest_runtest_teardown(): assert 0 """, test_simple=""" def test_foo(): pass """, ) result = pytester.runpytest("-rE") result.stdout.fnmatch_lines(["ERROR*test_foo*"]) def test_reportchars_all(pytester: Pytester) -> None: pytester.makepyfile( """ import pytest def test_1(): assert 0 @pytest.mark.xfail def test_2(): assert 0 @pytest.mark.xfail def test_3(): pass def test_4(): pytest.skip("four") @pytest.fixture def fail(): assert 0 def test_5(fail): pass """ ) result = pytester.runpytest("-ra") result.stdout.fnmatch_lines( [ "SKIP*four*", "XFAIL*test_2*", "XPASS*test_3*", "ERROR*test_5*", "FAIL*test_1*", ] ) def test_reportchars_all_error(pytester: Pytester) -> None: pytester.makepyfile( conftest=""" def pytest_runtest_teardown(): assert 0 """, test_simple=""" def test_foo(): pass """, ) result = pytester.runpytest("-ra") result.stdout.fnmatch_lines(["ERROR*test_foo*"]) def test_errors_in_xfail_skip_expressions(pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @pytest.mark.skipif("asd") def test_nameerror(): pass @pytest.mark.xfail("syntax error") def test_syntax(): pass def test_func(): pass """ ) result = pytester.runpytest() markline = " ^" pypy_version_info = getattr(sys, "pypy_version_info", None) if pypy_version_info is not None and pypy_version_info < (6,): markline = markline[1:] if sys.version_info[:2] >= (3, 10): expected = [ "*ERROR*test_nameerror*", "*asd*", "", "During handling of the above exception, another exception occurred:", ] else: expected = [ "*ERROR*test_nameerror*", ] expected += [ "*evaluating*skipif*condition*", "*asd*", "*ERROR*test_syntax*", "*evaluating*xfail*condition*", " syntax error", markline, "SyntaxError: invalid syntax", "*1 pass*2 errors*", ] result.stdout.fnmatch_lines(expected) def test_xfail_skipif_with_globals(pytester: Pytester) -> None: pytester.makepyfile( """ import pytest x = 3 @pytest.mark.skipif("x == 3") def test_skip1(): pass @pytest.mark.xfail("x == 3") def test_boolean(): assert 0 """ ) result = pytester.runpytest("-rsx") result.stdout.fnmatch_lines(["*SKIP*x == 3*", "*XFAIL*test_boolean*x == 3*"]) def test_default_markers(pytester: Pytester) -> None: result = pytester.runpytest("--markers") result.stdout.fnmatch_lines( [ "*skipif(condition, ..., [*], reason=...)*skip*", "*xfail(condition, ..., [*], reason=..., run=True, raises=None, strict=xfail_strict)*expected failure*", ] ) def test_xfail_test_setup_exception(pytester: Pytester) -> None: pytester.makeconftest( """ def pytest_runtest_setup(): 0 / 0 """ ) p = pytester.makepyfile( """ import pytest @pytest.mark.xfail def test_func(): assert 0 """ ) result = pytester.runpytest(p) assert result.ret == 0 assert "xfailed" in result.stdout.str() result.stdout.no_fnmatch_line("*xpassed*") def test_imperativeskip_on_xfail_test(pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @pytest.mark.xfail def test_that_fails(): assert 0 @pytest.mark.skipif("True") def test_hello(): pass """ ) pytester.makeconftest( """ import pytest def pytest_runtest_setup(item): pytest.skip("abc") """ ) result = pytester.runpytest("-rsxX") result.stdout.fnmatch_lines_random( """ *SKIP*abc* *SKIP*condition: True* *2 skipped* """ ) class TestBooleanCondition: def test_skipif(self, pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @pytest.mark.skipif(True, reason="True123") def test_func1(): pass @pytest.mark.skipif(False, reason="True123") def test_func2(): pass """ ) result = pytester.runpytest() result.stdout.fnmatch_lines( """ *1 passed*1 skipped* """ ) def test_skipif_noreason(self, pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @pytest.mark.skipif(True) def test_func(): pass """ ) result = pytester.runpytest("-rs") result.stdout.fnmatch_lines( """ *1 error* """ ) def test_xfail(self, pytester: Pytester) -> None: pytester.makepyfile( """ import pytest @pytest.mark.xfail(True, reason="True123") def test_func(): assert 0 """ ) result = pytester.runpytest("-rxs") result.stdout.fnmatch_lines( """ *XFAIL*True123* *1 xfail* """ ) def test_xfail_item(pytester: Pytester) -> None: # Ensure pytest.xfail works with non-Python Item pytester.makeconftest( """ import pytest class MyItem(pytest.Item): nodeid = 'foo' def runtest(self): pytest.xfail("Expected Failure") def pytest_collect_file(file_path, parent): return MyItem.from_parent(name="foo", parent=parent) """ ) result = pytester.inline_run() passed, skipped, failed = result.listoutcomes() assert not failed xfailed = [r for r in skipped if hasattr(r, "wasxfail")] assert xfailed def test_module_level_skip_error(pytester: Pytester) -> None: """Verify that using pytest.skip at module level causes a collection error.""" pytester.makepyfile( """ import pytest pytest.skip("skip_module_level") def test_func(): assert True """ ) result = pytester.runpytest() result.stdout.fnmatch_lines( ["*Using pytest.skip outside of a test will skip the entire module*"] ) def test_module_level_skip_with_allow_module_level(pytester: Pytester) -> None: """Verify that using pytest.skip(allow_module_level=True) is allowed.""" pytester.makepyfile( """ import pytest pytest.skip("skip_module_level", allow_module_level=True) def test_func(): assert 0 """ ) result = pytester.runpytest("-rxs") result.stdout.fnmatch_lines(["*SKIP*skip_module_level"]) def test_invalid_skip_keyword_parameter(pytester: Pytester) -> None: """Verify that using pytest.skip() with unknown parameter raises an error.""" pytester.makepyfile( """ import pytest pytest.skip("skip_module_level", unknown=1) def test_func(): assert 0 """ ) result = pytester.runpytest() result.stdout.fnmatch_lines(["*TypeError:*['unknown']*"]) def test_mark_xfail_item(pytester: Pytester) -> None: # Ensure pytest.mark.xfail works with non-Python Item pytester.makeconftest( """ import pytest class MyItem(pytest.Item): nodeid = 'foo' def setup(self): marker = pytest.mark.xfail("1 == 2", reason="Expected failure - false") self.add_marker(marker) marker = pytest.mark.xfail(True, reason="Expected failure - true") self.add_marker(marker) def runtest(self): assert False def pytest_collect_file(file_path, parent): return MyItem.from_parent(name="foo", parent=parent) """ ) result = pytester.inline_run() passed, skipped, failed = result.listoutcomes() assert not failed xfailed = [r for r in skipped if hasattr(r, "wasxfail")] assert xfailed def test_summary_list_after_errors(pytester: Pytester) -> None: """Ensure the list of errors/fails/xfails/skips appears after tracebacks in terminal reporting.""" pytester.makepyfile( """ import pytest def test_fail(): assert 0 """ ) result = pytester.runpytest("-ra") result.stdout.fnmatch_lines( [ "=* FAILURES *=", "*= short test summary info =*", "FAILED test_summary_list_after_errors.py::test_fail - assert 0", ] ) def test_importorskip() -> None: with pytest.raises( pytest.skip.Exception, match="^could not import 'doesnotexist': No module named .*", ): pytest.importorskip("doesnotexist") def test_relpath_rootdir(pytester: Pytester) -> None: pytester.makepyfile( **{ "tests/test_1.py": """ import pytest @pytest.mark.skip() def test_pass(): pass """, } ) result = pytester.runpytest("-rs", "tests/test_1.py", "--rootdir=tests") result.stdout.fnmatch_lines( ["SKIPPED [[]1[]] tests/test_1.py:2: unconditional skip"] ) def test_skip_from_fixture(pytester: Pytester) -> None: pytester.makepyfile( **{ "tests/test_1.py": """ import pytest def test_pass(arg): pass @pytest.fixture def arg(): condition = True if condition: pytest.skip("Fixture conditional skip") """, } ) result = pytester.runpytest("-rs", "tests/test_1.py", "--rootdir=tests") result.stdout.fnmatch_lines( ["SKIPPED [[]1[]] tests/test_1.py:2: Fixture conditional skip"] ) def test_skip_using_reason_works_ok(pytester: Pytester) -> None: p = pytester.makepyfile( """ import pytest def test_skipping_reason(): pytest.skip(reason="skippedreason") """ ) result = pytester.runpytest(p) result.stdout.no_fnmatch_line("*PytestDeprecationWarning*") result.assert_outcomes(skipped=1) def test_fail_using_reason_works_ok(pytester: Pytester) -> None: p = pytester.makepyfile( """ import pytest def test_failing_reason(): pytest.fail(reason="failedreason") """ ) result = pytester.runpytest(p) result.stdout.no_fnmatch_line("*PytestDeprecationWarning*") result.assert_outcomes(failed=1) def test_fail_fails_with_msg_and_reason(pytester: Pytester) -> None: p = pytester.makepyfile( """ import pytest def test_fail_both_arguments(): pytest.fail(reason="foo", msg="bar") """ ) result = pytester.runpytest(p) result.stdout.fnmatch_lines( "*UsageError: Passing both ``reason`` and ``msg`` to pytest.fail(...) is not permitted.*" ) result.assert_outcomes(failed=1) def test_skip_fails_with_msg_and_reason(pytester: Pytester) -> None: p = pytester.makepyfile( """ import pytest def test_skip_both_arguments(): pytest.skip(reason="foo", msg="bar") """ ) result = pytester.runpytest(p) result.stdout.fnmatch_lines( "*UsageError: Passing both ``reason`` and ``msg`` to pytest.skip(...) is not permitted.*" ) result.assert_outcomes(failed=1) def test_exit_with_msg_and_reason_fails(pytester: Pytester) -> None: p = pytester.makepyfile( """ import pytest def test_exit_both_arguments(): pytest.exit(reason="foo", msg="bar") """ ) result = pytester.runpytest(p) result.stdout.fnmatch_lines( "*UsageError: cannot pass reason and msg to exit(), `msg` is deprecated, use `reason`.*" ) result.assert_outcomes(failed=1) def test_exit_with_reason_works_ok(pytester: Pytester) -> None: p = pytester.makepyfile( """ import pytest def test_exit_reason_only(): pytest.exit(reason="foo") """ ) result = pytester.runpytest(p) result.stdout.fnmatch_lines("*_pytest.outcomes.Exit: foo*")