import pytest import sys from _pytest.skipping import MarkEvaluator, folded_skips, pytest_runtest_setup from _pytest.runner import runtestprotocol class TestEvaluator: def test_no_marker(self, testdir): item = testdir.getitem("def test_func(): pass") evalskipif = MarkEvaluator(item, 'skipif') assert not evalskipif assert not evalskipif.istrue() def test_marked_no_args(self, testdir): item = testdir.getitem(""" import pytest @pytest.mark.xyz def test_func(): pass """) ev = MarkEvaluator(item, 'xyz') assert ev assert ev.istrue() expl = ev.getexplanation() assert expl == "" assert not ev.get("run", False) def test_marked_one_arg(self, testdir): item = testdir.getitem(""" import pytest @pytest.mark.xyz("hasattr(os, 'sep')") def test_func(): pass """) ev = MarkEvaluator(item, 'xyz') assert ev assert ev.istrue() expl = ev.getexplanation() assert expl == "condition: hasattr(os, 'sep')" @pytest.mark.skipif('sys.version_info[0] >= 3') def test_marked_one_arg_unicode(self, testdir): item = testdir.getitem(""" import pytest @pytest.mark.xyz(u"hasattr(os, 'sep')") def test_func(): pass """) ev = MarkEvaluator(item, 'xyz') assert ev assert ev.istrue() expl = ev.getexplanation() assert expl == "condition: hasattr(os, 'sep')" def test_marked_one_arg_with_reason(self, testdir): item = testdir.getitem(""" import pytest @pytest.mark.xyz("hasattr(os, 'sep')", attr=2, reason="hello world") def test_func(): pass """) ev = MarkEvaluator(item, 'xyz') assert ev assert ev.istrue() expl = ev.getexplanation() assert expl == "hello world" assert ev.get("attr") == 2 def test_marked_one_arg_twice(self, testdir): lines = [ '''@pytest.mark.skipif("not hasattr(os, 'murks')")''', '''@pytest.mark.skipif("hasattr(os, 'murks')")''' ] for i in range(0, 2): item = testdir.getitem(""" import pytest %s %s def test_func(): pass """ % (lines[i], lines[(i+1) %2])) ev = MarkEvaluator(item, 'skipif') assert ev assert ev.istrue() expl = ev.getexplanation() assert expl == "condition: not hasattr(os, 'murks')" def test_marked_one_arg_twice2(self, testdir): item = testdir.getitem(""" import pytest @pytest.mark.skipif("hasattr(os, 'murks')") @pytest.mark.skipif("not hasattr(os, 'murks')") def test_func(): pass """) ev = MarkEvaluator(item, 'skipif') assert ev assert ev.istrue() expl = ev.getexplanation() assert expl == "condition: not hasattr(os, 'murks')" def test_marked_skip_with_not_string(self, testdir): item = testdir.getitem(""" import pytest @pytest.mark.skipif(False) def test_func(): pass """) ev = MarkEvaluator(item, 'skipif') exc = pytest.raises(pytest.fail.Exception, ev.istrue) assert """Failed: you need to specify reason=STRING when using booleans as conditions.""" in exc.value.msg def test_skipif_class(self, testdir): item, = testdir.getitems(""" import pytest class TestClass: pytestmark = pytest.mark.skipif("config._hackxyz") def test_func(self): pass """) item.config._hackxyz = 3 ev = MarkEvaluator(item, 'skipif') assert ev.istrue() expl = ev.getexplanation() assert expl == "condition: config._hackxyz" class TestXFail: @pytest.mark.parametrize('strict', [True, False]) def test_xfail_simple(self, testdir, strict): item = testdir.getitem(""" import pytest @pytest.mark.xfail(strict=%s) def test_func(): assert 0 """ % strict) reports = runtestprotocol(item, log=False) assert len(reports) == 3 callreport = reports[1] assert callreport.skipped assert callreport.wasxfail == "" def test_xfail_xpassed(self, testdir): item = testdir.getitem(""" import pytest @pytest.mark.xfail def test_func(): assert 1 """) reports = runtestprotocol(item, log=False) assert len(reports) == 3 callreport = reports[1] assert callreport.failed assert callreport.wasxfail == "" def test_xfail_run_anyway(self, testdir): testdir.makepyfile(""" import pytest @pytest.mark.xfail def test_func(): assert 0 def test_func2(): pytest.xfail("hello") """) result = testdir.runpytest("--runxfail") result.stdout.fnmatch_lines([ "*def test_func():*", "*assert 0*", "*1 failed*1 pass*", ]) def test_xfail_evalfalse_but_fails(self, testdir): item = testdir.getitem(""" import pytest @pytest.mark.xfail('False') def test_func(): assert 0 """) reports = runtestprotocol(item, log=False) callreport = reports[1] assert callreport.failed assert not hasattr(callreport, "wasxfail") assert 'xfail' in callreport.keywords def test_xfail_not_report_default(self, testdir): p = testdir.makepyfile(test_one=""" import pytest @pytest.mark.xfail def test_this(): assert 0 """) testdir.runpytest(p, '-v') #result.stdout.fnmatch_lines([ # "*HINT*use*-r*" #]) def test_xfail_not_run_xfail_reporting(self, testdir): p = testdir.makepyfile(test_one=""" import pytest @pytest.mark.xfail(run=False, reason="noway") def test_this(): assert 0 @pytest.mark.xfail("True", run=False) def test_this_true(): assert 0 @pytest.mark.xfail("False", run=False, reason="huh") def test_this_false(): assert 1 """) result = testdir.runpytest(p, '--report=xfailed', ) result.stdout.fnmatch_lines([ "*test_one*test_this*", "*NOTRUN*noway", "*test_one*test_this_true*", "*NOTRUN*condition:*True*", "*1 passed*", ]) def test_xfail_not_run_no_setup_run(self, testdir): p = testdir.makepyfile(test_one=""" import pytest @pytest.mark.xfail(run=False, reason="hello") def test_this(): assert 0 def setup_module(mod): raise ValueError(42) """) result = testdir.runpytest(p, '--report=xfailed', ) result.stdout.fnmatch_lines([ "*test_one*test_this*", "*NOTRUN*hello", "*1 xfailed*", ]) def test_xfail_xpass(self, testdir): p = testdir.makepyfile(test_one=""" import pytest @pytest.mark.xfail def test_that(): assert 1 """) result = testdir.runpytest(p, '-rX') result.stdout.fnmatch_lines([ "*XPASS*test_that*", "*1 xpassed*" ]) assert result.ret == 0 def test_xfail_imperative(self, testdir): p = testdir.makepyfile(""" import pytest def test_this(): pytest.xfail("hello") """) result = testdir.runpytest(p) result.stdout.fnmatch_lines([ "*1 xfailed*", ]) result = testdir.runpytest(p, "-rx") result.stdout.fnmatch_lines([ "*XFAIL*test_this*", "*reason:*hello*", ]) result = testdir.runpytest(p, "--runxfail") result.stdout.fnmatch_lines("*1 pass*") def test_xfail_imperative_in_setup_function(self, testdir): p = testdir.makepyfile(""" import pytest def setup_function(function): pytest.xfail("hello") def test_this(): assert 0 """) result = testdir.runpytest(p) result.stdout.fnmatch_lines([ "*1 xfailed*", ]) result = testdir.runpytest(p, "-rx") result.stdout.fnmatch_lines([ "*XFAIL*test_this*", "*reason:*hello*", ]) result = testdir.runpytest(p, "--runxfail") result.stdout.fnmatch_lines(""" *def test_this* *1 fail* """) def xtest_dynamic_xfail_set_during_setup(self, testdir): p = testdir.makepyfile(""" import pytest def setup_function(function): pytest.mark.xfail(function) def test_this(): assert 0 def test_that(): assert 1 """) result = testdir.runpytest(p, '-rxX') result.stdout.fnmatch_lines([ "*XFAIL*test_this*", "*XPASS*test_that*", ]) def test_dynamic_xfail_no_run(self, testdir): p = testdir.makepyfile(""" import pytest def pytest_funcarg__arg(request): request.applymarker(pytest.mark.xfail(run=False)) def test_this(arg): assert 0 """) result = testdir.runpytest(p, '-rxX') result.stdout.fnmatch_lines([ "*XFAIL*test_this*", "*NOTRUN*", ]) def test_dynamic_xfail_set_during_funcarg_setup(self, testdir): p = testdir.makepyfile(""" import pytest def pytest_funcarg__arg(request): request.applymarker(pytest.mark.xfail) def test_this2(arg): assert 0 """) result = testdir.runpytest(p) result.stdout.fnmatch_lines([ "*1 xfailed*", ]) @pytest.mark.parametrize('expected, actual, matchline', [('TypeError', 'TypeError', "*1 xfailed*"), ('(AttributeError, TypeError)', 'TypeError', "*1 xfailed*"), ('TypeError', 'IndexError', "*1 failed*"), ('(AttributeError, TypeError)', 'IndexError', "*1 failed*"), ]) def test_xfail_raises(self, expected, actual, matchline, testdir): p = testdir.makepyfile(""" import pytest @pytest.mark.xfail(raises=%s) def test_raises(): raise %s() """ % (expected, actual)) result = testdir.runpytest(p) result.stdout.fnmatch_lines([ matchline, ]) def test_strict_sanity(self, testdir): """sanity check for xfail(strict=True): a failing test should behave exactly like a normal xfail. """ p = testdir.makepyfile(""" import pytest @pytest.mark.xfail(reason='unsupported feature', strict=True) def test_foo(): assert 0 """) result = testdir.runpytest(p, '-rxX') result.stdout.fnmatch_lines([ '*XFAIL*', '*unsupported feature*', ]) assert result.ret == 0 @pytest.mark.parametrize('strict', [True, False]) def test_strict_xfail(self, testdir, strict): p = testdir.makepyfile(""" import pytest @pytest.mark.xfail(reason='unsupported feature', strict=%s) def test_foo(): with open('foo_executed', 'w'): pass # make sure test executes """ % strict) result = testdir.runpytest(p, '-rxX') if strict: result.stdout.fnmatch_lines([ '*test_foo*', '*XPASS(strict)*unsupported feature*', ]) else: result.stdout.fnmatch_lines([ '*test_strict_xfail*', 'XPASS test_strict_xfail.py::test_foo unsupported feature', ]) assert result.ret == (1 if strict else 0) assert testdir.tmpdir.join('foo_executed').isfile() @pytest.mark.parametrize('strict', [True, False]) def test_strict_xfail_condition(self, testdir, strict): p = testdir.makepyfile(""" import pytest @pytest.mark.xfail(False, reason='unsupported feature', strict=%s) def test_foo(): pass """ % strict) result = testdir.runpytest(p, '-rxX') result.stdout.fnmatch_lines('*1 passed*') assert result.ret == 0 @pytest.mark.parametrize('strict_val', ['true', 'false']) def test_strict_xfail_default_from_file(self, testdir, strict_val): testdir.makeini(''' [pytest] xfail_strict = %s ''' % strict_val) p = testdir.makepyfile(""" import pytest @pytest.mark.xfail(reason='unsupported feature') def test_foo(): pass """) result = testdir.runpytest(p, '-rxX') strict = strict_val == 'true' result.stdout.fnmatch_lines('*1 failed*' if strict else '*1 xpassed*') assert result.ret == (1 if strict else 0) class TestXFailwithSetupTeardown: def test_failing_setup_issue9(self, testdir): testdir.makepyfile(""" import pytest def setup_function(func): assert 0 @pytest.mark.xfail def test_func(): pass """) result = testdir.runpytest() result.stdout.fnmatch_lines([ "*1 xfail*", ]) def test_failing_teardown_issue9(self, testdir): testdir.makepyfile(""" import pytest def teardown_function(func): assert 0 @pytest.mark.xfail def test_func(): pass """) result = testdir.runpytest() result.stdout.fnmatch_lines([ "*1 xfail*", ]) class TestSkip: def test_skip_class(self, testdir): testdir.makepyfile(""" import pytest @pytest.mark.skip class TestSomething(object): def test_foo(self): pass def test_bar(self): pass def test_baz(): pass """) rec = testdir.inline_run() rec.assertoutcome(skipped=2, passed=1) def test_skips_on_false_string(self, testdir): testdir.makepyfile(""" import pytest @pytest.mark.skip('False') def test_foo(): pass """) rec = testdir.inline_run() rec.assertoutcome(skipped=1) def test_arg_as_reason(self, testdir): testdir.makepyfile(""" import pytest @pytest.mark.skip('testing stuff') def test_bar(): pass """) result = testdir.runpytest('-rs') result.stdout.fnmatch_lines([ "*testing stuff*", "*1 skipped*", ]) def test_skip_no_reason(self, testdir): testdir.makepyfile(""" import pytest @pytest.mark.skip def test_foo(): pass """) result = testdir.runpytest('-rs') result.stdout.fnmatch_lines([ "*unconditional skip*", "*1 skipped*", ]) def test_skip_with_reason(self, testdir): testdir.makepyfile(""" import pytest @pytest.mark.skip(reason="for lolz") def test_bar(): pass """) result = testdir.runpytest('-rs') result.stdout.fnmatch_lines([ "*for lolz*", "*1 skipped*", ]) def test_only_skips_marked_test(self, testdir): testdir.makepyfile(""" import pytest @pytest.mark.skip def test_foo(): pass @pytest.mark.skip(reason="nothing in particular") def test_bar(): pass def test_baz(): assert True """) result = testdir.runpytest('-rs') result.stdout.fnmatch_lines([ "*nothing in particular*", "*1 passed*2 skipped*", ]) class TestSkipif: def test_skipif_conditional(self, testdir): item = testdir.getitem(""" import pytest @pytest.mark.skipif("hasattr(os, 'sep')") def test_func(): pass """) # noqa x = pytest.raises(pytest.skip.Exception, lambda: pytest_runtest_setup(item)) assert x.value.msg == "condition: hasattr(os, 'sep')" @pytest.mark.parametrize('params', [ '"hasattr(sys, \'platform\')"', 'True, reason="invalid platform"', ]) def test_skipif_reporting(self, testdir, params): p = testdir.makepyfile(test_foo=""" import pytest @pytest.mark.skipif(%(params)s) def test_that(): assert 0 """ % dict(params=params)) result = testdir.runpytest(p, '-s', '-rs') result.stdout.fnmatch_lines([ "*SKIP*1*test_foo.py*platform*", "*1 skipped*" ]) assert result.ret == 0 @pytest.mark.parametrize('marker, msg1, msg2', [ ('skipif', 'SKIP', 'skipped'), ('xfail', 'XPASS', 'xpassed'), ]) def test_skipif_reporting_multiple(self, testdir, marker, msg1, msg2): testdir.makepyfile(test_foo=""" import pytest @pytest.mark.{marker}(False, reason='first_condition') @pytest.mark.{marker}(True, reason='second_condition') def test_foobar(): assert 1 """.format(marker=marker)) result = testdir.runpytest('-s', '-rsxX') result.stdout.fnmatch_lines([ "*{msg1}*test_foo.py*second_condition*".format(msg1=msg1), "*1 {msg2}*".format(msg2=msg2), ]) assert result.ret == 0 def test_skip_not_report_default(testdir): p = testdir.makepyfile(test_one=""" import pytest def test_this(): pytest.skip("hello") """) result = testdir.runpytest(p, '-v') result.stdout.fnmatch_lines([ #"*HINT*use*-r*", "*1 skipped*", ]) def test_skipif_class(testdir): p = testdir.makepyfile(""" import pytest class TestClass: pytestmark = pytest.mark.skipif("True") def test_that(self): assert 0 def test_though(self): assert 0 """) result = testdir.runpytest(p) result.stdout.fnmatch_lines([ "*2 skipped*" ]) def test_skip_reasons_folding(): path = 'xyz' lineno = 3 message = "justso" longrepr = (path, lineno, message) class X: pass ev1 = X() ev1.when = "execute" ev1.skipped = True ev1.longrepr = longrepr ev2 = X() ev2.longrepr = longrepr ev2.skipped = True l = folded_skips([ev1, ev2]) assert len(l) == 1 num, fspath, lineno, reason = l[0] assert num == 2 assert fspath == path assert lineno == lineno assert reason == message def test_skipped_reasons_functional(testdir): testdir.makepyfile( test_one=""" from conftest import doskip def setup_function(func): doskip() def test_func(): pass class TestClass: def test_method(self): doskip() """, test_two = """ from conftest import doskip doskip() """, conftest = """ import pytest def doskip(): pytest.skip('test') """ ) result = testdir.runpytest('--report=skipped') result.stdout.fnmatch_lines([ "*SKIP*3*conftest.py:3: test", ]) assert result.ret == 0 def test_reportchars(testdir): testdir.makepyfile(""" import pytest def test_1(): assert 0 @pytest.mark.xfail def test_2(): assert 0 @pytest.mark.xfail def test_3(): pass def test_4(): pytest.skip("four") """) result = testdir.runpytest("-rfxXs") result.stdout.fnmatch_lines([ "FAIL*test_1*", "XFAIL*test_2*", "XPASS*test_3*", "SKIP*four*", ]) def test_reportchars_error(testdir): testdir.makepyfile( conftest=""" def pytest_runtest_teardown(): assert 0 """, test_simple=""" def test_foo(): pass """) result = testdir.runpytest('-rE') result.stdout.fnmatch_lines([ 'ERROR*test_foo*', ]) def test_reportchars_all(testdir): testdir.makepyfile(""" import pytest def test_1(): assert 0 @pytest.mark.xfail def test_2(): assert 0 @pytest.mark.xfail def test_3(): pass def test_4(): pytest.skip("four") """) result = testdir.runpytest("-ra") result.stdout.fnmatch_lines([ "FAIL*test_1*", "SKIP*four*", "XFAIL*test_2*", "XPASS*test_3*", ]) def test_reportchars_all_error(testdir): testdir.makepyfile( conftest=""" def pytest_runtest_teardown(): assert 0 """, test_simple=""" def test_foo(): pass """) result = testdir.runpytest('-ra') result.stdout.fnmatch_lines([ 'ERROR*test_foo*', ]) @pytest.mark.xfail("hasattr(sys, 'pypy_version_info')") def test_errors_in_xfail_skip_expressions(testdir): testdir.makepyfile(""" import pytest @pytest.mark.skipif("asd") def test_nameerror(): pass @pytest.mark.xfail("syntax error") def test_syntax(): pass def test_func(): pass """) result = testdir.runpytest() markline = " ^" if sys.platform.startswith("java"): # XXX report this to java markline = "*" + markline[8:] result.stdout.fnmatch_lines([ "*ERROR*test_nameerror*", "*evaluating*skipif*expression*", "*asd*", "*ERROR*test_syntax*", "*evaluating*xfail*expression*", " syntax error", markline, "SyntaxError: invalid syntax", "*1 pass*2 error*", ]) def test_xfail_skipif_with_globals(testdir): testdir.makepyfile(""" import pytest x = 3 @pytest.mark.skipif("x == 3") def test_skip1(): pass @pytest.mark.xfail("x == 3") def test_boolean(): assert 0 """) result = testdir.runpytest("-rsx") result.stdout.fnmatch_lines([ "*SKIP*x == 3*", "*XFAIL*test_boolean*", "*x == 3*", ]) def test_direct_gives_error(testdir): testdir.makepyfile(""" import pytest @pytest.mark.skipif(True) def test_skip1(): pass """) result = testdir.runpytest() result.stdout.fnmatch_lines([ "*1 error*", ]) def test_default_markers(testdir): result = testdir.runpytest("--markers") result.stdout.fnmatch_lines([ "*skipif(*condition)*skip*", "*xfail(*condition, reason=None, run=True, raises=None)*expected failure*", ]) def test_xfail_test_setup_exception(testdir): testdir.makeconftest(""" def pytest_runtest_setup(): 0 / 0 """) p = testdir.makepyfile(""" import pytest @pytest.mark.xfail def test_func(): assert 0 """) result = testdir.runpytest(p) assert result.ret == 0 assert 'xfailed' in result.stdout.str() assert 'xpassed' not in result.stdout.str() def test_imperativeskip_on_xfail_test(testdir): testdir.makepyfile(""" import pytest @pytest.mark.xfail def test_that_fails(): assert 0 @pytest.mark.skipif("True") def test_hello(): pass """) testdir.makeconftest(""" import pytest def pytest_runtest_setup(item): pytest.skip("abc") """) result = testdir.runpytest("-rsxX") result.stdout.fnmatch_lines_random(""" *SKIP*abc* *SKIP*condition: True* *2 skipped* """) class TestBooleanCondition: def test_skipif(self, testdir): testdir.makepyfile(""" import pytest @pytest.mark.skipif(True, reason="True123") def test_func1(): pass @pytest.mark.skipif(False, reason="True123") def test_func2(): pass """) result = testdir.runpytest() result.stdout.fnmatch_lines(""" *1 passed*1 skipped* """) def test_skipif_noreason(self, testdir): testdir.makepyfile(""" import pytest @pytest.mark.skipif(True) def test_func(): pass """) result = testdir.runpytest("-rs") result.stdout.fnmatch_lines(""" *1 error* """) def test_xfail(self, testdir): testdir.makepyfile(""" import pytest @pytest.mark.xfail(True, reason="True123") def test_func(): assert 0 """) result = testdir.runpytest("-rxs") result.stdout.fnmatch_lines(""" *XFAIL* *True123* *1 xfail* """) def test_xfail_item(testdir): # Ensure pytest.xfail works with non-Python Item testdir.makeconftest(""" import pytest class MyItem(pytest.Item): nodeid = 'foo' def runtest(self): pytest.xfail("Expected Failure") def pytest_collect_file(path, parent): return MyItem("foo", parent) """) result = testdir.inline_run() passed, skipped, failed = result.listoutcomes() assert not failed xfailed = [r for r in skipped if hasattr(r, 'wasxfail')] assert xfailed