Raise CollectError if pytest.skip() is called during collection

pytest.skip() must not be used at module level because it can easily be
misunderstood and used as a decorator instead of pytest.mark.skip, causing the
whole module to be skipped instead of just the test being decorated.

This is unexpected for users used to the @unittest.skip decorator and therefore
it is best to bail out with a clean error when it happens.

The pytest equivalent of @unittest.skip is @pytest.mark.skip .

Adapt existing tests that were actually relying on this behaviour and add a
test that explicitly test that collection fails.

fix #607
This commit is contained in:
Omar Kohl 2016-04-10 19:57:45 +02:00
parent f2bb3df310
commit d81f23009b
7 changed files with 66 additions and 53 deletions

View File

@ -656,6 +656,12 @@ class Module(pytest.File, PyCollector):
"Make sure your test modules/packages have valid Python names."
% (self.fspath, exc or exc_class)
)
except _pytest.runner.Skipped:
raise self.CollectError(
"Using @pytest.skip outside a test (e.g. as a test function "
"decorator) is not allowed. Use @pytest.mark.skip or "
"@pytest.mark.skipif instead."
)
#print "imported test module", mod
self.config.pluginmanager.consider_module(mod)
return mod

View File

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
from xml.dom import minidom
from _pytest.main import EXIT_NOTESTSCOLLECTED
import py
import sys
import os
@ -158,6 +157,47 @@ class TestPython:
snode = tnode.find_first_by_tag("skipped")
snode.assert_attr(type="pytest.skip", message="hello23", )
def test_mark_skip_contains_name_reason(self, testdir):
testdir.makepyfile("""
import pytest
@pytest.mark.skip(reason="hello24")
def test_skip():
assert True
""")
result, dom = runandparse(testdir)
assert result.ret == 0
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_mark_skip_contains_name_reason.py",
line="1",
classname="test_mark_skip_contains_name_reason",
name="test_skip")
snode = tnode.find_first_by_tag("skipped")
snode.assert_attr(type="pytest.skip", message="hello24", )
def test_mark_skipif_contains_name_reason(self, testdir):
testdir.makepyfile("""
import pytest
GLOBAL_CONDITION = True
@pytest.mark.skipif(GLOBAL_CONDITION, reason="hello25")
def test_skip():
assert True
""")
result, dom = runandparse(testdir)
assert result.ret == 0
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_mark_skipif_contains_name_reason.py",
line="2",
classname="test_mark_skipif_contains_name_reason",
name="test_skip")
snode = tnode.find_first_by_tag("skipped")
snode.assert_attr(type="pytest.skip", message="hello25", )
def test_classname_instance(self, testdir):
testdir.makepyfile("""
class TestClass:
@ -351,23 +391,6 @@ class TestPython:
fnode.assert_attr(message="collection failure")
assert "SyntaxError" in fnode.toxml()
def test_collect_skipped(self, testdir):
testdir.makepyfile("import pytest; pytest.skip('xyz')")
result, dom = runandparse(testdir)
assert result.ret == EXIT_NOTESTSCOLLECTED
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=1, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_collect_skipped.py",
name="test_collect_skipped")
# pytest doesn't give us a line here.
assert tnode["line"] is None
fnode = tnode.find_first_by_tag("skipped")
fnode.assert_attr(message="collection skipped")
def test_unicode(self, testdir):
value = 'hx\xc4\x85\xc4\x87\n'
testdir.makepyfile("""

View File

@ -83,19 +83,10 @@ class TestWithFunctionIntegration:
def test_collection_report(self, testdir):
ok = testdir.makepyfile(test_collection_ok="")
skip = testdir.makepyfile(test_collection_skip=
"import pytest ; pytest.skip('hello')")
fail = testdir.makepyfile(test_collection_fail="XXX")
lines = self.getresultlog(testdir, ok)
assert not lines
lines = self.getresultlog(testdir, skip)
assert len(lines) == 2
assert lines[0].startswith("S ")
assert lines[0].endswith("test_collection_skip.py")
assert lines[1].startswith(" ")
assert lines[1].endswith("test_collection_skip.py:1: Skipped: hello")
lines = self.getresultlog(testdir, fail)
assert lines
assert lines[0].startswith("F ")

View File

@ -365,18 +365,6 @@ class TestSessionReports:
assert res[0].name == "test_func1"
assert res[1].name == "TestClass"
def test_skip_at_module_scope(self, testdir):
col = testdir.getmodulecol("""
import pytest
pytest.skip("hello")
def test_func():
pass
""")
rep = main.collect_one_node(col)
assert not rep.failed
assert not rep.passed
assert rep.skipped
reporttypes = [
runner.BaseReport,

View File

@ -174,10 +174,6 @@ class TestNewSession(SessionTests):
class TestY(TestX):
pass
""",
test_two="""
import pytest
pytest.skip('xxx')
""",
test_three="xxxdsadsadsadsa",
__init__=""
)
@ -189,11 +185,9 @@ class TestNewSession(SessionTests):
started = reprec.getcalls("pytest_collectstart")
finished = reprec.getreports("pytest_collectreport")
assert len(started) == len(finished)
assert len(started) == 8 # XXX extra TopCollector
assert len(started) == 7 # XXX extra TopCollector
colfail = [x for x in finished if x.failed]
colskipped = [x for x in finished if x.skipped]
assert len(colfail) == 1
assert len(colskipped) == 1
def test_minus_x_import_error(self, testdir):
testdir.makepyfile(__init__="")

View File

@ -682,10 +682,6 @@ def test_skipped_reasons_functional(testdir):
def test_method(self):
doskip()
""",
test_two = """
from conftest import doskip
doskip()
""",
conftest = """
import pytest
def doskip():
@ -694,7 +690,7 @@ def test_skipped_reasons_functional(testdir):
)
result = testdir.runpytest('--report=skipped')
result.stdout.fnmatch_lines([
"*SKIP*3*conftest.py:3: test",
"*SKIP*2*conftest.py:3: test",
])
assert result.ret == 0
@ -941,3 +937,19 @@ def test_xfail_item(testdir):
assert not failed
xfailed = [r for r in skipped if hasattr(r, 'wasxfail')]
assert xfailed
def test_module_level_skip_error(testdir):
"""
Verify that using pytest.skip at module level causes a collection error
"""
testdir.makepyfile("""
import pytest
@pytest.skip
def test_func():
assert True
""")
result = testdir.runpytest()
result.stdout.fnmatch_lines(
"*Using @pytest.skip outside a test * is not allowed*"
)

View File

@ -223,8 +223,7 @@ class TestCollectonly:
""")
result = testdir.runpytest("--collect-only", "-rs")
result.stdout.fnmatch_lines([
"SKIP*hello*",
"*1 skip*",
"*ERROR collecting*",
])
def test_collectonly_failed_module(self, testdir):