Merge pull request #817 from nicoddemus/non-collection-exit-error

return non-zero exit code if no tests are collected
This commit is contained in:
Ronny Pfannschmidt 2015-08-21 18:30:57 +02:00
commit 287df16c9c
20 changed files with 99 additions and 40 deletions

View File

@ -28,7 +28,7 @@ env:
- TESTENV=py35
- TESTENV=pypy
script: tox --recreate -i ALL=https://devpi.net/hpk/dev/ -e $TESTENV
script: tox --recreate -e $TESTENV
notifications:
irc:

View File

@ -15,6 +15,7 @@ Bob Ippolito
Brian Dorsey
Brian Okken
Brianna Laugher
Bruno Oliveira
Carl Friedrich Bolz
Charles Cloud
Chris Lamb

View File

@ -58,6 +58,15 @@
- Summary bar now is colored yellow for warning
situations such as: all tests either were skipped or xpass/xfailed,
or no tests were run at all (this is a partial fix for issue500).
- fix issue812: pytest now exits with status code 5 in situations where no
tests were run at all, such as the directory given in the command line does
not contain any tests or as result of a command line option filters
all out all tests (-k for example).
Thanks Eric Siegerman (issue812) and Bruno Oliveira for the PR.
- Summary bar now is colored yellow for warning
situations such as: all tests either were skipped or xpass/xfailed,
or no tests were run at all (related to issue500).
Thanks Eric Siegerman.
- New `testpaths` ini option: list of directories to search for tests

View File

@ -19,6 +19,7 @@ EXIT_TESTSFAILED = 1
EXIT_INTERRUPTED = 2
EXIT_INTERNALERROR = 3
EXIT_USAGEERROR = 4
EXIT_NOTESTSCOLLECTED = 5
name_re = re.compile("^[a-zA-Z_]\w*$")
@ -100,8 +101,10 @@ def wrap_session(config, doit):
if excinfo.errisinstance(SystemExit):
sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
else:
if session._testsfailed:
if session.testsfailed:
session.exitstatus = EXIT_TESTSFAILED
elif session.testscollected == 0:
session.exitstatus = EXIT_NOTESTSCOLLECTED
finally:
excinfo = None # Explicitly break reference cycle.
session.startdir.chdir()
@ -509,7 +512,8 @@ class Session(FSCollector):
FSCollector.__init__(self, config.rootdir, parent=None,
config=config, session=self)
self._fs2hookproxy = {}
self._testsfailed = 0
self.testsfailed = 0
self.testscollected = 0
self.shouldstop = False
self.trace = config.trace.root.get("collection")
self._norecursepatterns = config.getini("norecursedirs")
@ -527,11 +531,11 @@ class Session(FSCollector):
@pytest.hookimpl(tryfirst=True)
def pytest_runtest_logreport(self, report):
if report.failed and not hasattr(report, 'wasxfail'):
self._testsfailed += 1
self.testsfailed += 1
maxfail = self.config.getvalue("maxfail")
if maxfail and self._testsfailed >= maxfail:
if maxfail and self.testsfailed >= maxfail:
self.shouldstop = "stopping after %d failures" % (
self._testsfailed)
self.testsfailed)
pytest_collectreport = pytest_runtest_logreport
def isinitpath(self, path):
@ -564,6 +568,7 @@ class Session(FSCollector):
config=self.config, items=items)
finally:
hook.pytest_collection_finish(session=self)
self.testscollected = len(items)
return items
def _perform_collect(self, args, genitems):

View File

@ -2,6 +2,8 @@
This is a good source for looking at the various reporting hooks.
"""
from _pytest.main import EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, \
EXIT_USAGEERROR, EXIT_NOTESTSCOLLECTED
import pytest
import pluggy
import py
@ -355,12 +357,15 @@ class TerminalReporter:
outcome = yield
outcome.get_result()
self._tw.line("")
if exitstatus in (0, 1, 2, 4):
summary_exit_codes = (
EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, EXIT_USAGEERROR,
EXIT_NOTESTSCOLLECTED)
if exitstatus in summary_exit_codes:
self.summary_errors()
self.summary_failures()
self.summary_warnings()
self.config.hook.pytest_terminal_summary(terminalreporter=self)
if exitstatus == 2:
if exitstatus == EXIT_INTERRUPTED:
self._report_keyboardinterrupt()
del self._keyboardinterrupt_memo
self.summary_deselected()

View File

@ -1,5 +1,7 @@
import sys
import py, pytest
from _pytest.main import EXIT_NOTESTSCOLLECTED, EXIT_USAGEERROR
class TestGeneralUsage:
def test_config_error(self, testdir):
@ -147,7 +149,7 @@ class TestGeneralUsage:
pytest.skip("early")
""")
result = testdir.runpytest()
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
result.stdout.fnmatch_lines([
"*1 skip*"
])
@ -177,7 +179,7 @@ class TestGeneralUsage:
sys.stderr.write("stder42\\n")
""")
result = testdir.runpytest()
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
assert "should not be seen" not in result.stdout.str()
assert "stderr42" not in result.stderr.str()
@ -212,13 +214,13 @@ class TestGeneralUsage:
sub2 = testdir.tmpdir.mkdir("sub2")
sub1.join("conftest.py").write("assert 0")
result = testdir.runpytest(sub2)
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
sub2.ensure("__init__.py")
p = sub2.ensure("test_hello.py")
result = testdir.runpytest(p)
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
result = testdir.runpytest(sub1)
assert result.ret != 0
assert result.ret == EXIT_USAGEERROR
def test_directory_skipped(self, testdir):
testdir.makeconftest("""
@ -228,7 +230,7 @@ class TestGeneralUsage:
""")
testdir.makepyfile("def test_hello(): pass")
result = testdir.runpytest()
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
result.stdout.fnmatch_lines([
"*1 skipped*"
])
@ -479,7 +481,7 @@ class TestInvocationVariants:
def test_invoke_with_path(self, tmpdir, capsys):
retcode = pytest.main(tmpdir)
assert not retcode
assert retcode == EXIT_NOTESTSCOLLECTED
out, err = capsys.readouterr()
def test_invoke_plugin_api(self, testdir, capsys):

View File

@ -1,6 +1,8 @@
import sys
from textwrap import dedent
import pytest, py
from _pytest.main import EXIT_NOTESTSCOLLECTED
class TestModule:
def test_failing_import(self, testdir):
@ -906,7 +908,7 @@ def test_unorderable_types(testdir):
""")
result = testdir.runpytest()
assert "TypeError" not in result.stdout.str()
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
def test_collect_functools_partial(testdir):

View File

@ -12,6 +12,7 @@ if sys.platform.startswith("java"):
from _pytest.assertion import util
from _pytest.assertion.rewrite import rewrite_asserts, PYTEST_TAG
from _pytest.main import EXIT_NOTESTSCOLLECTED
def setup_module(mod):
@ -429,7 +430,7 @@ class TestRewriteOnImport:
import sys
sys.path.append(%r)
import test_gum.test_lizard""" % (z_fn,))
assert testdir.runpytest().ret == 0
assert testdir.runpytest().ret == EXIT_NOTESTSCOLLECTED
def test_readonly(self, testdir):
sub = testdir.mkdir("testing")
@ -497,7 +498,7 @@ def test_rewritten():
pkg = testdir.mkdir('a_package_without_init_py')
pkg.join('module.py').ensure()
testdir.makepyfile("import a_package_without_init_py.module")
assert testdir.runpytest().ret == 0
assert testdir.runpytest().ret == EXIT_NOTESTSCOLLECTED
class TestAssertionRewriteHookDetails(object):
def test_loader_is_package_false_for_module(self, testdir):

View File

@ -10,6 +10,7 @@ import contextlib
from _pytest import capture
from _pytest.capture import CaptureManager
from _pytest.main import EXIT_NOTESTSCOLLECTED
from py.builtin import print_
needsosdup = pytest.mark.xfail("not hasattr(os, 'dup')")
@ -365,7 +366,7 @@ class TestLoggingInteraction:
""")
# make sure that logging is still captured in tests
result = testdir.runpytest_subprocess("-s", "-p", "no:capturelog")
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
result.stderr.fnmatch_lines([
"WARNING*hello435*",
])

View File

@ -1,6 +1,6 @@
import pytest, py
from _pytest.main import Session
from _pytest.main import Session, EXIT_NOTESTSCOLLECTED
class TestCollector:
def test_collect_versus_item(self):
@ -247,10 +247,10 @@ class TestCustomConftests:
p = testdir.makepyfile("def test_hello(): pass")
result = testdir.runpytest(p)
assert result.ret == 0
assert "1 passed" in result.stdout.str()
result.stdout.fnmatch_lines("*1 passed*")
result = testdir.runpytest()
assert result.ret == 0
assert "1 passed" not in result.stdout.str()
assert result.ret == EXIT_NOTESTSCOLLECTED
result.stdout.fnmatch_lines("*collected 0 items*")
def test_collectignore_exclude_on_option(self, testdir):
testdir.makeconftest("""
@ -264,7 +264,7 @@ class TestCustomConftests:
testdir.mkdir("hello")
testdir.makepyfile(test_world="def test_hello(): pass")
result = testdir.runpytest()
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
assert "passed" not in result.stdout.str()
result = testdir.runpytest("--XX")
assert result.ret == 0

View File

@ -1,6 +1,7 @@
import py, pytest
from _pytest.config import getcfg, get_common_ancestor, determine_setup
from _pytest.main import EXIT_NOTESTSCOLLECTED
class TestParseIni:
def test_getcfg_and_config(self, testdir, tmpdir):
@ -343,7 +344,7 @@ def test_invalid_options_show_extra_information(testdir):
@pytest.mark.skipif("sys.platform == 'win32'")
def test_toolongargs_issue224(testdir):
result = testdir.runpytest("-m", "hello" * 500)
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
def test_notify_exception(testdir, capfd):
config = testdir.parseconfig()

View File

@ -1,6 +1,7 @@
from textwrap import dedent
import py, pytest
from _pytest.config import PytestPluginManager
from _pytest.main import EXIT_NOTESTSCOLLECTED, EXIT_USAGEERROR
@pytest.fixture(scope="module", params=["global", "inpackage"])
@ -166,7 +167,10 @@ def test_conftest_confcutdir(testdir):
def test_no_conftest(testdir):
testdir.makeconftest("assert 0")
result = testdir.runpytest("--noconftest")
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
result = testdir.runpytest()
assert result.ret == EXIT_USAGEERROR
def test_conftest_existing_resultlog(testdir):
x = testdir.mkdir("tests")

View File

@ -1,3 +1,4 @@
from _pytest.main import EXIT_NOTESTSCOLLECTED
import pytest
def test_version(testdir, pytestconfig):
@ -43,7 +44,7 @@ def test_hookvalidation_optional(testdir):
pass
""")
result = testdir.runpytest()
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
def test_traceconfig(testdir):
result = testdir.runpytest("--traceconfig")
@ -54,14 +55,14 @@ def test_traceconfig(testdir):
def test_debug(testdir, monkeypatch):
result = testdir.runpytest_subprocess("--debug")
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
p = testdir.tmpdir.join("pytestdebug.log")
assert "pytest_sessionstart" in p.read()
def test_PYTEST_DEBUG(testdir, monkeypatch):
monkeypatch.setenv("PYTEST_DEBUG", "1")
result = testdir.runpytest_subprocess()
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
result.stderr.fnmatch_lines([
"*pytest_plugin_registered*",
"*manager*PluginManager*"

View File

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
from xml.dom import minidom
from _pytest.main import EXIT_NOTESTSCOLLECTED
import py, sys, os
from _pytest.junitxml import LogXML
@ -298,7 +299,7 @@ class TestPython:
def test_collect_skipped(self, testdir):
testdir.makepyfile("import pytest; pytest.skip('xyz')")
result, dom = runandparse(testdir)
assert not result.ret
assert result.ret == EXIT_NOTESTSCOLLECTED
node = dom.getElementsByTagName("testsuite")[0]
assert_attr(node, skips=1, tests=0)
tnode = node.getElementsByTagName("testcase")[0]

View File

@ -3,6 +3,7 @@ import py
import os
from _pytest.config import get_config, PytestPluginManager
from _pytest.main import EXIT_NOTESTSCOLLECTED
@pytest.fixture
def pytestpm():
@ -223,7 +224,7 @@ class TestPytestPluginManager:
p.copy(p.dirpath("skipping2.py"))
monkeypatch.setenv("PYTEST_PLUGINS", "skipping2")
result = testdir.runpytest("-rw", "-p", "skipping1", syspathinsert=True)
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
result.stdout.fnmatch_lines([
"WI1*skipped plugin*skipping1*hello*",
"WI1*skipped plugin*skipping2*hello*",

View File

@ -293,8 +293,8 @@ class TestExecutionForked(BaseFunctionalTests):
def getrunner(self):
# XXX re-arrange this test to live in pytest-xdist
xplugin = pytest.importorskip("xdist.plugin")
return xplugin.forked_run_report
boxed = pytest.importorskip("xdist.boxed")
return boxed.forked_run_report
def test_suicide(self, testdir):
reports = testdir.runitem("""
@ -431,6 +431,27 @@ def test_pytest_fail_notrace(testdir):
])
assert 'def teardown_function' not in result.stdout.str()
def test_pytest_no_tests_collected_exit_status(testdir):
result = testdir.runpytest()
result.stdout.fnmatch_lines('*collected 0 items*')
assert result.ret == main.EXIT_NOTESTSCOLLECTED
testdir.makepyfile(test_foo="""
def test_foo():
assert 1
""")
result = testdir.runpytest()
result.stdout.fnmatch_lines('*collected 1 items*')
result.stdout.fnmatch_lines('*1 passed*')
assert result.ret == main.EXIT_OK
result = testdir.runpytest('-k nonmatch')
result.stdout.fnmatch_lines('*collected 1 items*')
result.stdout.fnmatch_lines('*1 deselected*')
assert result.ret == main.EXIT_NOTESTSCOLLECTED
def test_exception_printing_skip():
try:
pytest.skip("hello")

View File

@ -1,5 +1,7 @@
import pytest
from _pytest.main import EXIT_NOTESTSCOLLECTED
class SessionTests:
def test_basic_testitem_events(self, testdir):
tfile = testdir.makepyfile("""
@ -239,4 +241,4 @@ def test_sessionfinish_with_start(testdir):
""")
res = testdir.runpytest("--collect-only")
assert res.ret == 0
assert res.ret == EXIT_NOTESTSCOLLECTED

View File

@ -7,6 +7,7 @@ import py
import pluggy
import sys
from _pytest.main import EXIT_NOTESTSCOLLECTED
from _pytest.terminal import TerminalReporter, repr_pythonversion, getreportopt
from _pytest.terminal import build_summary_stats_line, _plugin_nameversions
from _pytest import runner
@ -596,7 +597,7 @@ def test_traceconfig(testdir, monkeypatch):
result.stdout.fnmatch_lines([
"*active plugins*"
])
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
class TestGenericReporting:

View File

@ -1,3 +1,4 @@
from _pytest.main import EXIT_NOTESTSCOLLECTED
import pytest
def test_simple_unittest(testdir):
@ -41,7 +42,7 @@ def test_isclasscheck_issue53(testdir):
E = _E()
""")
result = testdir.runpytest(testpath)
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
def test_setup(testdir):
testpath = testdir.makepyfile("""
@ -572,7 +573,7 @@ def test_unorderable_types(testdir):
""")
result = testdir.runpytest()
assert "TypeError" not in result.stdout.str()
assert result.ret == 0
assert result.ret == EXIT_NOTESTSCOLLECTED
def test_unittest_typerror_traceback(testdir):
testdir.makepyfile("""

View File

@ -22,7 +22,7 @@ deps=
[testenv:py27-subprocess]
changedir=.
basepython=python2.7
deps=pytest-xdist
deps=pytest-xdist>=1.13
mock
nose
commands=
@ -37,7 +37,7 @@ deps = pytest-flakes>=0.2
commands = py.test --flakes -m flakes _pytest testing
[testenv:py27-xdist]
deps=pytest-xdist
deps=pytest-xdist>=1.13
mock
nose
commands=
@ -63,7 +63,7 @@ commands=
py.test -rfsxX test_pdb.py test_terminal.py test_unittest.py
[testenv:py27-nobyte]
deps=pytest-xdist
deps=pytest-xdist>=1.13
distribute=true
setenv=
PYTHONDONTWRITEBYTECODE=1