diff --git a/.travis.yml b/.travis.yml
index 89ed8d187..e83220105 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -28,7 +28,7 @@ env:
- TESTENV=py35
- TESTENV=pypy
-script: tox --recreate -i ALL=https://devpi.net/hpk/dev/ -e $TESTENV
+script: tox --recreate -e $TESTENV
notifications:
irc:
diff --git a/AUTHORS b/AUTHORS
index 2fdc39fc2..7a4674388 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -15,6 +15,7 @@ Bob Ippolito
Brian Dorsey
Brian Okken
Brianna Laugher
+Bruno Oliveira
Carl Friedrich Bolz
Charles Cloud
Chris Lamb
diff --git a/CHANGELOG b/CHANGELOG
index 7d96418c8..c375c9141 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -58,6 +58,15 @@
- Summary bar now is colored yellow for warning
situations such as: all tests either were skipped or xpass/xfailed,
or no tests were run at all (this is a partial fix for issue500).
+- fix issue812: pytest now exits with status code 5 in situations where no
+ tests were run at all, such as the directory given in the command line does
+ not contain any tests or as result of a command line option filters
+ all out all tests (-k for example).
+ Thanks Eric Siegerman (issue812) and Bruno Oliveira for the PR.
+
+- Summary bar now is colored yellow for warning
+ situations such as: all tests either were skipped or xpass/xfailed,
+ or no tests were run at all (related to issue500).
Thanks Eric Siegerman.
- New `testpaths` ini option: list of directories to search for tests
diff --git a/_pytest/main.py b/_pytest/main.py
index fc9d64cf6..4f3d2625f 100644
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -19,6 +19,7 @@ EXIT_TESTSFAILED = 1
EXIT_INTERRUPTED = 2
EXIT_INTERNALERROR = 3
EXIT_USAGEERROR = 4
+EXIT_NOTESTSCOLLECTED = 5
name_re = re.compile("^[a-zA-Z_]\w*$")
@@ -100,8 +101,10 @@ def wrap_session(config, doit):
if excinfo.errisinstance(SystemExit):
sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
else:
- if session._testsfailed:
+ if session.testsfailed:
session.exitstatus = EXIT_TESTSFAILED
+ elif session.testscollected == 0:
+ session.exitstatus = EXIT_NOTESTSCOLLECTED
finally:
excinfo = None # Explicitly break reference cycle.
session.startdir.chdir()
@@ -509,7 +512,8 @@ class Session(FSCollector):
FSCollector.__init__(self, config.rootdir, parent=None,
config=config, session=self)
self._fs2hookproxy = {}
- self._testsfailed = 0
+ self.testsfailed = 0
+ self.testscollected = 0
self.shouldstop = False
self.trace = config.trace.root.get("collection")
self._norecursepatterns = config.getini("norecursedirs")
@@ -527,11 +531,11 @@ class Session(FSCollector):
@pytest.hookimpl(tryfirst=True)
def pytest_runtest_logreport(self, report):
if report.failed and not hasattr(report, 'wasxfail'):
- self._testsfailed += 1
+ self.testsfailed += 1
maxfail = self.config.getvalue("maxfail")
- if maxfail and self._testsfailed >= maxfail:
+ if maxfail and self.testsfailed >= maxfail:
self.shouldstop = "stopping after %d failures" % (
- self._testsfailed)
+ self.testsfailed)
pytest_collectreport = pytest_runtest_logreport
def isinitpath(self, path):
@@ -564,6 +568,7 @@ class Session(FSCollector):
config=self.config, items=items)
finally:
hook.pytest_collection_finish(session=self)
+ self.testscollected = len(items)
return items
def _perform_collect(self, args, genitems):
diff --git a/_pytest/terminal.py b/_pytest/terminal.py
index e3ca23533..3fd7f00c5 100644
--- a/_pytest/terminal.py
+++ b/_pytest/terminal.py
@@ -2,6 +2,8 @@
This is a good source for looking at the various reporting hooks.
"""
+from _pytest.main import EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, \
+ EXIT_USAGEERROR, EXIT_NOTESTSCOLLECTED
import pytest
import pluggy
import py
@@ -355,12 +357,15 @@ class TerminalReporter:
outcome = yield
outcome.get_result()
self._tw.line("")
- if exitstatus in (0, 1, 2, 4):
+ summary_exit_codes = (
+ EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, EXIT_USAGEERROR,
+ EXIT_NOTESTSCOLLECTED)
+ if exitstatus in summary_exit_codes:
self.summary_errors()
self.summary_failures()
self.summary_warnings()
self.config.hook.pytest_terminal_summary(terminalreporter=self)
- if exitstatus == 2:
+ if exitstatus == EXIT_INTERRUPTED:
self._report_keyboardinterrupt()
del self._keyboardinterrupt_memo
self.summary_deselected()
diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py
index 2e87ac466..b9a3fa381 100644
--- a/testing/acceptance_test.py
+++ b/testing/acceptance_test.py
@@ -1,5 +1,7 @@
import sys
import py, pytest
+from _pytest.main import EXIT_NOTESTSCOLLECTED, EXIT_USAGEERROR
+
class TestGeneralUsage:
def test_config_error(self, testdir):
@@ -147,7 +149,7 @@ class TestGeneralUsage:
pytest.skip("early")
""")
result = testdir.runpytest()
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
result.stdout.fnmatch_lines([
"*1 skip*"
])
@@ -177,7 +179,7 @@ class TestGeneralUsage:
sys.stderr.write("stder42\\n")
""")
result = testdir.runpytest()
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
assert "should not be seen" not in result.stdout.str()
assert "stderr42" not in result.stderr.str()
@@ -212,13 +214,13 @@ class TestGeneralUsage:
sub2 = testdir.tmpdir.mkdir("sub2")
sub1.join("conftest.py").write("assert 0")
result = testdir.runpytest(sub2)
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
sub2.ensure("__init__.py")
p = sub2.ensure("test_hello.py")
result = testdir.runpytest(p)
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
result = testdir.runpytest(sub1)
- assert result.ret != 0
+ assert result.ret == EXIT_USAGEERROR
def test_directory_skipped(self, testdir):
testdir.makeconftest("""
@@ -228,7 +230,7 @@ class TestGeneralUsage:
""")
testdir.makepyfile("def test_hello(): pass")
result = testdir.runpytest()
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
result.stdout.fnmatch_lines([
"*1 skipped*"
])
@@ -479,7 +481,7 @@ class TestInvocationVariants:
def test_invoke_with_path(self, tmpdir, capsys):
retcode = pytest.main(tmpdir)
- assert not retcode
+ assert retcode == EXIT_NOTESTSCOLLECTED
out, err = capsys.readouterr()
def test_invoke_plugin_api(self, testdir, capsys):
diff --git a/testing/python/collect.py b/testing/python/collect.py
index 029b0b693..6a302f291 100644
--- a/testing/python/collect.py
+++ b/testing/python/collect.py
@@ -1,6 +1,8 @@
import sys
from textwrap import dedent
import pytest, py
+from _pytest.main import EXIT_NOTESTSCOLLECTED
+
class TestModule:
def test_failing_import(self, testdir):
@@ -906,7 +908,7 @@ def test_unorderable_types(testdir):
""")
result = testdir.runpytest()
assert "TypeError" not in result.stdout.str()
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
def test_collect_functools_partial(testdir):
diff --git a/testing/test_assertrewrite.py b/testing/test_assertrewrite.py
index fbac2b9c1..544250ad5 100644
--- a/testing/test_assertrewrite.py
+++ b/testing/test_assertrewrite.py
@@ -12,6 +12,7 @@ if sys.platform.startswith("java"):
from _pytest.assertion import util
from _pytest.assertion.rewrite import rewrite_asserts, PYTEST_TAG
+from _pytest.main import EXIT_NOTESTSCOLLECTED
def setup_module(mod):
@@ -429,7 +430,7 @@ class TestRewriteOnImport:
import sys
sys.path.append(%r)
import test_gum.test_lizard""" % (z_fn,))
- assert testdir.runpytest().ret == 0
+ assert testdir.runpytest().ret == EXIT_NOTESTSCOLLECTED
def test_readonly(self, testdir):
sub = testdir.mkdir("testing")
@@ -497,7 +498,7 @@ def test_rewritten():
pkg = testdir.mkdir('a_package_without_init_py')
pkg.join('module.py').ensure()
testdir.makepyfile("import a_package_without_init_py.module")
- assert testdir.runpytest().ret == 0
+ assert testdir.runpytest().ret == EXIT_NOTESTSCOLLECTED
class TestAssertionRewriteHookDetails(object):
def test_loader_is_package_false_for_module(self, testdir):
diff --git a/testing/test_capture.py b/testing/test_capture.py
index 81238432a..539333525 100644
--- a/testing/test_capture.py
+++ b/testing/test_capture.py
@@ -10,6 +10,7 @@ import contextlib
from _pytest import capture
from _pytest.capture import CaptureManager
+from _pytest.main import EXIT_NOTESTSCOLLECTED
from py.builtin import print_
needsosdup = pytest.mark.xfail("not hasattr(os, 'dup')")
@@ -365,7 +366,7 @@ class TestLoggingInteraction:
""")
# make sure that logging is still captured in tests
result = testdir.runpytest_subprocess("-s", "-p", "no:capturelog")
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
result.stderr.fnmatch_lines([
"WARNING*hello435*",
])
diff --git a/testing/test_collection.py b/testing/test_collection.py
index a7cb8a8c4..749c5b7ce 100644
--- a/testing/test_collection.py
+++ b/testing/test_collection.py
@@ -1,6 +1,6 @@
import pytest, py
-from _pytest.main import Session
+from _pytest.main import Session, EXIT_NOTESTSCOLLECTED
class TestCollector:
def test_collect_versus_item(self):
@@ -247,10 +247,10 @@ class TestCustomConftests:
p = testdir.makepyfile("def test_hello(): pass")
result = testdir.runpytest(p)
assert result.ret == 0
- assert "1 passed" in result.stdout.str()
+ result.stdout.fnmatch_lines("*1 passed*")
result = testdir.runpytest()
- assert result.ret == 0
- assert "1 passed" not in result.stdout.str()
+ assert result.ret == EXIT_NOTESTSCOLLECTED
+ result.stdout.fnmatch_lines("*collected 0 items*")
def test_collectignore_exclude_on_option(self, testdir):
testdir.makeconftest("""
@@ -264,7 +264,7 @@ class TestCustomConftests:
testdir.mkdir("hello")
testdir.makepyfile(test_world="def test_hello(): pass")
result = testdir.runpytest()
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
assert "passed" not in result.stdout.str()
result = testdir.runpytest("--XX")
assert result.ret == 0
diff --git a/testing/test_config.py b/testing/test_config.py
index 490fa96d0..9d3f7632c 100644
--- a/testing/test_config.py
+++ b/testing/test_config.py
@@ -1,6 +1,7 @@
import py, pytest
from _pytest.config import getcfg, get_common_ancestor, determine_setup
+from _pytest.main import EXIT_NOTESTSCOLLECTED
class TestParseIni:
def test_getcfg_and_config(self, testdir, tmpdir):
@@ -343,7 +344,7 @@ def test_invalid_options_show_extra_information(testdir):
@pytest.mark.skipif("sys.platform == 'win32'")
def test_toolongargs_issue224(testdir):
result = testdir.runpytest("-m", "hello" * 500)
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
def test_notify_exception(testdir, capfd):
config = testdir.parseconfig()
diff --git a/testing/test_conftest.py b/testing/test_conftest.py
index 64fa6d5e5..6700502c4 100644
--- a/testing/test_conftest.py
+++ b/testing/test_conftest.py
@@ -1,6 +1,7 @@
from textwrap import dedent
import py, pytest
from _pytest.config import PytestPluginManager
+from _pytest.main import EXIT_NOTESTSCOLLECTED, EXIT_USAGEERROR
@pytest.fixture(scope="module", params=["global", "inpackage"])
@@ -166,7 +167,10 @@ def test_conftest_confcutdir(testdir):
def test_no_conftest(testdir):
testdir.makeconftest("assert 0")
result = testdir.runpytest("--noconftest")
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
+
+ result = testdir.runpytest()
+ assert result.ret == EXIT_USAGEERROR
def test_conftest_existing_resultlog(testdir):
x = testdir.mkdir("tests")
diff --git a/testing/test_helpconfig.py b/testing/test_helpconfig.py
index d9cb52bcb..9f8d87b7c 100644
--- a/testing/test_helpconfig.py
+++ b/testing/test_helpconfig.py
@@ -1,3 +1,4 @@
+from _pytest.main import EXIT_NOTESTSCOLLECTED
import pytest
def test_version(testdir, pytestconfig):
@@ -43,7 +44,7 @@ def test_hookvalidation_optional(testdir):
pass
""")
result = testdir.runpytest()
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
def test_traceconfig(testdir):
result = testdir.runpytest("--traceconfig")
@@ -54,14 +55,14 @@ def test_traceconfig(testdir):
def test_debug(testdir, monkeypatch):
result = testdir.runpytest_subprocess("--debug")
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
p = testdir.tmpdir.join("pytestdebug.log")
assert "pytest_sessionstart" in p.read()
def test_PYTEST_DEBUG(testdir, monkeypatch):
monkeypatch.setenv("PYTEST_DEBUG", "1")
result = testdir.runpytest_subprocess()
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
result.stderr.fnmatch_lines([
"*pytest_plugin_registered*",
"*manager*PluginManager*"
diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py
index f8acd1576..d8d0c17c7 100644
--- a/testing/test_junitxml.py
+++ b/testing/test_junitxml.py
@@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
from xml.dom import minidom
+from _pytest.main import EXIT_NOTESTSCOLLECTED
import py, sys, os
from _pytest.junitxml import LogXML
@@ -298,7 +299,7 @@ class TestPython:
def test_collect_skipped(self, testdir):
testdir.makepyfile("import pytest; pytest.skip('xyz')")
result, dom = runandparse(testdir)
- assert not result.ret
+ assert result.ret == EXIT_NOTESTSCOLLECTED
node = dom.getElementsByTagName("testsuite")[0]
assert_attr(node, skips=1, tests=0)
tnode = node.getElementsByTagName("testcase")[0]
diff --git a/testing/test_pluginmanager.py b/testing/test_pluginmanager.py
index 62ecc544f..92afba9bc 100644
--- a/testing/test_pluginmanager.py
+++ b/testing/test_pluginmanager.py
@@ -3,6 +3,7 @@ import py
import os
from _pytest.config import get_config, PytestPluginManager
+from _pytest.main import EXIT_NOTESTSCOLLECTED
@pytest.fixture
def pytestpm():
@@ -223,7 +224,7 @@ class TestPytestPluginManager:
p.copy(p.dirpath("skipping2.py"))
monkeypatch.setenv("PYTEST_PLUGINS", "skipping2")
result = testdir.runpytest("-rw", "-p", "skipping1", syspathinsert=True)
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
result.stdout.fnmatch_lines([
"WI1*skipped plugin*skipping1*hello*",
"WI1*skipped plugin*skipping2*hello*",
diff --git a/testing/test_runner.py b/testing/test_runner.py
index 167ddc57b..3641ab8ca 100644
--- a/testing/test_runner.py
+++ b/testing/test_runner.py
@@ -293,8 +293,8 @@ class TestExecutionForked(BaseFunctionalTests):
def getrunner(self):
# XXX re-arrange this test to live in pytest-xdist
- xplugin = pytest.importorskip("xdist.plugin")
- return xplugin.forked_run_report
+ boxed = pytest.importorskip("xdist.boxed")
+ return boxed.forked_run_report
def test_suicide(self, testdir):
reports = testdir.runitem("""
@@ -431,6 +431,27 @@ def test_pytest_fail_notrace(testdir):
])
assert 'def teardown_function' not in result.stdout.str()
+
+def test_pytest_no_tests_collected_exit_status(testdir):
+ result = testdir.runpytest()
+ result.stdout.fnmatch_lines('*collected 0 items*')
+ assert result.ret == main.EXIT_NOTESTSCOLLECTED
+
+ testdir.makepyfile(test_foo="""
+ def test_foo():
+ assert 1
+ """)
+ result = testdir.runpytest()
+ result.stdout.fnmatch_lines('*collected 1 items*')
+ result.stdout.fnmatch_lines('*1 passed*')
+ assert result.ret == main.EXIT_OK
+
+ result = testdir.runpytest('-k nonmatch')
+ result.stdout.fnmatch_lines('*collected 1 items*')
+ result.stdout.fnmatch_lines('*1 deselected*')
+ assert result.ret == main.EXIT_NOTESTSCOLLECTED
+
+
def test_exception_printing_skip():
try:
pytest.skip("hello")
diff --git a/testing/test_session.py b/testing/test_session.py
index 0ddb92ac1..76f804b4f 100644
--- a/testing/test_session.py
+++ b/testing/test_session.py
@@ -1,5 +1,7 @@
import pytest
+from _pytest.main import EXIT_NOTESTSCOLLECTED
+
class SessionTests:
def test_basic_testitem_events(self, testdir):
tfile = testdir.makepyfile("""
@@ -239,4 +241,4 @@ def test_sessionfinish_with_start(testdir):
""")
res = testdir.runpytest("--collect-only")
- assert res.ret == 0
+ assert res.ret == EXIT_NOTESTSCOLLECTED
diff --git a/testing/test_terminal.py b/testing/test_terminal.py
index 50c27509f..0493b850d 100644
--- a/testing/test_terminal.py
+++ b/testing/test_terminal.py
@@ -7,6 +7,7 @@ import py
import pluggy
import sys
+from _pytest.main import EXIT_NOTESTSCOLLECTED
from _pytest.terminal import TerminalReporter, repr_pythonversion, getreportopt
from _pytest.terminal import build_summary_stats_line, _plugin_nameversions
from _pytest import runner
@@ -596,7 +597,7 @@ def test_traceconfig(testdir, monkeypatch):
result.stdout.fnmatch_lines([
"*active plugins*"
])
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
class TestGenericReporting:
diff --git a/testing/test_unittest.py b/testing/test_unittest.py
index b9ce7b5fa..aa055f89c 100644
--- a/testing/test_unittest.py
+++ b/testing/test_unittest.py
@@ -1,3 +1,4 @@
+from _pytest.main import EXIT_NOTESTSCOLLECTED
import pytest
def test_simple_unittest(testdir):
@@ -41,7 +42,7 @@ def test_isclasscheck_issue53(testdir):
E = _E()
""")
result = testdir.runpytest(testpath)
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
def test_setup(testdir):
testpath = testdir.makepyfile("""
@@ -572,7 +573,7 @@ def test_unorderable_types(testdir):
""")
result = testdir.runpytest()
assert "TypeError" not in result.stdout.str()
- assert result.ret == 0
+ assert result.ret == EXIT_NOTESTSCOLLECTED
def test_unittest_typerror_traceback(testdir):
testdir.makepyfile("""
diff --git a/tox.ini b/tox.ini
index c5dfa03f9..73ecdfb45 100644
--- a/tox.ini
+++ b/tox.ini
@@ -22,7 +22,7 @@ deps=
[testenv:py27-subprocess]
changedir=.
basepython=python2.7
-deps=pytest-xdist
+deps=pytest-xdist>=1.13
mock
nose
commands=
@@ -37,7 +37,7 @@ deps = pytest-flakes>=0.2
commands = py.test --flakes -m flakes _pytest testing
[testenv:py27-xdist]
-deps=pytest-xdist
+deps=pytest-xdist>=1.13
mock
nose
commands=
@@ -63,7 +63,7 @@ commands=
py.test -rfsxX test_pdb.py test_terminal.py test_unittest.py
[testenv:py27-nobyte]
-deps=pytest-xdist
+deps=pytest-xdist>=1.13
distribute=true
setenv=
PYTHONDONTWRITEBYTECODE=1