diff --git a/AUTHORS b/AUTHORS index 2fdc39fc2..7a4674388 100644 --- a/AUTHORS +++ b/AUTHORS @@ -15,6 +15,7 @@ Bob Ippolito Brian Dorsey Brian Okken Brianna Laugher +Bruno Oliveira Carl Friedrich Bolz Charles Cloud Chris Lamb diff --git a/CHANGELOG b/CHANGELOG index 7d96418c8..c375c9141 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -58,6 +58,15 @@ - Summary bar now is colored yellow for warning situations such as: all tests either were skipped or xpass/xfailed, or no tests were run at all (this is a partial fix for issue500). +- fix issue812: pytest now exits with status code 5 in situations where no + tests were run at all, such as the directory given in the command line does + not contain any tests or as result of a command line option filters + all out all tests (-k for example). + Thanks Eric Siegerman (issue812) and Bruno Oliveira for the PR. + +- Summary bar now is colored yellow for warning + situations such as: all tests either were skipped or xpass/xfailed, + or no tests were run at all (related to issue500). Thanks Eric Siegerman. - New `testpaths` ini option: list of directories to search for tests diff --git a/_pytest/main.py b/_pytest/main.py index fc9d64cf6..f9f3584c8 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -19,6 +19,7 @@ EXIT_TESTSFAILED = 1 EXIT_INTERRUPTED = 2 EXIT_INTERNALERROR = 3 EXIT_USAGEERROR = 4 +EXIT_NOTESTSCOLLECTED = 5 name_re = re.compile("^[a-zA-Z_]\w*$") @@ -102,6 +103,8 @@ def wrap_session(config, doit): else: if session._testsfailed: session.exitstatus = EXIT_TESTSFAILED + elif session._testscollected == 0: + session.exitstatus = EXIT_NOTESTSCOLLECTED finally: excinfo = None # Explicitly break reference cycle. session.startdir.chdir() @@ -510,6 +513,7 @@ class Session(FSCollector): config=config, session=self) self._fs2hookproxy = {} self._testsfailed = 0 + self._testscollected = 0 self.shouldstop = False self.trace = config.trace.root.get("collection") self._norecursepatterns = config.getini("norecursedirs") @@ -564,6 +568,7 @@ class Session(FSCollector): config=self.config, items=items) finally: hook.pytest_collection_finish(session=self) + self._testscollected = len(items) return items def _perform_collect(self, args, genitems): diff --git a/_pytest/terminal.py b/_pytest/terminal.py index 5365b4300..7c51d0835 100644 --- a/_pytest/terminal.py +++ b/_pytest/terminal.py @@ -2,6 +2,8 @@ This is a good source for looking at the various reporting hooks. """ +from _pytest.main import EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, \ + EXIT_USAGEERROR, EXIT_NOTESTSCOLLECTED import pytest import pluggy import py @@ -359,12 +361,15 @@ class TerminalReporter: outcome = yield outcome.get_result() self._tw.line("") - if exitstatus in (0, 1, 2, 4): + summary_exit_codes = ( + EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, EXIT_USAGEERROR, + EXIT_NOTESTSCOLLECTED) + if exitstatus in summary_exit_codes: self.summary_errors() self.summary_failures() self.summary_warnings() self.config.hook.pytest_terminal_summary(terminalreporter=self) - if exitstatus == 2: + if exitstatus == EXIT_INTERRUPTED: self._report_keyboardinterrupt() del self._keyboardinterrupt_memo self.summary_deselected() diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 2e87ac466..b9a3fa381 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -1,5 +1,7 @@ import sys import py, pytest +from _pytest.main import EXIT_NOTESTSCOLLECTED, EXIT_USAGEERROR + class TestGeneralUsage: def test_config_error(self, testdir): @@ -147,7 +149,7 @@ class TestGeneralUsage: pytest.skip("early") """) result = testdir.runpytest() - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED result.stdout.fnmatch_lines([ "*1 skip*" ]) @@ -177,7 +179,7 @@ class TestGeneralUsage: sys.stderr.write("stder42\\n") """) result = testdir.runpytest() - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED assert "should not be seen" not in result.stdout.str() assert "stderr42" not in result.stderr.str() @@ -212,13 +214,13 @@ class TestGeneralUsage: sub2 = testdir.tmpdir.mkdir("sub2") sub1.join("conftest.py").write("assert 0") result = testdir.runpytest(sub2) - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED sub2.ensure("__init__.py") p = sub2.ensure("test_hello.py") result = testdir.runpytest(p) - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED result = testdir.runpytest(sub1) - assert result.ret != 0 + assert result.ret == EXIT_USAGEERROR def test_directory_skipped(self, testdir): testdir.makeconftest(""" @@ -228,7 +230,7 @@ class TestGeneralUsage: """) testdir.makepyfile("def test_hello(): pass") result = testdir.runpytest() - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED result.stdout.fnmatch_lines([ "*1 skipped*" ]) @@ -479,7 +481,7 @@ class TestInvocationVariants: def test_invoke_with_path(self, tmpdir, capsys): retcode = pytest.main(tmpdir) - assert not retcode + assert retcode == EXIT_NOTESTSCOLLECTED out, err = capsys.readouterr() def test_invoke_plugin_api(self, testdir, capsys): diff --git a/testing/python/collect.py b/testing/python/collect.py index 029b0b693..6a302f291 100644 --- a/testing/python/collect.py +++ b/testing/python/collect.py @@ -1,6 +1,8 @@ import sys from textwrap import dedent import pytest, py +from _pytest.main import EXIT_NOTESTSCOLLECTED + class TestModule: def test_failing_import(self, testdir): @@ -906,7 +908,7 @@ def test_unorderable_types(testdir): """) result = testdir.runpytest() assert "TypeError" not in result.stdout.str() - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED def test_collect_functools_partial(testdir): diff --git a/testing/test_assertrewrite.py b/testing/test_assertrewrite.py index fbac2b9c1..544250ad5 100644 --- a/testing/test_assertrewrite.py +++ b/testing/test_assertrewrite.py @@ -12,6 +12,7 @@ if sys.platform.startswith("java"): from _pytest.assertion import util from _pytest.assertion.rewrite import rewrite_asserts, PYTEST_TAG +from _pytest.main import EXIT_NOTESTSCOLLECTED def setup_module(mod): @@ -429,7 +430,7 @@ class TestRewriteOnImport: import sys sys.path.append(%r) import test_gum.test_lizard""" % (z_fn,)) - assert testdir.runpytest().ret == 0 + assert testdir.runpytest().ret == EXIT_NOTESTSCOLLECTED def test_readonly(self, testdir): sub = testdir.mkdir("testing") @@ -497,7 +498,7 @@ def test_rewritten(): pkg = testdir.mkdir('a_package_without_init_py') pkg.join('module.py').ensure() testdir.makepyfile("import a_package_without_init_py.module") - assert testdir.runpytest().ret == 0 + assert testdir.runpytest().ret == EXIT_NOTESTSCOLLECTED class TestAssertionRewriteHookDetails(object): def test_loader_is_package_false_for_module(self, testdir): diff --git a/testing/test_capture.py b/testing/test_capture.py index 81238432a..539333525 100644 --- a/testing/test_capture.py +++ b/testing/test_capture.py @@ -10,6 +10,7 @@ import contextlib from _pytest import capture from _pytest.capture import CaptureManager +from _pytest.main import EXIT_NOTESTSCOLLECTED from py.builtin import print_ needsosdup = pytest.mark.xfail("not hasattr(os, 'dup')") @@ -365,7 +366,7 @@ class TestLoggingInteraction: """) # make sure that logging is still captured in tests result = testdir.runpytest_subprocess("-s", "-p", "no:capturelog") - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED result.stderr.fnmatch_lines([ "WARNING*hello435*", ]) diff --git a/testing/test_collection.py b/testing/test_collection.py index a7cb8a8c4..749c5b7ce 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -1,6 +1,6 @@ import pytest, py -from _pytest.main import Session +from _pytest.main import Session, EXIT_NOTESTSCOLLECTED class TestCollector: def test_collect_versus_item(self): @@ -247,10 +247,10 @@ class TestCustomConftests: p = testdir.makepyfile("def test_hello(): pass") result = testdir.runpytest(p) assert result.ret == 0 - assert "1 passed" in result.stdout.str() + result.stdout.fnmatch_lines("*1 passed*") result = testdir.runpytest() - assert result.ret == 0 - assert "1 passed" not in result.stdout.str() + assert result.ret == EXIT_NOTESTSCOLLECTED + result.stdout.fnmatch_lines("*collected 0 items*") def test_collectignore_exclude_on_option(self, testdir): testdir.makeconftest(""" @@ -264,7 +264,7 @@ class TestCustomConftests: testdir.mkdir("hello") testdir.makepyfile(test_world="def test_hello(): pass") result = testdir.runpytest() - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED assert "passed" not in result.stdout.str() result = testdir.runpytest("--XX") assert result.ret == 0 diff --git a/testing/test_config.py b/testing/test_config.py index 490fa96d0..9d3f7632c 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -1,6 +1,7 @@ import py, pytest from _pytest.config import getcfg, get_common_ancestor, determine_setup +from _pytest.main import EXIT_NOTESTSCOLLECTED class TestParseIni: def test_getcfg_and_config(self, testdir, tmpdir): @@ -343,7 +344,7 @@ def test_invalid_options_show_extra_information(testdir): @pytest.mark.skipif("sys.platform == 'win32'") def test_toolongargs_issue224(testdir): result = testdir.runpytest("-m", "hello" * 500) - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED def test_notify_exception(testdir, capfd): config = testdir.parseconfig() diff --git a/testing/test_conftest.py b/testing/test_conftest.py index 64fa6d5e5..6700502c4 100644 --- a/testing/test_conftest.py +++ b/testing/test_conftest.py @@ -1,6 +1,7 @@ from textwrap import dedent import py, pytest from _pytest.config import PytestPluginManager +from _pytest.main import EXIT_NOTESTSCOLLECTED, EXIT_USAGEERROR @pytest.fixture(scope="module", params=["global", "inpackage"]) @@ -166,7 +167,10 @@ def test_conftest_confcutdir(testdir): def test_no_conftest(testdir): testdir.makeconftest("assert 0") result = testdir.runpytest("--noconftest") - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED + + result = testdir.runpytest() + assert result.ret == EXIT_USAGEERROR def test_conftest_existing_resultlog(testdir): x = testdir.mkdir("tests") diff --git a/testing/test_helpconfig.py b/testing/test_helpconfig.py index d9cb52bcb..9f8d87b7c 100644 --- a/testing/test_helpconfig.py +++ b/testing/test_helpconfig.py @@ -1,3 +1,4 @@ +from _pytest.main import EXIT_NOTESTSCOLLECTED import pytest def test_version(testdir, pytestconfig): @@ -43,7 +44,7 @@ def test_hookvalidation_optional(testdir): pass """) result = testdir.runpytest() - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED def test_traceconfig(testdir): result = testdir.runpytest("--traceconfig") @@ -54,14 +55,14 @@ def test_traceconfig(testdir): def test_debug(testdir, monkeypatch): result = testdir.runpytest_subprocess("--debug") - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED p = testdir.tmpdir.join("pytestdebug.log") assert "pytest_sessionstart" in p.read() def test_PYTEST_DEBUG(testdir, monkeypatch): monkeypatch.setenv("PYTEST_DEBUG", "1") result = testdir.runpytest_subprocess() - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED result.stderr.fnmatch_lines([ "*pytest_plugin_registered*", "*manager*PluginManager*" diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index f8acd1576..d8d0c17c7 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from xml.dom import minidom +from _pytest.main import EXIT_NOTESTSCOLLECTED import py, sys, os from _pytest.junitxml import LogXML @@ -298,7 +299,7 @@ class TestPython: def test_collect_skipped(self, testdir): testdir.makepyfile("import pytest; pytest.skip('xyz')") result, dom = runandparse(testdir) - assert not result.ret + assert result.ret == EXIT_NOTESTSCOLLECTED node = dom.getElementsByTagName("testsuite")[0] assert_attr(node, skips=1, tests=0) tnode = node.getElementsByTagName("testcase")[0] diff --git a/testing/test_pluginmanager.py b/testing/test_pluginmanager.py index 62ecc544f..92afba9bc 100644 --- a/testing/test_pluginmanager.py +++ b/testing/test_pluginmanager.py @@ -3,6 +3,7 @@ import py import os from _pytest.config import get_config, PytestPluginManager +from _pytest.main import EXIT_NOTESTSCOLLECTED @pytest.fixture def pytestpm(): @@ -223,7 +224,7 @@ class TestPytestPluginManager: p.copy(p.dirpath("skipping2.py")) monkeypatch.setenv("PYTEST_PLUGINS", "skipping2") result = testdir.runpytest("-rw", "-p", "skipping1", syspathinsert=True) - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED result.stdout.fnmatch_lines([ "WI1*skipped plugin*skipping1*hello*", "WI1*skipped plugin*skipping2*hello*", diff --git a/testing/test_runner.py b/testing/test_runner.py index 167ddc57b..0245ff627 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -431,6 +431,27 @@ def test_pytest_fail_notrace(testdir): ]) assert 'def teardown_function' not in result.stdout.str() + +def test_pytest_no_tests_collected_exit_status(testdir): + result = testdir.runpytest() + result.stdout.fnmatch_lines('*collected 0 items*') + assert result.ret == main.EXIT_NOTESTSCOLLECTED + + testdir.makepyfile(test_foo=""" + def test_foo(): + assert 1 + """) + result = testdir.runpytest() + result.stdout.fnmatch_lines('*collected 1 items*') + result.stdout.fnmatch_lines('*1 passed*') + assert result.ret == main.EXIT_OK + + result = testdir.runpytest('-k nonmatch') + result.stdout.fnmatch_lines('*collected 1 items*') + result.stdout.fnmatch_lines('*1 deselected*') + assert result.ret == main.EXIT_NOTESTSCOLLECTED + + def test_exception_printing_skip(): try: pytest.skip("hello") diff --git a/testing/test_session.py b/testing/test_session.py index 0ddb92ac1..76f804b4f 100644 --- a/testing/test_session.py +++ b/testing/test_session.py @@ -1,5 +1,7 @@ import pytest +from _pytest.main import EXIT_NOTESTSCOLLECTED + class SessionTests: def test_basic_testitem_events(self, testdir): tfile = testdir.makepyfile(""" @@ -239,4 +241,4 @@ def test_sessionfinish_with_start(testdir): """) res = testdir.runpytest("--collect-only") - assert res.ret == 0 + assert res.ret == EXIT_NOTESTSCOLLECTED diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 7ad74a921..de7f2292d 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -6,6 +6,7 @@ import py import pluggy import sys +from _pytest.main import EXIT_NOTESTSCOLLECTED from _pytest.terminal import TerminalReporter, repr_pythonversion, getreportopt from _pytest.terminal import build_summary_stats_line from _pytest import runner @@ -577,7 +578,7 @@ def test_traceconfig(testdir, monkeypatch): result.stdout.fnmatch_lines([ "*active plugins*" ]) - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED class TestGenericReporting: diff --git a/testing/test_unittest.py b/testing/test_unittest.py index b9ce7b5fa..aa055f89c 100644 --- a/testing/test_unittest.py +++ b/testing/test_unittest.py @@ -1,3 +1,4 @@ +from _pytest.main import EXIT_NOTESTSCOLLECTED import pytest def test_simple_unittest(testdir): @@ -41,7 +42,7 @@ def test_isclasscheck_issue53(testdir): E = _E() """) result = testdir.runpytest(testpath) - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED def test_setup(testdir): testpath = testdir.makepyfile(""" @@ -572,7 +573,7 @@ def test_unorderable_types(testdir): """) result = testdir.runpytest() assert "TypeError" not in result.stdout.str() - assert result.ret == 0 + assert result.ret == EXIT_NOTESTSCOLLECTED def test_unittest_typerror_traceback(testdir): testdir.makepyfile("""