diff --git a/CHANGELOG b/CHANGELOG index 9368d7a95..fe8910b5d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,6 +4,10 @@ Changes between 1.0.0b8 and 1.0.0b9 * fix svn-1.6 compat issue with py.path.svnwc().versioned() (thanks Wouter Vanden Hove) +* setup/teardown or collection problems now show as ERRORs + or with big "E"'s in the progress lines. they are reported + and counted separately. + * dist-testing: properly handle test items that get locally collected but cannot be collected on the remote side - often due to platform/dependency reasons diff --git a/py/test/plugin/pytest_runner.py b/py/test/plugin/pytest_runner.py index 96d34c420..6b8ed913a 100644 --- a/py/test/plugin/pytest_runner.py +++ b/py/test/plugin/pytest_runner.py @@ -70,6 +70,15 @@ def pytest_runtest_makereport(item, call): def pytest_runtest_teardown(item): item.config._setupstate.teardown_exact(item) +def pytest_report_teststatus(rep): + if rep.when in ("setup", "teardown"): + if rep.failed: + # category, shortletter, verbose-word + return "error", "E", "ERROR" + elif rep.skipped: + return "skipped", "s", "SKIPPED" + else: + return "", "", "" # # Implementation diff --git a/py/test/plugin/pytest_terminal.py b/py/test/plugin/pytest_terminal.py index 97f2da8e2..9dcfcff3d 100644 --- a/py/test/plugin/pytest_terminal.py +++ b/py/test/plugin/pytest_terminal.py @@ -167,10 +167,11 @@ class TerminalReporter: self.write_fspath_result(fspath, "") def pytest_runtest_logreport(self, rep): - if rep.passed and rep.when in ("setup", "teardown"): - return fspath = rep.item.fspath cat, letter, word = self.getcategoryletterword(rep) + if not letter and not word: + # probably passed setup/teardown + return if isinstance(word, tuple): word, markup = word else: @@ -194,9 +195,9 @@ class TerminalReporter: def pytest_collectreport(self, rep): if not rep.passed: if rep.failed: - self.stats.setdefault("failed", []).append(rep) + self.stats.setdefault("error", []).append(rep) msg = rep.longrepr.reprcrash.message - self.write_fspath_result(rep.collector.fspath, "F") + self.write_fspath_result(rep.collector.fspath, "E") elif rep.skipped: self.stats.setdefault("skipped", []).append(rep) self.write_fspath_result(rep.collector.fspath, "S") @@ -237,6 +238,7 @@ class TerminalReporter: __call__.execute() self._tw.line("") if exitstatus in (0, 1, 2): + self.summary_errors() self.summary_failures() self.summary_skips() self.config.hook.pytest_terminal_summary(terminalreporter=self) @@ -312,17 +314,39 @@ class TerminalReporter: for rep in self.stats['failed']: msg = self._getfailureheadline(rep) self.write_sep("_", msg) - if hasattr(rep, 'node'): - self.write_line(self.gateway2info.get( - rep.node.gateway, - "node %r (platinfo not found? strange)") - [:self._tw.fullwidth-1]) + self.write_platinfo(rep) rep.toterminal(self._tw) + def summary_errors(self): + if 'error' in self.stats and self.config.option.tbstyle != "no": + self.write_sep("=", "ERRORS") + for rep in self.stats['error']: + msg = self._getfailureheadline(rep) + if not hasattr(rep, 'when'): + # collect + msg = "ERROR during collection " + msg + elif rep.when == "setup": + msg = "ERROR at setup of " + msg + elif rep.when == "teardown": + msg = "ERROR at teardown of " + msg + self.write_sep("_", msg) + self.write_platinfo(rep) + rep.toterminal(self._tw) + + def write_platinfo(self, rep): + if hasattr(rep, 'node'): + self.write_line(self.gateway2info.get( + rep.node.gateway, + "node %r (platinfo not found? strange)") + [:self._tw.fullwidth-1]) + def summary_stats(self): session_duration = py.std.time.time() - self._sessionstarttime keys = "failed passed skipped deselected".split() + for key in self.stats.keys(): + if key not in keys: + keys.append(key) parts = [] for key in keys: val = self.stats.get(key, None) diff --git a/py/test/plugin/test_pytest_iocapture.py b/py/test/plugin/test_pytest_iocapture.py index 2890f071e..1a7b3ff0b 100644 --- a/py/test/plugin/test_pytest_iocapture.py +++ b/py/test/plugin/test_pytest_iocapture.py @@ -222,9 +222,9 @@ class TestLoggingInteraction: result = testdir.runpytest(p, *optargs) s = result.stdout.str() result.stdout.fnmatch_lines([ + "*WARN*hello3", # errors show first! "*WARN*hello1", "*WARN*hello2", - "*WARN*hello3", ]) # verify proper termination assert "closed" not in s @@ -286,7 +286,7 @@ class TestCaptureFuncarg: result = testdir.runpytest(p) assert result.stdout.fnmatch_lines([ "*test_partial_setup_failure*", - "*1 failed*", + "*1 error*", ]) def test_keyboardinterrupt_disables_capturing(self, testdir): @@ -304,60 +304,3 @@ class TestCaptureFuncarg: -class TestFixtureReporting: - @py.test.mark.xfail - def test_setup_fixture_error(self, testdir): - p = testdir.makepyfile(""" - def setup_function(function): - print "setup func" - assert 0 - def test_nada(): - pass - """) - result = testdir.runpytest() - result.stdout.fnmatch_lines([ - "*FIXTURE ERROR at setup of test_nada*", - "*setup_function(function):*", - "*setup func*", - "*assert 0*", - "*0 passed*1 error*", - ]) - assert result.ret != 0 - - @py.test.mark.xfail - def test_teardown_fixture_error(self, testdir): - p = testdir.makepyfile(""" - def test_nada(): - pass - def teardown_function(function): - print "teardown func" - assert 0 - """) - result = testdir.runpytest() - result.stdout.fnmatch_lines([ - "*FIXTURE ERROR at teardown*", - "*teardown_function(function):*", - "*teardown func*", - "*assert 0*", - "*1 passed*1 error*", - ]) - - @py.test.mark.xfail - def test_teardown_fixture_error_and_test_failure(self, testdir): - p = testdir.makepyfile(""" - def test_fail(): - assert 0, "failingfunc" - - def teardown_function(function): - print "teardown func" - assert 0 - """) - result = testdir.runpytest() - result.stdout.fnmatch_lines([ - "*failingfunc*", - "*FIXTURE ERROR at teardown*", - "*teardown_function(function):*", - "*teardown func*", - "*assert 0*", - "*1 failed*1 error", - ]) diff --git a/py/test/plugin/test_pytest_terminal.py b/py/test/plugin/test_pytest_terminal.py index fbe9272bd..47d1c7231 100644 --- a/py/test/plugin/test_pytest_terminal.py +++ b/py/test/plugin/test_pytest_terminal.py @@ -89,9 +89,10 @@ class TestTerminal: p = testdir.makepyfile("import xyz") result = testdir.runpytest(*option._getcmdargs()) result.stdout.fnmatch_lines([ - "*test_collect_fail.py F*", + "*test_collect_fail.py E*", "> import xyz", "E ImportError: No module named xyz", + "*1 error*", ]) def test_internalerror(self, testdir, linecomp): @@ -357,3 +358,62 @@ def test_repr_python_version(monkeypatch): py.std.sys.version_info = x = (2,3) assert repr_pythonversion() == str(x) +class TestFixtureReporting: + def test_setup_fixture_error(self, testdir): + p = testdir.makepyfile(""" + def setup_function(function): + print "setup func" + assert 0 + def test_nada(): + pass + """) + result = testdir.runpytest() + result.stdout.fnmatch_lines([ + "*ERROR at setup of test_nada*", + "*setup_function(function):*", + "*setup func*", + "*assert 0*", + "*1 error*", + ]) + assert result.ret != 0 + + def test_teardown_fixture_error(self, testdir): + p = testdir.makepyfile(""" + def test_nada(): + pass + def teardown_function(function): + print "teardown func" + assert 0 + """) + result = testdir.runpytest() + result.stdout.fnmatch_lines([ + "*ERROR at teardown*", + "*teardown_function(function):*", + "*assert 0*", + "*Captured stdout*", + "*teardown func*", + "*1 passed*1 error*", + ]) + + def test_teardown_fixture_error_and_test_failure(self, testdir): + p = testdir.makepyfile(""" + def test_fail(): + assert 0, "failingfunc" + + def teardown_function(function): + print "teardown func" + assert False + """) + result = testdir.runpytest() + result.stdout.fnmatch_lines([ + "*ERROR at teardown of test_fail*", + "*teardown_function(function):*", + "*assert False*", + "*Captured stdout*", + "*teardown func*", + + "*test_fail*", + "*def test_fail():", + "*failingfunc*", + "*1 failed*1 error*", + ]) diff --git a/py/test/testing/test_funcargs.py b/py/test/testing/test_funcargs.py index bb5e5556b..2b132058d 100644 --- a/py/test/testing/test_funcargs.py +++ b/py/test/testing/test_funcargs.py @@ -194,7 +194,7 @@ class TestRequest: """) result = testdir.runpytest(p) assert result.stdout.fnmatch_lines([ - "*1 failed*1 passed*" + "*1 passed*1 error*" ]) def test_request_getmodulepath(self, testdir):