introduce new "Error" outcome and group setup/teardown and collection failures into that category. Report them separately.

--HG--
branch : 1.0.x

CHANGELOG

@@ -4,6 +4,10 @@ Changes between 1.0.0b8 and 1.0.0b9
 * fix svn-1.6 compat issue with py.path.svnwc().versioned()
   (thanks Wouter Vanden Hove)
 
+* setup/teardown or collection problems now show as ERRORs
+  or with big "E"'s in the progress lines.  they are reported
+  and counted separately. 
+ 
 * dist-testing: properly handle test items that get locally 
   collected but cannot be collected on the remote side - often 
   due to platform/dependency reasons

py/test/plugin/pytest_runner.py

@@ -70,6 +70,15 @@ def pytest_runtest_makereport(item, call):
 def pytest_runtest_teardown(item):
     item.config._setupstate.teardown_exact(item)
 
+def pytest_report_teststatus(rep):
+    if rep.when in ("setup", "teardown"):
+        if rep.failed:
+            #      category, shortletter, verbose-word 
+            return "error", "E", "ERROR"
+        elif rep.skipped:
+            return "skipped", "s", "SKIPPED"
+        else:
+            return "", "", ""
 #
 # Implementation
 

py/test/plugin/pytest_terminal.py

@@ -167,10 +167,11 @@ class TerminalReporter:
                 self.write_fspath_result(fspath, "")
 
     def pytest_runtest_logreport(self, rep):
-        if rep.passed and rep.when in ("setup", "teardown"):
-            return 
         fspath = rep.item.fspath 
         cat, letter, word = self.getcategoryletterword(rep)
+        if not letter and not word:
+            # probably passed setup/teardown
+            return
         if isinstance(word, tuple):
             word, markup = word
         else:
@@ -194,9 +195,9 @@ class TerminalReporter:
     def pytest_collectreport(self, rep):
         if not rep.passed:
             if rep.failed:
-                self.stats.setdefault("failed", []).append(rep)
+                self.stats.setdefault("error", []).append(rep)
                 msg = rep.longrepr.reprcrash.message 
-                self.write_fspath_result(rep.collector.fspath, "F")
+                self.write_fspath_result(rep.collector.fspath, "E")
             elif rep.skipped:
                 self.stats.setdefault("skipped", []).append(rep)
                 self.write_fspath_result(rep.collector.fspath, "S")
@@ -237,6 +238,7 @@ class TerminalReporter:
         __call__.execute() 
         self._tw.line("")
         if exitstatus in (0, 1, 2):
+            self.summary_errors()
             self.summary_failures()
             self.summary_skips()
             self.config.hook.pytest_terminal_summary(terminalreporter=self)
@@ -312,17 +314,39 @@ class TerminalReporter:
             for rep in self.stats['failed']:
                 msg = self._getfailureheadline(rep)
                 self.write_sep("_", msg)
-                if hasattr(rep, 'node'):
+                self.write_platinfo(rep)
-                    self.write_line(self.gateway2info.get(
-                        rep.node.gateway, 
-                        "node %r (platinfo not found? strange)")
-                            [:self._tw.fullwidth-1])
                 rep.toterminal(self._tw)
 
+    def summary_errors(self):
+        if 'error' in self.stats and self.config.option.tbstyle != "no":
+            self.write_sep("=", "ERRORS")
+            for rep in self.stats['error']:
+                msg = self._getfailureheadline(rep)
+                if not hasattr(rep, 'when'):
+                    # collect
+                    msg = "ERROR during collection " + msg
+                elif rep.when == "setup":
+                    msg = "ERROR at setup of " + msg 
+                elif rep.when == "teardown":
+                    msg = "ERROR at teardown of " + msg 
+                self.write_sep("_", msg)
+                self.write_platinfo(rep)
+                rep.toterminal(self._tw)
+
+    def write_platinfo(self, rep):
+        if hasattr(rep, 'node'):
+            self.write_line(self.gateway2info.get(
+                rep.node.gateway, 
+                "node %r (platinfo not found? strange)")
+                    [:self._tw.fullwidth-1])
+
     def summary_stats(self):
         session_duration = py.std.time.time() - self._sessionstarttime
 
         keys = "failed passed skipped deselected".split()
+        for key in self.stats.keys():
+            if key not in keys:
+                keys.append(key)
         parts = []
         for key in keys:
             val = self.stats.get(key, None)

py/test/plugin/test_pytest_iocapture.py

@@ -222,9 +222,9 @@ class TestLoggingInteraction:
             result = testdir.runpytest(p, *optargs)
             s = result.stdout.str()
             result.stdout.fnmatch_lines([
+                "*WARN*hello3",  # errors show first!
                 "*WARN*hello1", 
                 "*WARN*hello2", 
-                "*WARN*hello3", 
             ])
             # verify proper termination
             assert "closed" not in s
@@ -286,7 +286,7 @@ class TestCaptureFuncarg:
         result = testdir.runpytest(p)
         assert result.stdout.fnmatch_lines([
             "*test_partial_setup_failure*",
-            "*1 failed*",
+            "*1 error*",
         ])
 
     def test_keyboardinterrupt_disables_capturing(self, testdir):        
@@ -304,60 +304,3 @@ class TestCaptureFuncarg:
 
 
 
-class TestFixtureReporting:
-    @py.test.mark.xfail
-    def test_setup_fixture_error(self, testdir):
-        p = testdir.makepyfile("""
-            def setup_function(function):
-                print "setup func"
-                assert 0
-            def test_nada():
-                pass
-        """)
-        result = testdir.runpytest()
-        result.stdout.fnmatch_lines([
-            "*FIXTURE ERROR at setup of test_nada*",
-            "*setup_function(function):*",
-            "*setup func*",
-            "*assert 0*",
-            "*0 passed*1 error*",
-        ])
-        assert result.ret != 0
-    
-    @py.test.mark.xfail
-    def test_teardown_fixture_error(self, testdir):
-        p = testdir.makepyfile("""
-            def test_nada():
-                pass
-            def teardown_function(function):
-                print "teardown func"
-                assert 0
-        """)
-        result = testdir.runpytest()
-        result.stdout.fnmatch_lines([
-            "*FIXTURE ERROR at teardown*", 
-            "*teardown_function(function):*",
-            "*teardown func*",
-            "*assert 0*",
-            "*1 passed*1 error*",
-        ])
-
-    @py.test.mark.xfail
-    def test_teardown_fixture_error_and_test_failure(self, testdir):
-        p = testdir.makepyfile("""
-            def test_fail():
-                assert 0, "failingfunc"
-
-            def teardown_function(function):
-                print "teardown func"
-                assert 0
-        """)
-        result = testdir.runpytest()
-        result.stdout.fnmatch_lines([
-            "*failingfunc*", 
-            "*FIXTURE ERROR at teardown*", 
-            "*teardown_function(function):*",
-            "*teardown func*",
-            "*assert 0*",
-            "*1 failed*1 error",
-         ])

py/test/plugin/test_pytest_terminal.py

@@ -89,9 +89,10 @@ class TestTerminal:
         p = testdir.makepyfile("import xyz")
         result = testdir.runpytest(*option._getcmdargs())
         result.stdout.fnmatch_lines([
-            "*test_collect_fail.py F*",
+            "*test_collect_fail.py E*",
             ">   import xyz",
             "E   ImportError: No module named xyz",
+            "*1 error*",
         ])
 
     def test_internalerror(self, testdir, linecomp):
@@ -357,3 +358,62 @@ def test_repr_python_version(monkeypatch):
     py.std.sys.version_info = x = (2,3)
     assert repr_pythonversion() == str(x) 
 
+class TestFixtureReporting:
+    def test_setup_fixture_error(self, testdir):
+        p = testdir.makepyfile("""
+            def setup_function(function):
+                print "setup func"
+                assert 0
+            def test_nada():
+                pass
+        """)
+        result = testdir.runpytest()
+        result.stdout.fnmatch_lines([
+            "*ERROR at setup of test_nada*",
+            "*setup_function(function):*",
+            "*setup func*",
+            "*assert 0*",
+            "*1 error*",
+        ])
+        assert result.ret != 0
+    
+    def test_teardown_fixture_error(self, testdir):
+        p = testdir.makepyfile("""
+            def test_nada():
+                pass
+            def teardown_function(function):
+                print "teardown func"
+                assert 0
+        """)
+        result = testdir.runpytest()
+        result.stdout.fnmatch_lines([
+            "*ERROR at teardown*", 
+            "*teardown_function(function):*",
+            "*assert 0*",
+            "*Captured stdout*",
+            "*teardown func*",
+            "*1 passed*1 error*",
+        ])
+
+    def test_teardown_fixture_error_and_test_failure(self, testdir):
+        p = testdir.makepyfile("""
+            def test_fail():
+                assert 0, "failingfunc"
+
+            def teardown_function(function):
+                print "teardown func"
+                assert False
+        """)
+        result = testdir.runpytest()
+        result.stdout.fnmatch_lines([
+            "*ERROR at teardown of test_fail*", 
+            "*teardown_function(function):*",
+            "*assert False*",
+            "*Captured stdout*",
+            "*teardown func*",
+
+            "*test_fail*", 
+            "*def test_fail():",
+            "*failingfunc*",
+            "*1 failed*1 error*",
+         ])

py/test/testing/test_funcargs.py

@@ -194,7 +194,7 @@ class TestRequest:
         """)
         result = testdir.runpytest(p)
         assert result.stdout.fnmatch_lines([
-            "*1 failed*1 passed*"
+            "*1 passed*1 error*"
             ])
 
     def test_request_getmodulepath(self, testdir):