Merge pull request #2236 from KKoukiou/junitxml-change-schema

Change junitxml.py to produce results that comply with Junitxml schema

CHANGELOG.rst

@@ -87,7 +87,10 @@ Bug Fixes
 3.0.7 (unreleased)
 =======================
 
-*
+* Change junitxml.py to produce reports that comply with Junitxml schema.
+  If the same test fails with failure in call and then errors in teardown
+  we split testcase element into two, one containing the error and the other
+  the failure. (`#2228`_) Thanks to `@kkoukiou`_ for the PR.
 
 *
 
@@ -95,6 +98,10 @@ Bug Fixes
 
 *
 
+.. _@kkoukiou: https://github.com/KKoukiou
+
+.. _#2228:  https://github.com/pytest-dev/pytest/issues/2228
+
 
 3.0.6 (2017-01-22)
 ==================

_pytest/junitxml.py

@@ -273,6 +273,9 @@ class LogXML(object):
         self.node_reporters = {}  # nodeid -> _NodeReporter
         self.node_reporters_ordered = []
         self.global_properties = []
+        # List of reports that failed on call but teardown is pending.
+        self.open_reports = []
+        self.cnt_double_fail_tests = 0
 
     def finalize(self, report):
         nodeid = getattr(report, 'nodeid', report)
@@ -332,14 +335,33 @@ class LogXML(object):
             -> teardown node2
             -> teardown node1
         """
+        close_report = None
         if report.passed:
             if report.when == "call":  # ignore setup/teardown
                 reporter = self._opentestcase(report)
                 reporter.append_pass(report)
         elif report.failed:
+            if report.when == "teardown":
+                # The following vars are needed when xdist plugin is used
+                report_wid = getattr(report, "worker_id", None)
+                report_ii = getattr(report, "item_index", None)
+                close_report = next(
+                    (rep for rep in self.open_reports
+                     if (rep.nodeid == report.nodeid and
+                         getattr(rep, "item_index", None) == report_ii and
+                         getattr(rep, "worker_id", None) == report_wid
+                         )
+                     ), None)
+                if close_report:
+                    # We need to open new testcase in case we have failure in
+                    # call and error in teardown in order to follow junit
+                    # schema
+                    self.finalize(close_report)
+                    self.cnt_double_fail_tests += 1
             reporter = self._opentestcase(report)
             if report.when == "call":
                 reporter.append_failure(report)
+                self.open_reports.append(report)
             else:
                 reporter.append_error(report)
         elif report.skipped:
@@ -348,6 +370,17 @@ class LogXML(object):
         self.update_testcase_duration(report)
         if report.when == "teardown":
             self.finalize(report)
+            report_wid = getattr(report, "worker_id", None)
+            report_ii = getattr(report, "item_index", None)
+            close_report = next(
+                (rep for rep in self.open_reports
+                 if (rep.nodeid == report.nodeid and
+                      getattr(rep, "item_index", None) == report_ii and
+                      getattr(rep, "worker_id", None) == report_wid
+                     )
+                 ), None)
+            if close_report:
+                self.open_reports.remove(close_report)
 
     def update_testcase_duration(self, report):
         """accumulates total duration for nodeid from given report and updates
@@ -380,8 +413,9 @@ class LogXML(object):
         suite_stop_time = time.time()
         suite_time_delta = suite_stop_time - self.suite_start_time
 
-        numtests = self.stats['passed'] + self.stats['failure'] + self.stats['skipped'] + self.stats['error']
-
+        numtests = (self.stats['passed'] + self.stats['failure'] +
+                    self.stats['skipped'] + self.stats['error'] -
+                    self.cnt_double_fail_tests)
         logfile.write('<?xml version="1.0" encoding="utf-8"?>')
 
         logfile.write(Junit.testsuite(

testing/test_junitxml.py

@@ -189,6 +189,29 @@ class TestPython(object):
         fnode.assert_attr(message="test teardown failure")
         assert "ValueError" in fnode.toxml()
 
+    def test_call_failure_teardown_error(self, testdir):
+        testdir.makepyfile("""
+            import pytest
+
+            @pytest.fixture
+            def arg():
+                yield
+                raise Exception("Teardown Exception")
+            def test_function(arg):
+                raise Exception("Call Exception")
+        """)
+        result, dom = runandparse(testdir)
+        assert result.ret
+        node = dom.find_first_by_tag("testsuite")
+        node.assert_attr(errors=1, failures=1, tests=1)
+        first, second = dom.find_by_tag("testcase")
+        if not first or not second or first == second:
+            assert 0
+        fnode = first.find_first_by_tag("failure")
+        fnode.assert_attr(message="Exception: Call Exception")
+        snode = second.find_first_by_tag("error")
+        snode.assert_attr(message="test teardown failure")
+
     def test_skip_contains_name_reason(self, testdir):
         testdir.makepyfile("""
             import pytest