p53209761
/
test_ok2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Validate xunit2 files against the schema 

Fix #5095
This commit is contained in:
Bruno Oliveira 2019-07-12 15:14:04 -03:00
parent f5fab2bfa1
commit ba76080b59
4 changed files with 340 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
changelog/5095.trivial.rst Normal file
View File

@ -0,0 +1,2 @@
XML files of the ``xunit2`` family are now validated against the schema by pytest's own test suite
to avoid future regressions.

5
setup.py
View File

@ -21,7 +21,6 @@ def main():
use_scm_version={"write_to": "src/_pytest/_version.py"},
setup_requires=["setuptools-scm", "setuptools>=40.0"],
package_dir={"": "src"},
# fmt: off
extras_require={
"testing": [
"argcomplete",
@ -29,9 +28,9 @@ def main():
"mock",
"nose",
"requests",
],
"xmlschema",
]
},
# fmt: on
install_requires=INSTALL_REQUIRES,
)

147
testing/example_scripts/junit-10.xsd Normal file
View File

@ -0,0 +1,147 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
The MIT License (MIT)
Copyright (c) 2014, Gregory Boissinot
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-->
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:simpleType name="SUREFIRE_TIME">
<xs:restriction base="xs:string">
<xs:pattern value="(([0-9]{0,3},)*[0-9]{3}|[0-9]{0,3})*(\.[0-9]{0,3})?"/>
</xs:restriction>
</xs:simpleType>
<xs:complexType name="rerunType" mixed="true"> <!-- mixed (XML contains text) to be compatible with version previous than 2.22.1 -->
<xs:sequence>
<xs:element name="stackTrace" type="xs:string" minOccurs="0" /> <!-- optional to be compatible with version previous than 2.22.1 -->
<xs:element name="system-out" type="xs:string" minOccurs="0" />
<xs:element name="system-err" type="xs:string" minOccurs="0" />
</xs:sequence>
<xs:attribute name="message" type="xs:string" />
<xs:attribute name="type" type="xs:string" use="required" />
</xs:complexType>
<xs:element name="failure">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string"/>
<xs:attribute name="message" type="xs:string"/>
</xs:complexType>
</xs:element>
<xs:element name="error">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string"/>
<xs:attribute name="message" type="xs:string"/>
</xs:complexType>
</xs:element>
<xs:element name="skipped">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string"/>
<xs:attribute name="message" type="xs:string"/>
</xs:complexType>
</xs:element>
<xs:element name="properties">
<xs:complexType>
<xs:sequence>
<xs:element ref="property" minOccurs="0" maxOccurs="unbounded"/>
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="property">
<xs:complexType>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="value" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>
<xs:element name="system-err" type="xs:string"/>
<xs:element name="system-out" type="xs:string"/>
<xs:element name="rerunFailure" type="rerunType"/>
<xs:element name="rerunError" type="rerunType"/>
<xs:element name="flakyFailure" type="rerunType"/>
<xs:element name="flakyError" type="rerunType"/>
<xs:element name="testcase">
<xs:complexType>
<xs:sequence>
<xs:choice minOccurs="0" maxOccurs="unbounded">
<xs:element ref="skipped"/>
<xs:element ref="error"/>
<xs:element ref="failure"/>
<xs:element ref="rerunFailure" minOccurs="0" maxOccurs="unbounded"/>
<xs:element ref="rerunError" minOccurs="0" maxOccurs="unbounded"/>
<xs:element ref="flakyFailure" minOccurs="0" maxOccurs="unbounded"/>
<xs:element ref="flakyError" minOccurs="0" maxOccurs="unbounded"/>
<xs:element ref="system-out"/>
<xs:element ref="system-err"/>
</xs:choice>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="time" type="xs:string"/>
<xs:attribute name="classname" type="xs:string"/>
<xs:attribute name="group" type="xs:string"/>
</xs:complexType>
</xs:element>
<xs:element name="testsuite">
<xs:complexType>
<xs:choice minOccurs="0" maxOccurs="unbounded">
<xs:element ref="testsuite"/>
<xs:element ref="properties"/>
<xs:element ref="testcase"/>
<xs:element ref="system-out"/>
<xs:element ref="system-err"/>
</xs:choice>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="tests" type="xs:string" use="required"/>
<xs:attribute name="failures" type="xs:string" use="required"/>
<xs:attribute name="errors" type="xs:string" use="required"/>
<xs:attribute name="group" type="xs:string" />
<xs:attribute name="time" type="SUREFIRE_TIME"/>
<xs:attribute name="skipped" type="xs:string" />
<xs:attribute name="timestamp" type="xs:string" />
<xs:attribute name="hostname" type="xs:string" />
<xs:attribute name="id" type="xs:string" />
<xs:attribute name="package" type="xs:string" />
<xs:attribute name="file" type="xs:string"/>
<xs:attribute name="log" type="xs:string"/>
<xs:attribute name="url" type="xs:string"/>
<xs:attribute name="version" type="xs:string"/>
</xs:complexType>
</xs:element>
<xs:element name="testsuites">
<xs:complexType>
<xs:sequence>
<xs:element ref="testsuite" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
<xs:attribute name="name" type="xs:string" />
<xs:attribute name="time" type="SUREFIRE_TIME"/>
<xs:attribute name="tests" type="xs:string" />
<xs:attribute name="failures" type="xs:string" />
<xs:attribute name="errors" type="xs:string" />
</xs:complexType>
</xs:element>
</xs:schema>

305
testing/test_junitxml.py
View File

@ -1,20 +1,47 @@
import os
import platform
from datetime import datetime
from pathlib import Path
from xml.dom import minidom
import py
import xmlschema
import pytest
from _pytest.junitxml import LogXML
from _pytest.reports import BaseReport
def runandparse(testdir, *args):
resultpath = testdir.tmpdir.join("junit.xml")
result = testdir.runpytest("--junitxml=%s" % resultpath, *args)
xmldoc = minidom.parse(str(resultpath))
return result, DomNode(xmldoc)
@pytest.fixture(scope="session")
def schema():
"""Returns a xmlschema.XMLSchema object for the junit-10.xsd file"""
fn = Path(__file__).parent / "example_scripts/junit-10.xsd"
with fn.open() as f:
return xmlschema.XMLSchema(f)
@pytest.fixture
def run_and_parse(testdir, schema):
"""
Fixture that returns a function that can be used to execute pytest and return
the parsed ``DomNode`` of the root xml node.
The ``family`` parameter is used to configure the ``junit_family`` of the written report.
"xunit2" is also automatically validated against the schema.
"""
def run(*args, family="xunit1"):
if family:
args = ("-o", "junit_family=" + family) + args
xml_path = testdir.tmpdir.join("junit.xml")
result = testdir.runpytest("--junitxml=%s" % xml_path, *args)
if family == "xunit2":
with xml_path.open() as f:
schema.validate(f)
xmldoc = minidom.parse(str(xml_path))
return result, DomNode(xmldoc)
return run
def assert_attr(node, **kwargs):
@ -91,8 +118,12 @@ class DomNode:
return type(self)(self.__node.nextSibling)
parametrize_families = pytest.mark.parametrize("xunit_family", ["xunit1", "xunit2"])
class TestPython:
def test_summing_simple(self, testdir):
@parametrize_families
def test_summing_simple(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
import pytest
@ -110,12 +141,13 @@ class TestPython:
assert 1
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(name="pytest", errors=0, failures=1, skipped=2, tests=5)
def test_summing_simple_with_errors(self, testdir):
@parametrize_families
def test_summing_simple_with_errors(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
import pytest
@ -136,23 +168,25 @@ class TestPython:
assert True
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(name="pytest", errors=1, failures=2, skipped=1, tests=5)
def test_hostname_in_xml(self, testdir):
@parametrize_families
def test_hostname_in_xml(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
def test_pass():
pass
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
node = dom.find_first_by_tag("testsuite")
node.assert_attr(hostname=platform.node())
def test_timestamp_in_xml(self, testdir):
@parametrize_families
def test_timestamp_in_xml(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
def test_pass():
@ -160,12 +194,12 @@ class TestPython:
"""
)
start_time = datetime.now()
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
node = dom.find_first_by_tag("testsuite")
timestamp = datetime.strptime(node["timestamp"], "%Y-%m-%dT%H:%M:%S.%f")
assert start_time <= timestamp < datetime.now()
def test_timing_function(self, testdir):
def test_timing_function(self, testdir, run_and_parse):
testdir.makepyfile(
"""
import time, pytest
@ -177,14 +211,16 @@ class TestPython:
time.sleep(0.01)
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse()
node = dom.find_first_by_tag("testsuite")
tnode = node.find_first_by_tag("testcase")
val = tnode["time"]
assert round(float(val), 2) >= 0.03
@pytest.mark.parametrize("duration_report", ["call", "total"])
def test_junit_duration_report(self, testdir, monkeypatch, duration_report):
def test_junit_duration_report(
self, testdir, monkeypatch, duration_report, run_and_parse
):
# mock LogXML.node_reporter so it always sets a known duration to each test report object
original_node_reporter = LogXML.node_reporter
@ -202,8 +238,8 @@ class TestPython:
pass
"""
)
result, dom = runandparse(
testdir, "-o", "junit_duration_report={}".format(duration_report)
result, dom = run_and_parse(
"-o", "junit_duration_report={}".format(duration_report)
)
node = dom.find_first_by_tag("testsuite")
tnode = node.find_first_by_tag("testcase")
@ -214,7 +250,8 @@ class TestPython:
assert duration_report == "call"
assert val == 1.0
def test_setup_error(self, testdir):
@parametrize_families
def test_setup_error(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
import pytest
@ -226,7 +263,7 @@ class TestPython:
pass
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(errors=1, tests=1)
@ -236,7 +273,8 @@ class TestPython:
fnode.assert_attr(message="test setup failure")
assert "ValueError" in fnode.toxml()
def test_teardown_error(self, testdir):
@parametrize_families
def test_teardown_error(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
import pytest
@ -249,7 +287,7 @@ class TestPython:
pass
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret
node = dom.find_first_by_tag("testsuite")
tnode = node.find_first_by_tag("testcase")
@ -258,7 +296,8 @@ class TestPython:
fnode.assert_attr(message="test teardown failure")
assert "ValueError" in fnode.toxml()
def test_call_failure_teardown_error(self, testdir):
@parametrize_families
def test_call_failure_teardown_error(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
import pytest
@ -271,7 +310,7 @@ class TestPython:
raise Exception("Call Exception")
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(errors=1, failures=1, tests=1)
@ -283,7 +322,8 @@ class TestPython:
snode = second.find_first_by_tag("error")
snode.assert_attr(message="test teardown failure")
def test_skip_contains_name_reason(self, testdir):
@parametrize_families
def test_skip_contains_name_reason(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
import pytest
@ -291,7 +331,7 @@ class TestPython:
pytest.skip("hello23")
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret == 0
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skipped=1)
@ -300,7 +340,8 @@ class TestPython:
snode = tnode.find_first_by_tag("skipped")
snode.assert_attr(type="pytest.skip", message="hello23")
def test_mark_skip_contains_name_reason(self, testdir):
@parametrize_families
def test_mark_skip_contains_name_reason(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
import pytest
@ -309,7 +350,7 @@ class TestPython:
assert True
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret == 0
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skipped=1)
@ -320,7 +361,10 @@ class TestPython:
snode = tnode.find_first_by_tag("skipped")
snode.assert_attr(type="pytest.skip", message="hello24")
def test_mark_skipif_contains_name_reason(self, testdir):
@parametrize_families
def test_mark_skipif_contains_name_reason(
self, testdir, run_and_parse, xunit_family
):
testdir.makepyfile(
"""
import pytest
@ -330,7 +374,7 @@ class TestPython:
assert True
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret == 0
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skipped=1)
@ -341,7 +385,10 @@ class TestPython:
snode = tnode.find_first_by_tag("skipped")
snode.assert_attr(type="pytest.skip", message="hello25")
def test_mark_skip_doesnt_capture_output(self, testdir):
@parametrize_families
def test_mark_skip_doesnt_capture_output(
self, testdir, run_and_parse, xunit_family
):
testdir.makepyfile(
"""
import pytest
@ -350,12 +397,13 @@ class TestPython:
print("bar!")
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret == 0
node_xml = dom.find_first_by_tag("testsuite").toxml()
assert "bar!" not in node_xml
def test_classname_instance(self, testdir):
@parametrize_families
def test_classname_instance(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
class TestClass(object):
@ -363,7 +411,7 @@ class TestPython:
assert 0
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(failures=1)
@ -372,20 +420,22 @@ class TestPython:
classname="test_classname_instance.TestClass", name="test_method"
)
def test_classname_nested_dir(self, testdir):
@parametrize_families
def test_classname_nested_dir(self, testdir, run_and_parse, xunit_family):
p = testdir.tmpdir.ensure("sub", "test_hello.py")
p.write("def test_func(): 0/0")
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(failures=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(classname="sub.test_hello", name="test_func")
def test_internal_error(self, testdir):
@parametrize_families
def test_internal_error(self, testdir, run_and_parse, xunit_family):
testdir.makeconftest("def pytest_runtest_protocol(): 0 / 0")
testdir.makepyfile("def test_function(): pass")
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(errors=1, tests=1)
@ -396,7 +446,10 @@ class TestPython:
assert "Division" in fnode.toxml()
@pytest.mark.parametrize("junit_logging", ["no", "system-out", "system-err"])
def test_failure_function(self, testdir, junit_logging):
@parametrize_families
def test_failure_function(
self, testdir, junit_logging, run_and_parse, xunit_family
):
testdir.makepyfile(
"""
import logging
@ -411,7 +464,9 @@ class TestPython:
"""
)
result, dom = runandparse(testdir, "-o", "junit_logging=%s" % junit_logging)
result, dom = run_and_parse(
"-o", "junit_logging=%s" % junit_logging, family=xunit_family
)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(failures=1, tests=1)
@ -439,7 +494,8 @@ class TestPython:
assert "warning msg" not in systemout.toxml()
assert "warning msg" not in systemerr.toxml()
def test_failure_verbose_message(self, testdir):
@parametrize_families
def test_failure_verbose_message(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
import sys
@ -447,14 +503,14 @@ class TestPython:
assert 0, "An error"
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
node = dom.find_first_by_tag("testsuite")
tnode = node.find_first_by_tag("testcase")
fnode = tnode.find_first_by_tag("failure")
fnode.assert_attr(message="AssertionError: An error assert 0")
def test_failure_escape(self, testdir):
@parametrize_families
def test_failure_escape(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
import pytest
@ -464,7 +520,7 @@ class TestPython:
assert 0
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(failures=3, tests=3)
@ -479,7 +535,8 @@ class TestPython:
text = sysout.text
assert text == "%s\n" % char
def test_junit_prefixing(self, testdir):
@parametrize_families
def test_junit_prefixing(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
def test_func():
@ -489,7 +546,7 @@ class TestPython:
pass
"""
)
result, dom = runandparse(testdir, "--junitprefix=xyz")
result, dom = run_and_parse("--junitprefix=xyz", family=xunit_family)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(failures=1, tests=2)
@ -500,7 +557,8 @@ class TestPython:
classname="xyz.test_junit_prefixing.TestHello", name="test_hello"
)
def test_xfailure_function(self, testdir):
@parametrize_families
def test_xfailure_function(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
import pytest
@ -508,7 +566,7 @@ class TestPython:
pytest.xfail("42")
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert not result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skipped=1, tests=1)
@ -516,9 +574,9 @@ class TestPython:
tnode.assert_attr(classname="test_xfailure_function", name="test_xfail")
fnode = tnode.find_first_by_tag("skipped")
fnode.assert_attr(type="pytest.xfail", message="42")
# assert "ValueError" in fnode.toxml()
def test_xfailure_marker(self, testdir):
@parametrize_families
def test_xfailure_marker(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
import pytest
@ -527,7 +585,7 @@ class TestPython:
assert False
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert not result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skipped=1, tests=1)
@ -536,7 +594,7 @@ class TestPython:
fnode = tnode.find_first_by_tag("skipped")
fnode.assert_attr(type="pytest.xfail", message="42")
def test_xfail_captures_output_once(self, testdir):
def test_xfail_captures_output_once(self, testdir, run_and_parse):
testdir.makepyfile(
"""
import sys
@ -549,13 +607,14 @@ class TestPython:
assert 0
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse()
node = dom.find_first_by_tag("testsuite")
tnode = node.find_first_by_tag("testcase")
assert len(tnode.find_by_tag("system-err")) == 1
assert len(tnode.find_by_tag("system-out")) == 1
def test_xfailure_xpass(self, testdir):
@parametrize_families
def test_xfailure_xpass(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
import pytest
@ -564,14 +623,15 @@ class TestPython:
pass
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
# assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skipped=0, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(classname="test_xfailure_xpass", name="test_xpass")
def test_xfailure_xpass_strict(self, testdir):
@parametrize_families
def test_xfailure_xpass_strict(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
import pytest
@ -580,7 +640,7 @@ class TestPython:
pass
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
# assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skipped=0, tests=1)
@ -589,9 +649,10 @@ class TestPython:
fnode = tnode.find_first_by_tag("failure")
fnode.assert_attr(message="[XPASS(strict)] This needs to fail!")
def test_collect_error(self, testdir):
@parametrize_families
def test_collect_error(self, testdir, run_and_parse, xunit_family):
testdir.makepyfile("syntax error")
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(errors=1, tests=1)
@ -600,7 +661,7 @@ class TestPython:
fnode.assert_attr(message="collection failure")
assert "SyntaxError" in fnode.toxml()
def test_unicode(self, testdir):
def test_unicode(self, testdir, run_and_parse):
value = "hx\xc4\x85\xc4\x87\n"
testdir.makepyfile(
"""\
@ -611,14 +672,14 @@ class TestPython:
"""
% value
)
result, dom = runandparse(testdir)
result, dom = run_and_parse()
assert result.ret == 1
tnode = dom.find_first_by_tag("testcase")
fnode = tnode.find_first_by_tag("failure")
assert "hx" in fnode.toxml()
def test_assertion_binchars(self, testdir):
"""this test did fail when the escaping wasn't strict"""
def test_assertion_binchars(self, testdir, run_and_parse):
"""this test did fail when the escaping wasnt strict"""
testdir.makepyfile(
"""
@ -629,23 +690,23 @@ class TestPython:
assert M1 == M2
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse()
print(dom.toxml())
def test_pass_captures_stdout(self, testdir):
def test_pass_captures_stdout(self, testdir, run_and_parse):
testdir.makepyfile(
"""
def test_pass():
print('hello-stdout')
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse()
node = dom.find_first_by_tag("testsuite")
pnode = node.find_first_by_tag("testcase")
systemout = pnode.find_first_by_tag("system-out")
assert "hello-stdout" in systemout.toxml()
def test_pass_captures_stderr(self, testdir):
def test_pass_captures_stderr(self, testdir, run_and_parse):
testdir.makepyfile(
"""
import sys
@ -653,13 +714,13 @@ class TestPython:
sys.stderr.write('hello-stderr')
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse()
node = dom.find_first_by_tag("testsuite")
pnode = node.find_first_by_tag("testcase")
systemout = pnode.find_first_by_tag("system-err")
assert "hello-stderr" in systemout.toxml()
def test_setup_error_captures_stdout(self, testdir):
def test_setup_error_captures_stdout(self, testdir, run_and_parse):
testdir.makepyfile(
"""
import pytest
@ -672,13 +733,13 @@ class TestPython:
pass
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse()
node = dom.find_first_by_tag("testsuite")
pnode = node.find_first_by_tag("testcase")
systemout = pnode.find_first_by_tag("system-out")
assert "hello-stdout" in systemout.toxml()
def test_setup_error_captures_stderr(self, testdir):
def test_setup_error_captures_stderr(self, testdir, run_and_parse):
testdir.makepyfile(
"""
import sys
@ -692,13 +753,13 @@ class TestPython:
pass
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse()
node = dom.find_first_by_tag("testsuite")
pnode = node.find_first_by_tag("testcase")
systemout = pnode.find_first_by_tag("system-err")
assert "hello-stderr" in systemout.toxml()
def test_avoid_double_stdout(self, testdir):
def test_avoid_double_stdout(self, testdir, run_and_parse):
testdir.makepyfile(
"""
import sys
@ -713,7 +774,7 @@ class TestPython:
sys.stdout.write('hello-stdout call')
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse()
node = dom.find_first_by_tag("testsuite")
pnode = node.find_first_by_tag("testcase")
systemout = pnode.find_first_by_tag("system-out")
@ -756,7 +817,8 @@ def test_dont_configure_on_slaves(tmpdir):
class TestNonPython:
def test_summing_simple(self, testdir):
@parametrize_families
def test_summing_simple(self, testdir, run_and_parse, xunit_family):
testdir.makeconftest(
"""
import pytest
@ -774,7 +836,7 @@ class TestNonPython:
"""
)
testdir.tmpdir.join("myfile.xyz").write("hello")
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(errors=0, failures=1, skipped=0, tests=1)
@ -822,8 +884,8 @@ def test_nullbyte_replace(testdir):
def test_invalid_xml_escape():
# Test some more invalid xml chars, the full range should be
# tested really but let's just thest the edges of the ranges
# intead.
# tested really but let's just test the edges of the ranges
# instead.
# XXX This only tests low unicode character points for now as
# there are some issues with the testing infrastructure for
# the higher ones.
@ -907,7 +969,7 @@ def test_logxml_check_isdir(testdir):
result.stderr.fnmatch_lines(["*--junitxml must be a filename*"])
def test_escaped_parametrized_names_xml(testdir):
def test_escaped_parametrized_names_xml(testdir, run_and_parse):
testdir.makepyfile(
"""\
import pytest
@ -916,13 +978,13 @@ def test_escaped_parametrized_names_xml(testdir):
assert char
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse()
assert result.ret == 0
node = dom.find_first_by_tag("testcase")
node.assert_attr(name="test_func[\\x00]")
def test_double_colon_split_function_issue469(testdir):
def test_double_colon_split_function_issue469(testdir, run_and_parse):
testdir.makepyfile(
"""
import pytest
@ -931,14 +993,14 @@ def test_double_colon_split_function_issue469(testdir):
pass
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse()
assert result.ret == 0
node = dom.find_first_by_tag("testcase")
node.assert_attr(classname="test_double_colon_split_function_issue469")
node.assert_attr(name="test_func[double::colon]")
def test_double_colon_split_method_issue469(testdir):
def test_double_colon_split_method_issue469(testdir, run_and_parse):
testdir.makepyfile(
"""
import pytest
@ -948,7 +1010,7 @@ def test_double_colon_split_method_issue469(testdir):
pass
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse()
assert result.ret == 0
node = dom.find_first_by_tag("testcase")
node.assert_attr(classname="test_double_colon_split_method_issue469.TestClass")
@ -984,7 +1046,7 @@ def test_unicode_issue368(testdir):
log.pytest_sessionfinish()
def test_record_property(testdir):
def test_record_property(testdir, run_and_parse):
testdir.makepyfile(
"""
import pytest
@ -996,7 +1058,7 @@ def test_record_property(testdir):
record_property("foo", "<1");
"""
)
result, dom = runandparse(testdir, "-rwv")
result, dom = run_and_parse("-rwv")
node = dom.find_first_by_tag("testsuite")
tnode = node.find_first_by_tag("testcase")
psnode = tnode.find_first_by_tag("properties")
@ -1005,7 +1067,7 @@ def test_record_property(testdir):
pnodes[1].assert_attr(name="foo", value="<1")
def test_record_property_same_name(testdir):
def test_record_property_same_name(testdir, run_and_parse):
testdir.makepyfile(
"""
def test_record_with_same_name(record_property):
@ -1013,7 +1075,7 @@ def test_record_property_same_name(testdir):
record_property("foo", "baz")
"""
)
result, dom = runandparse(testdir, "-rw")
result, dom = run_and_parse("-rw")
node = dom.find_first_by_tag("testsuite")
tnode = node.find_first_by_tag("testcase")
psnode = tnode.find_first_by_tag("properties")
@ -1037,7 +1099,7 @@ def test_record_fixtures_without_junitxml(testdir, fixture_name):
@pytest.mark.filterwarnings("default")
def test_record_attribute(testdir):
def test_record_attribute(testdir, run_and_parse):
testdir.makeini(
"""
[pytest]
@ -1055,7 +1117,7 @@ def test_record_attribute(testdir):
record_xml_attribute("foo", "<1");
"""
)
result, dom = runandparse(testdir, "-rw")
result, dom = run_and_parse("-rw")
node = dom.find_first_by_tag("testsuite")
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(bar="1")
@ -1067,7 +1129,7 @@ def test_record_attribute(testdir):
@pytest.mark.filterwarnings("default")
@pytest.mark.parametrize("fixture_name", ["record_xml_attribute", "record_property"])
def test_record_fixtures_xunit2(testdir, fixture_name):
def test_record_fixtures_xunit2(testdir, fixture_name, run_and_parse):
"""Ensure record_xml_attribute and record_property drop values when outside of legacy family
"""
testdir.makeini(
@ -1090,7 +1152,7 @@ def test_record_fixtures_xunit2(testdir, fixture_name):
)
)
result, dom = runandparse(testdir, "-rw")
result, dom = run_and_parse("-rw", family=None)
expected_lines = []
if fixture_name == "record_xml_attribute":
expected_lines.append(
@ -1105,7 +1167,7 @@ def test_record_fixtures_xunit2(testdir, fixture_name):
result.stdout.fnmatch_lines(expected_lines)
def test_random_report_log_xdist(testdir, monkeypatch):
def test_random_report_log_xdist(testdir, monkeypatch, run_and_parse):
"""xdist calls pytest_runtest_logreport as they are executed by the slaves,
with nodes from several nodes overlapping, so junitxml must cope with that
to produce correct reports. #1064
@ -1120,7 +1182,7 @@ def test_random_report_log_xdist(testdir, monkeypatch):
assert i != 22
"""
)
_, dom = runandparse(testdir, "-n2")
_, dom = run_and_parse("-n2")
suite_node = dom.find_first_by_tag("testsuite")
failed = []
for case_node in suite_node.find_by_tag("testcase"):
@ -1130,21 +1192,22 @@ def test_random_report_log_xdist(testdir, monkeypatch):
assert failed == ["test_x[22]"]
def test_root_testsuites_tag(testdir):
@parametrize_families
def test_root_testsuites_tag(testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
def test_x():
pass
"""
)
_, dom = runandparse(testdir)
_, dom = run_and_parse(family=xunit_family)
root = dom.get_unique_child
assert root.tag == "testsuites"
suite_node = root.get_unique_child
assert suite_node.tag == "testsuite"
def test_runs_twice(testdir):
def test_runs_twice(testdir, run_and_parse):
f = testdir.makepyfile(
"""
def test_pass():
@ -1152,14 +1215,14 @@ def test_runs_twice(testdir):
"""
)
result, dom = runandparse(testdir, f, f)
result, dom = run_and_parse(f, f)
assert "INTERNALERROR" not in result.stdout.str()
first, second = [x["classname"] for x in dom.find_by_tag("testcase")]
assert first == second
@pytest.mark.xfail(reason="hangs", run=False)
def test_runs_twice_xdist(testdir):
def test_runs_twice_xdist(testdir, run_and_parse):
pytest.importorskip("xdist")
f = testdir.makepyfile(
"""
@ -1168,13 +1231,13 @@ def test_runs_twice_xdist(testdir):
"""
)
result, dom = runandparse(testdir, f, "--dist", "each", "--tx", "2*popen")
result, dom = run_and_parse(f, "--dist", "each", "--tx", "2*popen")
assert "INTERNALERROR" not in result.stdout.str()
first, second = [x["classname"] for x in dom.find_by_tag("testcase")]
assert first == second
def test_fancy_items_regression(testdir):
def test_fancy_items_regression(testdir, run_and_parse):
# issue 1259
testdir.makeconftest(
"""
@ -1207,7 +1270,7 @@ def test_fancy_items_regression(testdir):
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse()
assert "INTERNALERROR" not in result.stdout.str()
@ -1226,9 +1289,10 @@ def test_fancy_items_regression(testdir):
]
def test_global_properties(testdir):
@parametrize_families
def test_global_properties(testdir, xunit_family):
path = testdir.tmpdir.join("test_global_properties.xml")
log = LogXML(str(path), None)
log = LogXML(str(path), None, family=xunit_family)
class Report(BaseReport):
sections = []
@ -1286,7 +1350,8 @@ def test_url_property(testdir):
), "The URL did not get written to the xml"
def test_record_testsuite_property(testdir):
@parametrize_families
def test_record_testsuite_property(testdir, run_and_parse, xunit_family):
testdir.makepyfile(
"""
def test_func1(record_testsuite_property):
@ -1296,7 +1361,7 @@ def test_record_testsuite_property(testdir):
record_testsuite_property("stats", 10)
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret == 0
node = dom.find_first_by_tag("testsuite")
properties_node = node.find_first_by_tag("properties")
@ -1334,14 +1399,16 @@ def test_record_testsuite_property_type_checking(testdir, junit):
@pytest.mark.parametrize("suite_name", ["my_suite", ""])
def test_set_suite_name(testdir, suite_name):
@parametrize_families
def test_set_suite_name(testdir, suite_name, run_and_parse, xunit_family):
if suite_name:
testdir.makeini(
"""
[pytest]
junit_suite_name={}
junit_suite_name={suite_name}
junit_family={family}
""".format(
suite_name
suite_name=suite_name, family=xunit_family
)
)
expected = suite_name
@ -1355,13 +1422,13 @@ def test_set_suite_name(testdir, suite_name):
pass
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret == 0
node = dom.find_first_by_tag("testsuite")
node.assert_attr(name=expected)
def test_escaped_skipreason_issue3533(testdir):
def test_escaped_skipreason_issue3533(testdir, run_and_parse):
testdir.makepyfile(
"""
import pytest
@ -1370,20 +1437,26 @@ def test_escaped_skipreason_issue3533(testdir):
pass
"""
)
_, dom = runandparse(testdir)
_, dom = run_and_parse()
node = dom.find_first_by_tag("testcase")
snode = node.find_first_by_tag("skipped")
assert "1 <> 2" in snode.text
snode.assert_attr(message="1 <> 2")
def test_logging_passing_tests_disabled_does_not_log_test_output(testdir):
@parametrize_families
def test_logging_passing_tests_disabled_does_not_log_test_output(
testdir, run_and_parse, xunit_family
):
testdir.makeini(
"""
[pytest]
junit_log_passing_tests=False
junit_logging=system-out
"""
junit_family={family}
""".format(
family=xunit_family
)
)
testdir.makepyfile(
"""
@ -1397,7 +1470,7 @@ def test_logging_passing_tests_disabled_does_not_log_test_output(testdir):
logging.warning('hello')
"""
)
result, dom = runandparse(testdir)
result, dom = run_and_parse(family=xunit_family)
assert result.ret == 0
node = dom.find_first_by_tag("testcase")
assert len(node.find_by_tag("system-err")) == 0