jiuyuan
/
test_ok1
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #4511 from jhunkeler/junit-strict 

Toggle JUnit behavior with INI option
This commit is contained in:
Bruno Oliveira 2019-01-24 20:54:32 -02:00 committed by GitHub
parent 51dd738b1a 85c5fa9f64
commit 9905a73ae0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 138 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
AUTHORS
View File

@ -118,6 +118,7 @@ Jonas Obrist
Jordan Guymon
Jordan Moldow
Jordan Speicher
Joseph Hunkeler
Joshua Bronson
Jurko Gospodnetić
Justyna Janczyszyn

2
changelog/3547.bugfix.rst Normal file
View File

@ -0,0 +1,2 @@
``--junitxml`` can emit XML compatible with Jenkins xUnit.
``junit_family`` INI option accepts ``legacy|xunit1``, which produces old style output, and ``xunit2`` that conforms more strictly to https://github.com/jenkinsci/xunit-plugin/blob/xunit-2.3.2/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd

76
src/_pytest/junitxml.py
View File

@ -66,12 +66,34 @@ def bin_xml_escape(arg):
return py.xml.raw(illegal_xml_re.sub(repl, py.xml.escape(arg)))
def merge_family(left, right):
result = {}
for kl, vl in left.items():
for kr, vr in right.items():
if not isinstance(vl, list):
raise TypeError(type(vl))
result[kl] = vl + vr
left.update(result)
families = {}
families["_base"] = {"testcase": ["classname", "name"]}
families["_base_legacy"] = {"testcase": ["file", "line", "url"]}
# xUnit 1.x inherits legacy attributes
families["xunit1"] = families["_base"].copy()
merge_family(families["xunit1"], families["_base_legacy"])
# xUnit 2.x uses strict base attributes
families["xunit2"] = families["_base"]
class _NodeReporter(object):
def __init__(self, nodeid, xml):
self.id = nodeid
self.xml = xml
self.add_stats = self.xml.add_stats
self.family = self.xml.family
self.duration = 0
self.properties = []
self.nodes = []
@ -119,8 +141,20 @@ class _NodeReporter(object):
self.attrs = attrs
self.attrs.update(existing_attrs) # restore any user-defined attributes
# Preserve legacy testcase behavior
if self.family == "xunit1":
return
# Filter out attributes not permitted by this test family.
# Including custom attributes because they are not valid here.
temp_attrs = {}
for key in self.attrs.keys():
if key in families[self.family]["testcase"]:
temp_attrs[key] = self.attrs[key]
self.attrs = temp_attrs
def to_xml(self):
testcase = Junit.testcase(time=self.duration, **self.attrs)
testcase = Junit.testcase(time="%.3f" % self.duration, **self.attrs)
testcase.append(self.make_properties_node())
for node in self.nodes:
testcase.append(node)
@ -269,16 +303,26 @@ def record_xml_attribute(request):
from _pytest.warning_types import PytestWarning
request.node.warn(PytestWarning("record_xml_attribute is an experimental feature"))
# Declare noop
def add_attr_noop(name, value):
pass
attr_func = add_attr_noop
xml = getattr(request.config, "_xml", None)
if xml is not None:
if xml is not None and xml.family != "xunit1":
request.node.warn(
PytestWarning(
"record_xml_attribute is incompatible with junit_family: "
"%s (use: legacy|xunit1)" % xml.family
)
)
elif xml is not None:
node_reporter = xml.node_reporter(request.node.nodeid)
return node_reporter.add_attribute
else:
attr_func = node_reporter.add_attribute
def add_attr_noop(name, value):
pass
return add_attr_noop
return attr_func
def pytest_addoption(parser):
@ -315,6 +359,11 @@ def pytest_addoption(parser):
"Duration time to report: one of total|call",
default="total",
) # choices=['total', 'call'])
parser.addini(
"junit_family",
"Emit XML for schema: one of legacy|xunit1|xunit2",
default="xunit1",
)
def pytest_configure(config):
@ -327,6 +376,7 @@ def pytest_configure(config):
config.getini("junit_suite_name"),
config.getini("junit_logging"),
config.getini("junit_duration_report"),
config.getini("junit_family"),
)
config.pluginmanager.register(config._xml)
@ -361,6 +411,7 @@ class LogXML(object):
suite_name="pytest",
logging="no",
report_duration="total",
family="xunit1",
):
logfile = os.path.expanduser(os.path.expandvars(logfile))
self.logfile = os.path.normpath(os.path.abspath(logfile))
@ -368,6 +419,7 @@ class LogXML(object):
self.suite_name = suite_name
self.logging = logging
self.report_duration = report_duration
self.family = family
self.stats = dict.fromkeys(["error", "passed", "failure", "skipped"], 0)
self.node_reporters = {} # nodeid -> _NodeReporter
self.node_reporters_ordered = []
@ -376,6 +428,10 @@ class LogXML(object):
self.open_reports = []
self.cnt_double_fail_tests = 0
# Replaces convenience family with real family
if self.family == "legacy":
self.family = "xunit1"
def finalize(self, report):
nodeid = getattr(report, "nodeid", report)
# local hack to handle xdist report order
@ -545,7 +601,7 @@ class LogXML(object):
name=self.suite_name,
errors=self.stats["error"],
failures=self.stats["failure"],
skips=self.stats["skipped"],
skipped=self.stats["skipped"],
tests=numtests,
time="%.3f" % suite_time_delta,
).unicode(indent=0)

164
testing/test_junitxml.py
View File

@ -107,7 +107,7 @@ class TestPython(object):
result, dom = runandparse(testdir)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(name="pytest", errors=0, failures=1, skips=2, tests=5)
node.assert_attr(name="pytest", errors=0, failures=1, skipped=2, tests=5)
def test_summing_simple_with_errors(self, testdir):
testdir.makepyfile(
@ -133,7 +133,7 @@ class TestPython(object):
result, dom = runandparse(testdir)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(name="pytest", errors=1, failures=2, skips=1, tests=5)
node.assert_attr(name="pytest", errors=1, failures=2, skipped=1, tests=5)
def test_timing_function(self, testdir):
testdir.makepyfile(
@ -201,12 +201,7 @@ class TestPython(object):
node = dom.find_first_by_tag("testsuite")
node.assert_attr(errors=1, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_setup_error.py",
line="5",
classname="test_setup_error",
name="test_function",
)
tnode.assert_attr(classname="test_setup_error", name="test_function")
fnode = tnode.find_first_by_tag("error")
fnode.assert_attr(message="test setup failure")
assert "ValueError" in fnode.toxml()
@ -228,12 +223,7 @@ class TestPython(object):
assert result.ret
node = dom.find_first_by_tag("testsuite")
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_teardown_error.py",
line="6",
classname="test_teardown_error",
name="test_function",
)
tnode.assert_attr(classname="test_teardown_error", name="test_function")
fnode = tnode.find_first_by_tag("error")
fnode.assert_attr(message="test teardown failure")
assert "ValueError" in fnode.toxml()
@ -274,14 +264,9 @@ class TestPython(object):
result, dom = runandparse(testdir)
assert result.ret == 0
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=1)
node.assert_attr(skipped=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_skip_contains_name_reason.py",
line="1",
classname="test_skip_contains_name_reason",
name="test_skip",
)
tnode.assert_attr(classname="test_skip_contains_name_reason", name="test_skip")
snode = tnode.find_first_by_tag("skipped")
snode.assert_attr(type="pytest.skip", message="hello23")
@ -297,13 +282,10 @@ class TestPython(object):
result, dom = runandparse(testdir)
assert result.ret == 0
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=1)
node.assert_attr(skipped=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_mark_skip_contains_name_reason.py",
line="1",
classname="test_mark_skip_contains_name_reason",
name="test_skip",
classname="test_mark_skip_contains_name_reason", name="test_skip"
)
snode = tnode.find_first_by_tag("skipped")
snode.assert_attr(type="pytest.skip", message="hello24")
@ -321,13 +303,10 @@ class TestPython(object):
result, dom = runandparse(testdir)
assert result.ret == 0
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=1)
node.assert_attr(skipped=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_mark_skipif_contains_name_reason.py",
line="2",
classname="test_mark_skipif_contains_name_reason",
name="test_skip",
classname="test_mark_skipif_contains_name_reason", name="test_skip"
)
snode = tnode.find_first_by_tag("skipped")
snode.assert_attr(type="pytest.skip", message="hello25")
@ -360,10 +339,7 @@ class TestPython(object):
node.assert_attr(failures=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_classname_instance.py",
line="1",
classname="test_classname_instance.TestClass",
name="test_method",
classname="test_classname_instance.TestClass", name="test_method"
)
def test_classname_nested_dir(self, testdir):
@ -374,12 +350,7 @@ class TestPython(object):
node = dom.find_first_by_tag("testsuite")
node.assert_attr(failures=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file=os.path.join("sub", "test_hello.py"),
line="0",
classname="sub.test_hello",
name="test_func",
)
tnode.assert_attr(classname="sub.test_hello", name="test_func")
def test_internal_error(self, testdir):
testdir.makeconftest("def pytest_runtest_protocol(): 0 / 0")
@ -415,12 +386,7 @@ class TestPython(object):
node = dom.find_first_by_tag("testsuite")
node.assert_attr(failures=1, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_failure_function.py",
line="3",
classname="test_failure_function",
name="test_fail",
)
tnode.assert_attr(classname="test_failure_function", name="test_fail")
fnode = tnode.find_first_by_tag("failure")
fnode.assert_attr(message="ValueError: 42")
assert "ValueError" in fnode.toxml()
@ -477,10 +443,7 @@ class TestPython(object):
tnode = node.find_nth_by_tag("testcase", index)
tnode.assert_attr(
file="test_failure_escape.py",
line="1",
classname="test_failure_escape",
name="test_func[%s]" % char,
classname="test_failure_escape", name="test_func[%s]" % char
)
sysout = tnode.find_first_by_tag("system-out")
text = sysout.text
@ -501,18 +464,10 @@ class TestPython(object):
node = dom.find_first_by_tag("testsuite")
node.assert_attr(failures=1, tests=2)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_junit_prefixing.py",
line="0",
classname="xyz.test_junit_prefixing",
name="test_func",
)
tnode.assert_attr(classname="xyz.test_junit_prefixing", name="test_func")
tnode = node.find_nth_by_tag("testcase", 1)
tnode.assert_attr(
file="test_junit_prefixing.py",
line="3",
classname="xyz.test_junit_prefixing.TestHello",
name="test_hello",
classname="xyz.test_junit_prefixing.TestHello", name="test_hello"
)
def test_xfailure_function(self, testdir):
@ -526,14 +481,9 @@ class TestPython(object):
result, dom = runandparse(testdir)
assert not result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=1, tests=1)
node.assert_attr(skipped=1, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_xfailure_function.py",
line="1",
classname="test_xfailure_function",
name="test_xfail",
)
tnode.assert_attr(classname="test_xfailure_function", name="test_xfail")
fnode = tnode.find_first_by_tag("skipped")
fnode.assert_attr(message="expected test failure")
# assert "ValueError" in fnode.toxml()
@ -569,14 +519,9 @@ class TestPython(object):
result, dom = runandparse(testdir)
# assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=0, tests=1)
node.assert_attr(skipped=0, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_xfailure_xpass.py",
line="1",
classname="test_xfailure_xpass",
name="test_xpass",
)
tnode.assert_attr(classname="test_xfailure_xpass", name="test_xpass")
def test_xfailure_xpass_strict(self, testdir):
testdir.makepyfile(
@ -590,14 +535,9 @@ class TestPython(object):
result, dom = runandparse(testdir)
# assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=0, tests=1)
node.assert_attr(skipped=0, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_xfailure_xpass_strict.py",
line="1",
classname="test_xfailure_xpass_strict",
name="test_xpass",
)
tnode.assert_attr(classname="test_xfailure_xpass_strict", name="test_xpass")
fnode = tnode.find_first_by_tag("failure")
fnode.assert_attr(message="[XPASS(strict)] This needs to fail!")
@ -608,8 +548,6 @@ class TestPython(object):
node = dom.find_first_by_tag("testsuite")
node.assert_attr(errors=1, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(file="test_collect_error.py", name="test_collect_error")
assert tnode["line"] is None
fnode = tnode.find_first_by_tag("error")
fnode.assert_attr(message="collection failure")
assert "SyntaxError" in fnode.toxml()
@ -792,7 +730,7 @@ class TestNonPython(object):
result, dom = runandparse(testdir)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(errors=0, failures=1, skips=0, tests=1)
node.assert_attr(errors=0, failures=1, skipped=0, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(name="myfile.xyz")
fnode = tnode.find_first_by_tag("failure")
@ -1042,6 +980,12 @@ def test_record_property_same_name(testdir):
@pytest.mark.filterwarnings("default")
def test_record_attribute(testdir):
testdir.makeini(
"""
[pytest]
junit_family = xunit1
"""
)
testdir.makepyfile(
"""
import pytest
@ -1063,6 +1007,38 @@ def test_record_attribute(testdir):
)
@pytest.mark.filterwarnings("default")
def test_record_attribute_xunit2(testdir):
"""Ensure record_xml_attribute drops values when outside of legacy family
"""
testdir.makeini(
"""
[pytest]
junit_family = xunit2
"""
)
testdir.makepyfile(
"""
import pytest
@pytest.fixture
def other(record_xml_attribute):
record_xml_attribute("bar", 1)
def test_record(record_xml_attribute, other):
record_xml_attribute("foo", "<1");
"""
)
result, dom = runandparse(testdir, "-rw")
result.stdout.fnmatch_lines(
[
"*test_record_attribute_xunit2.py:6:*record_xml_attribute is an experimental feature",
"*test_record_attribute_xunit2.py:6:*record_xml_attribute is incompatible with "
"junit_family: xunit2 (use: legacy|xunit1)",
]
)
def test_random_report_log_xdist(testdir, monkeypatch):
"""xdist calls pytest_runtest_logreport as they are executed by the slaves,
with nodes from several nodes overlapping, so junitxml must cope with that
@ -1155,20 +1131,18 @@ def test_fancy_items_regression(testdir):
assert "INTERNALERROR" not in result.stdout.str()
items = sorted(
"%(classname)s %(name)s %(file)s" % x for x in dom.find_by_tag("testcase")
)
items = sorted("%(classname)s %(name)s" % x for x in dom.find_by_tag("testcase"))
import pprint
pprint.pprint(items)
assert items == [
u"conftest a conftest.py",
u"conftest a conftest.py",
u"conftest b conftest.py",
u"test_fancy_items_regression a test_fancy_items_regression.py",
u"test_fancy_items_regression a test_fancy_items_regression.py",
u"test_fancy_items_regression b test_fancy_items_regression.py",
u"test_fancy_items_regression test_pass" u" test_fancy_items_regression.py",
u"conftest a",
u"conftest a",
u"conftest b",
u"test_fancy_items_regression a",
u"test_fancy_items_regression a",
u"test_fancy_items_regression b",
u"test_fancy_items_regression test_pass",
]