From 587951966f9a56789155c5445986c87793abfb16 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 31 Dec 2009 11:25:07 +0100 Subject: [PATCH] adding a logxml plugin and a --xml=path option generating a junit-xml style result log. The xml result log can be parsed nicely by hudson. Initial code was based on Ross Lawley's pytest_xmlresult plugin. --HG-- branch : trunk --- AUTHORS | 3 +- CHANGELOG | 3 + ISSUES.txt | 13 ++ py/impl/test/pluginmanager.py | 3 +- py/plugin/pytest_logxml.py | 147 +++++++++++++++++++++ pytest_xmlresult.py | 189 --------------------------- testing/plugin/test_pytest_logxml.py | 121 +++++++++++++++++ 7 files changed, 288 insertions(+), 191 deletions(-) create mode 100644 py/plugin/pytest_logxml.py delete mode 100644 pytest_xmlresult.py create mode 100644 testing/plugin/test_pytest_logxml.py diff --git a/AUTHORS b/AUTHORS index 9c4dbc558..1af51deea 100644 --- a/AUTHORS +++ b/AUTHORS @@ -11,9 +11,10 @@ merlinux GmbH, Germany, office at merlinux eu Contributors include:: +Ross Lawley +Ralf Schmitt Chris Lamb Harald Armin Massa -Ralf Schmitt Martijn Faassen Ian Bicking Jan Balster diff --git a/CHANGELOG b/CHANGELOG index f26f499ec..54ee42296 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,9 @@ Changes between 1.X and 1.1.1 ===================================== +- new junitxml plugin: --xml=path will generate a junit style xml file + which is parseable e.g. by the hudson continous integration server. + - new option: --genscript=path will generate a standalone py.test script which will not need any libraries installed. thanks to Ralf Schmitt. diff --git a/ISSUES.txt b/ISSUES.txt index 298c61e81..17dddf0be 100644 --- a/ISSUES.txt +++ b/ISSUES.txt @@ -78,6 +78,19 @@ but a remote one fail because the tests directory does not contain an "__init__.py". Either give an error or make it work without the __init__.py +introduce a "RootCollector" +---------------------------------------------------------------- +tags: feature 1.2 + +Currently the top collector is a Directory node and +there also is the notion of a "topdir". See to refine +internal handling such that there is a RootCollector +which holds this topdir (or do away with topdirs?). +Make sure this leads to an improvement in how +tests are shown in hudson which currently sometimes +shows "workspace" and sometimes not as the leading +name. + deprecate ensuretemp / introduce funcargs to setup method -------------------------------------------------------------- tags: experimental-wish 1.2 diff --git a/py/impl/test/pluginmanager.py b/py/impl/test/pluginmanager.py index 89322aac2..5bda34ca5 100644 --- a/py/impl/test/pluginmanager.py +++ b/py/impl/test/pluginmanager.py @@ -8,7 +8,8 @@ from py.impl.test.outcome import Skipped default_plugins = ( "default runner capture terminal mark skipping tmpdir monkeypatch " - "recwarn pdb pastebin unittest helpconfig nose assertion genscript").split() + "recwarn pdb pastebin unittest helpconfig nose assertion genscript " + "logxml").split() def check_old_use(mod, modname): clsname = modname[len('pytest_'):].capitalize() + "Plugin" diff --git a/py/plugin/pytest_logxml.py b/py/plugin/pytest_logxml.py new file mode 100644 index 000000000..8c3a432e5 --- /dev/null +++ b/py/plugin/pytest_logxml.py @@ -0,0 +1,147 @@ +""" + logxml plugin for machine-readable logging of test results. + Based on initial code from Ross Lawley. +""" + +import py +import time + +def pytest_addoption(parser): + group = parser.getgroup("general") + group.addoption('--xml', action="store", dest="xmlpath", + metavar="path", default=None, + help="create junit-xml style report file at the given path.") + +def pytest_configure(config): + xmlpath = config.option.xmlpath + if xmlpath: + config._xml = LogXML(xmlpath) + config.pluginmanager.register(config._xml) + +def pytest_unconfigure(config): + xml = getattr(config, '_xml', None) + if xml: + del config._xml + config.pluginmanager.unregister(xml) + +class LogXML(object): + def __init__(self, logfile): + self.logfile = logfile + self.test_logs = [] + self.passed = self.skipped = 0 + self.failed = self.errors = 0 + self._durations = {} + + def _opentestcase(self, report): + node = report.item + d = {'time': self._durations.pop(report.item, "0")} + names = [x.replace(".py", "") for x in node.listnames()] + d['classname'] = ".".join(names[:-1]) + d['name'] = names[-1] + attrs = ['%s="%s"' % item for item in sorted(d.items())] + self.test_logs.append("\n" % " ".join(attrs)) + + def _closetestcase(self): + self.test_logs.append("") + + def append_pass(self, report): + self.passed += 1 + self._opentestcase(report) + self._closetestcase() + + def append_failure(self, report): + self._opentestcase(report) + s = py.xml.escape(str(report.longrepr)) + #msg = str(report.longrepr.reprtraceback.extraline) + self.test_logs.append( + '%s' % (s)) + self._closetestcase() + self.failed += 1 + + def _opentestcase_collectfailure(self, report): + node = report.collector + d = {'time': '???'} + names = [x.replace(".py", "") for x in node.listnames()] + d['classname'] = ".".join(names[:-1]) + d['name'] = names[-1] + attrs = ['%s="%s"' % item for item in sorted(d.items())] + self.test_logs.append("\n" % " ".join(attrs)) + + def append_collect_failure(self, report): + self._opentestcase_collectfailure(report) + s = py.xml.escape(str(report.longrepr)) + #msg = str(report.longrepr.reprtraceback.extraline) + self.test_logs.append( + '%s' % (s)) + self._closetestcase() + self.errors += 1 + + def append_error(self, report): + self._opentestcase(report) + s = py.xml.escape(str(report.longrepr)) + self.test_logs.append( + '%s' % s) + self._closetestcase() + self.errors += 1 + + def append_skipped(self, report): + self._opentestcase(report) + self.test_logs.append("") + self._closetestcase() + self.skipped += 1 + + def pytest_runtest_logreport(self, report): + if report.passed: + self.append_pass(report) + elif report.failed: + if report.when != "call": + self.append_error(report) + else: + self.append_failure(report) + elif report.skipped: + self.append_skipped(report) + + def pytest_runtest_call(self, item, __multicall__): + start = time.time() + try: + return __multicall__.execute() + finally: + self._durations[item] = time.time() - start + + def pytest_collectreport(self, report): + if not report.passed: + if report.failed: + self.append_collect_failure(report) + else: + self.append_collect_skipped(report) + + def pytest_internalerror(self, excrepr): + self.errors += 1 + data = py.xml.escape(str(excrepr)) + self.test_logs.append( + '\n' + ' ' + '%s' % data) + + def pytest_sessionstart(self, session): + self.suite_start_time = time.time() + + def pytest_sessionfinish(self, session, exitstatus, __multicall__): + logfile = open(self.logfile, 'w', 1) # line buffered + suite_stop_time = time.time() + suite_time_delta = suite_stop_time - self.suite_start_time + numtests = self.passed + self.skipped + self.failed + logfile.write('') + logfile.writelines(self.test_logs) + logfile.write('') + logfile.close() + tw = session.config.pluginmanager.getplugin("terminalreporter")._tw + tw.line() + tw.sep("-", "generated xml file: %s" %(self.logfile)) diff --git a/pytest_xmlresult.py b/pytest_xmlresult.py deleted file mode 100644 index 7192559d2..000000000 --- a/pytest_xmlresult.py +++ /dev/null @@ -1,189 +0,0 @@ -""" - xmlresult plugin for machine-readable logging of test results. - Useful for cruisecontrol integration code. - - An adaptation of pytest_resultlog.py -""" - -import time - -def pytest_addoption(parser): - group = parser.getgroup("xmlresult", "xmlresult plugin options") - group.addoption('--xmlresult', action="store", dest="xmlresult", metavar="path", default=None, - help="path for machine-readable xml result log.") - -def pytest_configure(config): - xmlresult = config.option.xmlresult - if xmlresult: - logfile = open(xmlresult, 'w', 1) # line buffered - config._xmlresult = XMLResult(logfile) - config.pluginmanager.register(config._xmlresult) - -def pytest_unconfigure(config): - xmlresult = getattr(config, '_xmlresult', None) - if xmlresult: - xmlresult.logfile.close() - del config._xmlresult - config.pluginmanager.unregister(xmlresult) - -def generic_path(item): - chain = item.listchain() - gpath = [chain[0].name] - fspath = chain[0].fspath - fspart = False - for node in chain[1:]: - newfspath = node.fspath - if newfspath == fspath: - if fspart: - gpath.append(':') - fspart = False - else: - gpath.append('.') - else: - gpath.append('/') - fspart = True - name = node.name - if name[0] in '([': - gpath.pop() - gpath.append(name) - fspath = newfspath - return ''.join(gpath) - -class XMLResult(object): - test_start_time = 0.0 - test_taken_time = 0.0 - test_count = 0 - error_count = 0 - failure_count = 0 - skip_count = 0 - - def __init__(self, logfile): - self.logfile = logfile - self.test_logs = [] - - def write_log_entry(self, testpath, shortrepr, longrepr): - self.test_count += 1 - # Create an xml log entry for the tests - self.test_logs.append('' % (testpath.split(':')[-1], testpath, self.test_taken_time)) - - # Do we have any other data to capture for Errors, Fails and Skips - if shortrepr in ['E', 'F', 'S']: - - if shortrepr == 'E': - self.error_count += 1 - elif shortrepr == 'F': - self.failure_count += 1 - elif shortrepr == 'S': - self.skip_count += 1 - - tag_map = {'E': 'error', 'F': 'failure', 'S': 'skipped'} - self.test_logs.append("<%s>" % tag_map[shortrepr]) - - # Output any more information - for line in longrepr.splitlines(): - self.test_logs.append("" % line) - self.test_logs.append("" % tag_map[shortrepr]) - self.test_logs.append("") - - def log_outcome(self, node, shortrepr, longrepr): - self.write_log_entry(node.name, shortrepr, longrepr) - - def pytest_runtest_logreport(self, report): - code = report.shortrepr - if report.passed: - longrepr = "" - code = "." - elif report.failed: - longrepr = str(report.longrepr) - code = "F" - elif report.skipped: - code = "S" - longrepr = str(report.longrepr.reprcrash.message) - self.log_outcome(report.item, code, longrepr) - - def pytest_runtest_setup(self, item): - self.test_start_time = time.time() - - def pytest_runtest_teardown(self, item): - self.test_taken_time = time.time() - self.test_start_time - - def pytest_collectreport(self, report): - if not report.passed: - if report.failed: - code = "F" - else: - assert report.skipped - code = "S" - longrepr = str(report.longrepr.reprcrash) - self.log_outcome(report.collector, code, longrepr) - - def pytest_internalerror(self, excrepr): - path = excrepr.reprcrash.path - self.errors += 1 - self.write_log_entry(path, '!', str(excrepr)) - - def pytest_sessionstart(self, session): - self.suite_start_time = time.time() - - def pytest_sessionfinish(self, session, exitstatus): - """ - Write the xml output - """ - suite_stop_time = time.time() - suite_time_delta = suite_stop_time - self.suite_start_time - self.logfile.write('') - self.logfile.writelines(self.test_logs) - self.logfile.write('') - self.logfile.close() - - -# Tests -def test_generic(testdir, LineMatcher): - testdir.plugins.append("resultlog") - testdir.makepyfile(""" - import py - def test_pass(): - pass - def test_fail(): - assert 0 - def test_skip(): - py.test.skip("") - """) - testdir.runpytest("--xmlresult=result.xml") - lines = testdir.tmpdir.join("result.xml").readlines(cr=0) - LineMatcher(lines).fnmatch_lines([ - '*testsuite errors="0" failures="1" skips="1" name="" tests="3"*' - ]) - LineMatcher(lines).fnmatch_lines([ - '**' - ]) - -def test_generic_path(): - from py.__.test.collect import Node, Item, FSCollector - p1 = Node('a') - assert p1.fspath is None - p2 = Node('B', parent=p1) - p3 = Node('()', parent = p2) - item = Item('c', parent = p3) - res = generic_path(item) - assert res == 'a.B().c' - - p0 = FSCollector('proj/test') - p1 = FSCollector('proj/test/a', parent=p0) - p2 = Node('B', parent=p1) - p3 = Node('()', parent = p2) - p4 = Node('c', parent=p3) - item = Item('[1]', parent = p4) - - res = generic_path(item) - assert res == 'test/a:B().c[1]' diff --git a/testing/plugin/test_pytest_logxml.py b/testing/plugin/test_pytest_logxml.py new file mode 100644 index 000000000..377cea996 --- /dev/null +++ b/testing/plugin/test_pytest_logxml.py @@ -0,0 +1,121 @@ + +from xml.dom import minidom + +def runandparse(testdir, *args): + resultpath = testdir.tmpdir.join("junit.xml") + result = testdir.runpytest("--xml=%s" % resultpath, *args) + xmldoc = minidom.parse(str(resultpath)) + return result, xmldoc + +def assert_attr(node, **kwargs): + for name, expected in kwargs.items(): + anode = node.getAttributeNode(name) + assert anode, "node %r has no attribute %r" %(node, name) + val = anode.value + assert val == str(expected) + +class TestPython: + def test_summing_simple(self, testdir): + testdir.makepyfile(""" + import py + def test_pass(): + pass + def test_fail(): + assert 0 + def test_skip(): + py.test.skip("") + """) + result, dom = runandparse(testdir) + assert result.ret + node = dom.getElementsByTagName("testsuite")[0] + assert_attr(node, errors=0, failures=1, skips=1, tests=3) + + def test_setup_error(self, testdir): + testdir.makepyfile(""" + def pytest_funcarg__arg(request): + raise ValueError() + def test_function(arg): + pass + """) + result, dom = runandparse(testdir) + assert result.ret + node = dom.getElementsByTagName("testsuite")[0] + assert_attr(node, errors=1, tests=0) + tnode = node.getElementsByTagName("testcase")[0] + assert_attr(tnode, + classname="test_setup_error.test_setup_error", + name="test_function") + fnode = tnode.getElementsByTagName("error")[0] + assert_attr(fnode, message="test setup failure") + assert "ValueError" in fnode.toxml() + + def test_internal_error(self, testdir): + testdir.makeconftest("def pytest_runtest_protocol(): 0 / 0") + testdir.makepyfile("def test_function(): pass") + result, dom = runandparse(testdir) + assert result.ret + node = dom.getElementsByTagName("testsuite")[0] + assert_attr(node, errors=1, tests=0) + tnode = node.getElementsByTagName("testcase")[0] + assert_attr(tnode, classname="pytest", name="internal") + fnode = tnode.getElementsByTagName("error")[0] + assert_attr(fnode, message="internal error") + assert "Division" in fnode.toxml() + + def test_failure_function(self, testdir): + testdir.makepyfile("def test_fail(): raise ValueError(42)") + result, dom = runandparse(testdir) + assert result.ret + node = dom.getElementsByTagName("testsuite")[0] + assert_attr(node, failures=1, tests=1) + tnode = node.getElementsByTagName("testcase")[0] + assert_attr(tnode, + classname="test_failure_function.test_failure_function", + name="test_fail") + fnode = tnode.getElementsByTagName("failure")[0] + assert_attr(fnode, message="test failure") + assert "ValueError" in fnode.toxml() + + def test_collect_error(self, testdir): + testdir.makepyfile("syntax error") + result, dom = runandparse(testdir) + assert result.ret + node = dom.getElementsByTagName("testsuite")[0] + assert_attr(node, errors=1, tests=0) + tnode = node.getElementsByTagName("testcase")[0] + assert_attr(tnode, + #classname="test_collect_error", + name="test_collect_error") + fnode = tnode.getElementsByTagName("failure")[0] + assert_attr(fnode, message="collection failure") + assert "invalid syntax" in fnode.toxml() + +class TestNonPython: + def test_summing_simple(self, testdir): + testdir.makeconftest(""" + import py + def pytest_collect_file(path, parent): + if path.ext == ".xyz": + return MyItem(path, parent) + class MyItem(py.test.collect.Item): + def __init__(self, path, parent): + super(MyItem, self).__init__(path.basename, parent) + self.fspath = path + def runtest(self): + raise ValueError(42) + def repr_failure(self, excinfo): + return "custom item runtest failed" + """) + testdir.tmpdir.join("myfile.xyz").write("hello") + result, dom = runandparse(testdir) + assert result.ret + node = dom.getElementsByTagName("testsuite")[0] + assert_attr(node, errors=0, failures=1, skips=0, tests=1) + tnode = node.getElementsByTagName("testcase")[0] + assert_attr(tnode, + #classname="test_collect_error", + name="myfile.xyz") + fnode = tnode.getElementsByTagName("failure")[0] + assert_attr(fnode, message="test failure") + assert "custom item runtest failed" in fnode.toxml() +