From cfbd387a5d7a27a3e7dfb754d7cd4482e931822b Mon Sep 17 00:00:00 2001 From: Andrey Paramonov Date: Tue, 11 Dec 2018 19:29:31 +0300 Subject: [PATCH 1/7] Add --junittime=call option --- src/_pytest/junitxml.py | 20 +++++++++++++++++--- testing/test_junitxml.py | 19 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/_pytest/junitxml.py b/src/_pytest/junitxml.py index 09847c942..c5cc2b0a9 100644 --- a/src/_pytest/junitxml.py +++ b/src/_pytest/junitxml.py @@ -314,6 +314,15 @@ def pytest_addoption(parser): default=None, help="prepend prefix to classnames in junit-xml output", ) + group.addoption( + "--junittime", + "--junit-time", + action="store", + metavar="str", + default="total", + # choices=["total", "call"], + help='duration time to report: "total" (default), "call"', + ) parser.addini( "junit_suite_name", "Test suite name for JUnit report", default="pytest" ) @@ -334,6 +343,7 @@ def pytest_configure(config): config.option.junitprefix, config.getini("junit_suite_name"), config.getini("junit_logging"), + config.option.junittime, ) config.pluginmanager.register(config._xml) @@ -361,12 +371,14 @@ def mangle_test_address(address): class LogXML(object): - def __init__(self, logfile, prefix, suite_name="pytest", logging="no"): + def __init__(self, logfile, prefix, suite_name="pytest", logging="no", + report_duration=None): logfile = os.path.expanduser(os.path.expandvars(logfile)) self.logfile = os.path.normpath(os.path.abspath(logfile)) self.prefix = prefix self.suite_name = suite_name self.logging = logging + self.report_duration = report_duration self.stats = dict.fromkeys(["error", "passed", "failure", "skipped"], 0) self.node_reporters = {} # nodeid -> _NodeReporter self.node_reporters_ordered = [] @@ -500,8 +512,10 @@ class LogXML(object): """accumulates total duration for nodeid from given report and updates the Junit.testcase with the new total if already created. """ - reporter = self.node_reporter(report) - reporter.duration += getattr(report, "duration", 0.0) + if not self.report_duration or self.report_duration == "total" or \ + report.when == self.report_duration: + reporter = self.node_reporter(report) + reporter.duration += getattr(report, "duration", 0.0) def pytest_collectreport(self, report): if not report.passed: diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index c9dc39f82..82ed7901c 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -153,6 +153,24 @@ class TestPython(object): val = tnode["time"] assert round(float(val), 2) >= 0.03 + def test_call_time(self, testdir): + testdir.makepyfile( + """ + import time, pytest + def setup_module(): + time.sleep(0.01) + def teardown_module(): + time.sleep(0.01) + def test_sleep(): + time.sleep(0.01) + """ + ) + result, dom = runandparse(testdir, "--junit-time=call") + node = dom.find_first_by_tag("testsuite") + tnode = node.find_first_by_tag("testcase") + val = tnode["time"] + assert 0.01 <= round(float(val), 2) < 0.02 + def test_setup_error(self, testdir): testdir.makepyfile( """ @@ -727,6 +745,7 @@ def test_dont_configure_on_slaves(tmpdir): junitprefix = None # XXX: shouldnt need tmpdir ? xmlpath = str(tmpdir.join("junix.xml")) + junittime = None register = gotten.append fake_config = FakeConfig() From a44776ed48755de7c7a0860cdba078b363b3d2b6 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 11 Dec 2018 15:16:11 -0200 Subject: [PATCH 2/7] Fix linting --- src/_pytest/junitxml.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/_pytest/junitxml.py b/src/_pytest/junitxml.py index c5cc2b0a9..eed2a9a77 100644 --- a/src/_pytest/junitxml.py +++ b/src/_pytest/junitxml.py @@ -371,8 +371,9 @@ def mangle_test_address(address): class LogXML(object): - def __init__(self, logfile, prefix, suite_name="pytest", logging="no", - report_duration=None): + def __init__( + self, logfile, prefix, suite_name="pytest", logging="no", report_duration=None + ): logfile = os.path.expanduser(os.path.expandvars(logfile)) self.logfile = os.path.normpath(os.path.abspath(logfile)) self.prefix = prefix @@ -512,8 +513,11 @@ class LogXML(object): """accumulates total duration for nodeid from given report and updates the Junit.testcase with the new total if already created. """ - if not self.report_duration or self.report_duration == "total" or \ - report.when == self.report_duration: + if ( + not self.report_duration + or self.report_duration == "total" + or report.when == self.report_duration + ): reporter = self.node_reporter(report) reporter.duration += getattr(report, "duration", 0.0) From 0bccfc44a754dfefce3e99602e767253b7f360f7 Mon Sep 17 00:00:00 2001 From: Andrey Paramonov Date: Wed, 12 Dec 2018 12:14:14 +0300 Subject: [PATCH 3/7] Fix flaky test --- testing/test_junitxml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index 82ed7901c..ba7b43826 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -169,7 +169,7 @@ class TestPython(object): node = dom.find_first_by_tag("testsuite") tnode = node.find_first_by_tag("testcase") val = tnode["time"] - assert 0.01 <= round(float(val), 2) < 0.02 + assert 0.01 <= round(float(val), 3) < 0.02 def test_setup_error(self, testdir): testdir.makepyfile( From 316cca204ff04c93529368ce63a7d3f5cc60aa6c Mon Sep 17 00:00:00 2001 From: Andrey Paramonov Date: Wed, 12 Dec 2018 13:19:39 +0300 Subject: [PATCH 4/7] Switch to ini config parameter --- src/_pytest/junitxml.py | 27 +++++++++++---------------- testing/test_junitxml.py | 3 +-- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/_pytest/junitxml.py b/src/_pytest/junitxml.py index eed2a9a77..696deb6e9 100644 --- a/src/_pytest/junitxml.py +++ b/src/_pytest/junitxml.py @@ -314,15 +314,6 @@ def pytest_addoption(parser): default=None, help="prepend prefix to classnames in junit-xml output", ) - group.addoption( - "--junittime", - "--junit-time", - action="store", - metavar="str", - default="total", - # choices=["total", "call"], - help='duration time to report: "total" (default), "call"', - ) parser.addini( "junit_suite_name", "Test suite name for JUnit report", default="pytest" ) @@ -332,6 +323,9 @@ def pytest_addoption(parser): "one of no|system-out|system-err", default="no", ) # choices=['no', 'stdout', 'stderr']) + parser.addini( + "junit_time", "Duration time to report: one of total|call", default="total" + ) # choices=['total', 'call']) def pytest_configure(config): @@ -343,7 +337,7 @@ def pytest_configure(config): config.option.junitprefix, config.getini("junit_suite_name"), config.getini("junit_logging"), - config.option.junittime, + config.getini("junit_time"), ) config.pluginmanager.register(config._xml) @@ -372,7 +366,12 @@ def mangle_test_address(address): class LogXML(object): def __init__( - self, logfile, prefix, suite_name="pytest", logging="no", report_duration=None + self, + logfile, + prefix, + suite_name="pytest", + logging="no", + report_duration="total", ): logfile = os.path.expanduser(os.path.expandvars(logfile)) self.logfile = os.path.normpath(os.path.abspath(logfile)) @@ -513,11 +512,7 @@ class LogXML(object): """accumulates total duration for nodeid from given report and updates the Junit.testcase with the new total if already created. """ - if ( - not self.report_duration - or self.report_duration == "total" - or report.when == self.report_duration - ): + if self.report_duration == "total" or report.when == self.report_duration: reporter = self.node_reporter(report) reporter.duration += getattr(report, "duration", 0.0) diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index ba7b43826..896b11b82 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -165,7 +165,7 @@ class TestPython(object): time.sleep(0.01) """ ) - result, dom = runandparse(testdir, "--junit-time=call") + result, dom = runandparse(testdir, "-o", "junit_time=call") node = dom.find_first_by_tag("testsuite") tnode = node.find_first_by_tag("testcase") val = tnode["time"] @@ -745,7 +745,6 @@ def test_dont_configure_on_slaves(tmpdir): junitprefix = None # XXX: shouldnt need tmpdir ? xmlpath = str(tmpdir.join("junix.xml")) - junittime = None register = gotten.append fake_config = FakeConfig() From b1e766c30ed3f9a5cf3fa7dd47b330dd8684565e Mon Sep 17 00:00:00 2001 From: Andrey Paramonov Date: Wed, 12 Dec 2018 13:27:44 +0300 Subject: [PATCH 5/7] Update docs --- AUTHORS | 1 + changelog/4483.feature.rst | 2 ++ doc/en/usage.rst | 14 ++++++++++++++ 3 files changed, 17 insertions(+) create mode 100644 changelog/4483.feature.rst diff --git a/AUTHORS b/AUTHORS index 684063778..00ced49f0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -17,6 +17,7 @@ Anders Hovmöller Andras Tim Andrea Cimatoribus Andreas Zeidler +Andrey Paramonov Andrzej Ostrowski Andy Freeland Anthon van der Neut diff --git a/changelog/4483.feature.rst b/changelog/4483.feature.rst new file mode 100644 index 000000000..d9bd4c717 --- /dev/null +++ b/changelog/4483.feature.rst @@ -0,0 +1,2 @@ +Add ini parameter ``junit_time`` to optionally report test call +durations less setup and teardown times. diff --git a/doc/en/usage.rst b/doc/en/usage.rst index 49c2aa577..91b91002b 100644 --- a/doc/en/usage.rst +++ b/doc/en/usage.rst @@ -294,6 +294,20 @@ To set the name of the root test suite xml item, you can configure the ``junit_s [pytest] junit_suite_name = my_suite +.. versionadded:: 4.0 + +JUnit XML specification seems to indicate that ``"time"`` attribute +should report total test execution times, including setup and teardown +(`1`_, +`2`_). +It is the default pytest behavior. To report just call durations +instead, configure the ``junit_time`` option like this: + +.. code-block:: ini + + [pytest] + junit_time = call + .. _record_property example: record_property From ec4507d12a12cf9cc8b3bd33952abc1042e11344 Mon Sep 17 00:00:00 2001 From: Andrey Paramonov Date: Wed, 12 Dec 2018 14:33:02 +0300 Subject: [PATCH 6/7] Fix doc formatting --- doc/en/usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/en/usage.rst b/doc/en/usage.rst index 91b91002b..7c3ef19fb 100644 --- a/doc/en/usage.rst +++ b/doc/en/usage.rst @@ -298,8 +298,8 @@ To set the name of the root test suite xml item, you can configure the ``junit_s JUnit XML specification seems to indicate that ``"time"`` attribute should report total test execution times, including setup and teardown -(`1`_, -`2`_). +(`1 `_, `2 +`_). It is the default pytest behavior. To report just call durations instead, configure the ``junit_time`` option like this: From 5d79baf3f8cc2986dacfe0e34ff0b84794beecb4 Mon Sep 17 00:00:00 2001 From: Andrey Paramonov Date: Wed, 12 Dec 2018 15:33:29 +0300 Subject: [PATCH 7/7] Fix flaky test attempt 2 --- testing/test_junitxml.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index 896b11b82..aafbb8da9 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -158,18 +158,18 @@ class TestPython(object): """ import time, pytest def setup_module(): - time.sleep(0.01) + time.sleep(0.1) def teardown_module(): - time.sleep(0.01) + time.sleep(0.1) def test_sleep(): - time.sleep(0.01) + time.sleep(0.1) """ ) result, dom = runandparse(testdir, "-o", "junit_time=call") node = dom.find_first_by_tag("testsuite") tnode = node.find_first_by_tag("testcase") val = tnode["time"] - assert 0.01 <= round(float(val), 3) < 0.02 + assert 0.1 <= round(float(val), 2) < 0.2 def test_setup_error(self, testdir): testdir.makepyfile(