diff --git a/changelog/4488.feature.rst b/changelog/4488.feature.rst new file mode 100644 index 000000000..ddbca65d6 --- /dev/null +++ b/changelog/4488.feature.rst @@ -0,0 +1,9 @@ +New ``--report-log=FILE`` option that writes *report logs* into a file as the test session executes. + +Each line of the report log contains a self contained JSON object corresponding to a testing event, +such as a collection or a test result report. The file is guaranteed to be flushed after writing +each line, so systems can read and process events in real-time. + +This option is meant to replace ``--resultlog``, which is deprecated and meant to be removed +in a future release. If you use ``--resultlog``, please try out ``--report-log`` and +provide feedback. diff --git a/doc/en/contents.rst b/doc/en/contents.rst index c623d0602..5d7599f50 100644 --- a/doc/en/contents.rst +++ b/doc/en/contents.rst @@ -27,6 +27,7 @@ Full pytest documentation unittest nose xunit_setup + report_log plugins writing_plugins logging diff --git a/doc/en/deprecations.rst b/doc/en/deprecations.rst index 9d01e5f23..5cf3b0903 100644 --- a/doc/en/deprecations.rst +++ b/doc/en/deprecations.rst @@ -40,15 +40,14 @@ Result log (``--result-log``) .. deprecated:: 4.0 The ``--result-log`` option produces a stream of test reports which can be -analysed at runtime. It uses a custom format which requires users to implement their own -parser, but the team believes using a line-based format that can be parsed using standard -tools would provide a suitable and better alternative. +analysed at runtime, but it uses a custom format which requires users to implement their own +parser. -The current plan is to provide an alternative in the pytest 5.0 series and remove the ``--result-log`` -option in pytest 6.0 after the new implementation proves satisfactory to all users and is deemed -stable. +The :ref:`--report-log ` option provides a more standard and extensible alternative, producing +one JSON object per-line, and should cover the same use cases. Please try it out and provide feedback. -The actual alternative is still being discussed in issue `#4488 `__. +The plan is remove the ``--result-log`` option in pytest 6.0 after ``--result-log`` proves satisfactory +to all users and is deemed stable. Removed Features diff --git a/doc/en/report_log.rst b/doc/en/report_log.rst new file mode 100644 index 000000000..619925180 --- /dev/null +++ b/doc/en/report_log.rst @@ -0,0 +1,70 @@ +.. _report_log: + +Report files +============ + +.. versionadded:: 5.3 + +The ``--report-log=FILE`` option writes *report logs* into a file as the test session executes. + +Each line of the report log contains a self contained JSON object corresponding to a testing event, +such as a collection or a test result report. The file is guaranteed to be flushed after writing +each line, so systems can read and process events in real-time. + +Each JSON object contains a special key ``$report_type``, which contains a unique identifier for +that kind of report object. For future compatibility, consumers of the file should ignore reports +they don't recognize, as well as ignore unknown properties/keys in JSON objects that they do know, +as future pytest versions might enrich the objects with more properties/keys. + +.. note:: + This option is meant to the replace ``--resultlog``, which is deprecated and meant to be removed + in a future release. If you use ``--resultlog``, please try out ``--report-log`` and + provide feedback. + +Example +------- + +Consider this file: + +.. code-block:: python + + # content of test_report_example.py + + + def test_ok(): + assert 5 + 5 == 10 + + + def test_fail(): + assert 4 + 4 == 1 + + +.. code-block:: pytest + + $ pytest test_report_example.py -q --report-log=log.json + .F [100%] + ================================= FAILURES ================================= + ________________________________ test_fail _________________________________ + + def test_fail(): + > assert 4 + 4 == 1 + E assert (4 + 4) == 1 + + test_report_example.py:8: AssertionError + ------------------- generated report log file: log.json -------------------- + 1 failed, 1 passed in 0.12s + +The generated ``log.json`` will contain a JSON object per line: + +:: + + $ cat log.json + {"pytest_version": "5.2.3.dev90+gd1129cf96.d20191026", "$report_type": "Header"} + {"nodeid": "", "outcome": "passed", "longrepr": null, "result": null, "sections": [], "$report_type": "CollectReport"} + {"nodeid": "test_report_example.py", "outcome": "passed", "longrepr": null, "result": null, "sections": [], "$report_type": "CollectReport"} + {"nodeid": "test_report_example.py::test_ok", "location": ["test_report_example.py", 2, "test_ok"], "keywords": {"report_log.rst-39": 1, "test_report_example.py": 1, "test_ok": 1}, "outcome": "passed", "longrepr": null, "when": "setup", "user_properties": [], "sections": [], "duration": 0.00021314620971679688, "$report_type": "TestReport"} + {"nodeid": "test_report_example.py::test_ok", "location": ["test_report_example.py", 2, "test_ok"], "keywords": {"report_log.rst-39": 1, "test_report_example.py": 1, "test_ok": 1}, "outcome": "passed", "longrepr": null, "when": "call", "user_properties": [], "sections": [], "duration": 0.00014543533325195312, "$report_type": "TestReport"} + {"nodeid": "test_report_example.py::test_ok", "location": ["test_report_example.py", 2, "test_ok"], "keywords": {"report_log.rst-39": 1, "test_report_example.py": 1, "test_ok": 1}, "outcome": "passed", "longrepr": null, "when": "teardown", "user_properties": [], "sections": [], "duration": 0.00016427040100097656, "$report_type": "TestReport"} + {"nodeid": "test_report_example.py::test_fail", "location": ["test_report_example.py", 6, "test_fail"], "keywords": {"test_fail": 1, "test_report_example.py": 1, "report_log.rst-39": 1}, "outcome": "passed", "longrepr": null, "when": "setup", "user_properties": [], "sections": [], "duration": 0.00013589859008789062, "$report_type": "TestReport"} + {"nodeid": "test_report_example.py::test_fail", "location": ["test_report_example.py", 6, "test_fail"], "keywords": {"test_fail": 1, "test_report_example.py": 1, "report_log.rst-39": 1}, "outcome": "failed", "longrepr": {"reprcrash": {"path": "$REGENDOC_TMPDIR/test_report_example.py", "lineno": 8, "message": "assert (4 + 4) == 1"}, "reprtraceback": {"reprentries": [{"type": "ReprEntry", "data": {"lines": [" def test_fail():", "> assert 4 + 4 == 1", "E assert (4 + 4) == 1"], "reprfuncargs": {"args": []}, "reprlocals": null, "reprfileloc": {"path": "test_report_example.py", "lineno": 8, "message": "AssertionError"}, "style": "long"}}], "extraline": null, "style": "long"}, "sections": [], "chain": [[{"reprentries": [{"type": "ReprEntry", "data": {"lines": [" def test_fail():", "> assert 4 + 4 == 1", "E assert (4 + 4) == 1"], "reprfuncargs": {"args": []}, "reprlocals": null, "reprfileloc": {"path": "test_report_example.py", "lineno": 8, "message": "AssertionError"}, "style": "long"}}], "extraline": null, "style": "long"}, {"path": "$REGENDOC_TMPDIR/test_report_example.py", "lineno": 8, "message": "assert (4 + 4) == 1"}, null]]}, "when": "call", "user_properties": [], "sections": [], "duration": 0.00027489662170410156, "$report_type": "TestReport"} + {"nodeid": "test_report_example.py::test_fail", "location": ["test_report_example.py", 6, "test_fail"], "keywords": {"test_fail": 1, "test_report_example.py": 1, "report_log.rst-39": 1}, "outcome": "passed", "longrepr": null, "when": "teardown", "user_properties": [], "sections": [], "duration": 0.00016689300537109375, "$report_type": "TestReport"} diff --git a/doc/en/usage.rst b/doc/en/usage.rst index 167c7fa9b..a23cf764a 100644 --- a/doc/en/usage.rst +++ b/doc/en/usage.rst @@ -679,12 +679,6 @@ Creating resultlog format files ---------------------------------------------------- - - This option is rarely used and is scheduled for removal in 5.0. - - See `the deprecation docs `__ - for more information. - To create plain-text machine-readable result files you can issue: .. code-block:: bash @@ -694,6 +688,16 @@ To create plain-text machine-readable result files you can issue: and look at the content at the ``path`` location. Such files are used e.g. by the `PyPy-test`_ web page to show test results over several revisions. +.. warning:: + + This option is rarely used and is scheduled for removal in pytest 6.0. + + If you use this option, consider using the new :ref:`--result-log `. + + See `the deprecation docs `__ + for more information. + + .. _`PyPy-test`: http://buildbot.pypy.org/summary diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 4746fd6c7..2b0f48c07 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -154,6 +154,7 @@ default_plugins = essential_plugins + ( "assertion", "junitxml", "resultlog", + "report_log", "doctest", "cacheprovider", "freeze_support", diff --git a/src/_pytest/hookspec.py b/src/_pytest/hookspec.py index 10a9857d7..7a21837bd 100644 --- a/src/_pytest/hookspec.py +++ b/src/_pytest/hookspec.py @@ -381,16 +381,6 @@ def pytest_runtest_logreport(report): @hookspec(firstresult=True) def pytest_report_to_serializable(config, report): """ - .. warning:: - This hook is experimental and subject to change between pytest releases, even - bug fixes. - - The intent is for this to be used by plugins maintained by the core-devs, such - as ``pytest-xdist``, ``pytest-subtests``, and as a replacement for the internal - 'resultlog' plugin. - - In the future it might become part of the public hook API. - Serializes the given report object into a data structure suitable for sending over the wire, e.g. converted to JSON. """ @@ -399,16 +389,6 @@ def pytest_report_to_serializable(config, report): @hookspec(firstresult=True) def pytest_report_from_serializable(config, data): """ - .. warning:: - This hook is experimental and subject to change between pytest releases, even - bug fixes. - - The intent is for this to be used by plugins maintained by the core-devs, such - as ``pytest-xdist``, ``pytest-subtests``, and as a replacement for the internal - 'resultlog' plugin. - - In the future it might become part of the public hook API. - Restores a report object previously serialized with pytest_report_to_serializable(). """ diff --git a/src/_pytest/report_log.py b/src/_pytest/report_log.py new file mode 100644 index 000000000..b12d0a55d --- /dev/null +++ b/src/_pytest/report_log.py @@ -0,0 +1,72 @@ +import json +from pathlib import Path + +import pytest + + +def pytest_addoption(parser): + group = parser.getgroup("terminal reporting", "report-log plugin options") + group.addoption( + "--report-log", + action="store", + metavar="path", + default=None, + help="Path to line-based json objects of test session events.", + ) + + +def pytest_configure(config): + report_log = config.option.report_log + if report_log and not hasattr(config, "slaveinput"): + config._report_log_plugin = ReportLogPlugin(config, Path(report_log)) + config.pluginmanager.register(config._report_log_plugin) + + +def pytest_unconfigure(config): + report_log_plugin = getattr(config, "_report_log_plugin", None) + if report_log_plugin: + report_log_plugin.close() + del config._report_log_plugin + + +class ReportLogPlugin: + def __init__(self, config, log_path: Path): + self._config = config + self._log_path = log_path + + log_path.parent.mkdir(parents=True, exist_ok=True) + self._file = log_path.open("w", buffering=1, encoding="UTF-8") + + def close(self): + if self._file is not None: + self._file.close() + self._file = None + + def _write_json_data(self, data): + self._file.write(json.dumps(data) + "\n") + self._file.flush() + + def pytest_sessionstart(self): + data = {"pytest_version": pytest.__version__, "$report_type": "SessionStart"} + self._write_json_data(data) + + def pytest_sessionfinish(self, exitstatus): + data = {"exitstatus": exitstatus, "$report_type": "SessionFinish"} + self._write_json_data(data) + + def pytest_runtest_logreport(self, report): + data = self._config.hook.pytest_report_to_serializable( + config=self._config, report=report + ) + self._write_json_data(data) + + def pytest_collectreport(self, report): + data = self._config.hook.pytest_report_to_serializable( + config=self._config, report=report + ) + self._write_json_data(data) + + def pytest_terminal_summary(self, terminalreporter): + terminalreporter.write_sep( + "-", "generated report log file: {}".format(self._log_path) + ) diff --git a/src/_pytest/reports.py b/src/_pytest/reports.py index 49eec6129..b1592f817 100644 --- a/src/_pytest/reports.py +++ b/src/_pytest/reports.py @@ -329,18 +329,18 @@ class CollectErrorRepr(TerminalRepr): def pytest_report_to_serializable(report): if isinstance(report, (TestReport, CollectReport)): data = report._to_json() - data["_report_type"] = report.__class__.__name__ + data["$report_type"] = report.__class__.__name__ return data def pytest_report_from_serializable(data): - if "_report_type" in data: - if data["_report_type"] == "TestReport": + if "$report_type" in data: + if data["$report_type"] == "TestReport": return TestReport._from_json(data) - elif data["_report_type"] == "CollectReport": + elif data["$report_type"] == "CollectReport": return CollectReport._from_json(data) assert False, "Unknown report_type unserialize data: {}".format( - data["_report_type"] + data["$report_type"] ) diff --git a/testing/test_report_log.py b/testing/test_report_log.py new file mode 100644 index 000000000..cc2a431ec --- /dev/null +++ b/testing/test_report_log.py @@ -0,0 +1,54 @@ +import json + +import pytest +from _pytest.reports import BaseReport + + +def test_basics(testdir, tmp_path, pytestconfig): + """Basic testing of the report log functionality. + + We don't test the test reports extensively because they have been + tested already in ``test_reports``. + """ + testdir.makepyfile( + """ + def test_ok(): + pass + + def test_fail(): + assert 0 + """ + ) + + log_file = tmp_path / "log.json" + + result = testdir.runpytest("--report-log", str(log_file)) + assert result.ret == pytest.ExitCode.TESTS_FAILED + result.stdout.fnmatch_lines(["* generated report log file: {}*".format(log_file)]) + + json_objs = [json.loads(x) for x in log_file.read_text().splitlines()] + assert len(json_objs) == 10 + + # first line should be the session_start + session_start = json_objs[0] + assert session_start == { + "pytest_version": pytest.__version__, + "$report_type": "SessionStart", + } + + # last line should be the session_finish + session_start = json_objs[-1] + assert session_start == { + "exitstatus": pytest.ExitCode.TESTS_FAILED, + "$report_type": "SessionFinish", + } + + # rest of the json objects should be unserialized into report objects; we don't test + # the actual report object extensively because it has been tested in ``test_reports`` + # already. + pm = pytestconfig.pluginmanager + for json_obj in json_objs[1:-1]: + rep = pm.hook.pytest_report_from_serializable( + config=pytestconfig, data=json_obj + ) + assert isinstance(rep, BaseReport) diff --git a/testing/test_reports.py b/testing/test_reports.py index 9f6c56186..ff813543c 100644 --- a/testing/test_reports.py +++ b/testing/test_reports.py @@ -330,7 +330,7 @@ class TestHooks: data = pytestconfig.hook.pytest_report_to_serializable( config=pytestconfig, report=rep ) - assert data["_report_type"] == "TestReport" + assert data["$report_type"] == "TestReport" new_rep = pytestconfig.hook.pytest_report_from_serializable( config=pytestconfig, data=data ) @@ -352,7 +352,7 @@ class TestHooks: data = pytestconfig.hook.pytest_report_to_serializable( config=pytestconfig, report=rep ) - assert data["_report_type"] == "CollectReport" + assert data["$report_type"] == "CollectReport" new_rep = pytestconfig.hook.pytest_report_from_serializable( config=pytestconfig, data=data ) @@ -376,7 +376,7 @@ class TestHooks: data = pytestconfig.hook.pytest_report_to_serializable( config=pytestconfig, report=rep ) - data["_report_type"] = "Unknown" + data["$report_type"] = "Unknown" with pytest.raises(AssertionError): _ = pytestconfig.hook.pytest_report_from_serializable( config=pytestconfig, data=data