diff --git a/changelog/4488.feature.rst b/changelog/4488.feature.rst index ddbca65d6..1e0387f44 100644 --- a/changelog/4488.feature.rst +++ b/changelog/4488.feature.rst @@ -1,9 +1,10 @@ -New ``--report-log=FILE`` option that writes *report logs* into a file as the test session executes. +The pytest team has created the `pytest-reportlog `__ +plugin, which provides a new ``--report-log=FILE`` option that writes *report logs* into a file as the test session executes. Each line of the report log contains a self contained JSON object corresponding to a testing event, such as a collection or a test result report. The file is guaranteed to be flushed after writing each line, so systems can read and process events in real-time. -This option is meant to replace ``--resultlog``, which is deprecated and meant to be removed -in a future release. If you use ``--resultlog``, please try out ``--report-log`` and +The plugin is meant to replace the ``--resultlog`` option, which is deprecated and meant to be removed +in a future release. If you use ``--resultlog``, please try out ``pytest-reportlog`` and provide feedback. diff --git a/doc/en/contents.rst b/doc/en/contents.rst index 5d7599f50..c623d0602 100644 --- a/doc/en/contents.rst +++ b/doc/en/contents.rst @@ -27,7 +27,6 @@ Full pytest documentation unittest nose xunit_setup - report_log plugins writing_plugins logging diff --git a/doc/en/deprecations.rst b/doc/en/deprecations.rst index 34a05e1e6..748d3ac65 100644 --- a/doc/en/deprecations.rst +++ b/doc/en/deprecations.rst @@ -64,11 +64,12 @@ The ``--result-log`` option produces a stream of test reports which can be analysed at runtime, but it uses a custom format which requires users to implement their own parser. -The :ref:`--report-log ` option provides a more standard and extensible alternative, producing +The `pytest-reportlog `__ plugin provides a ``--report-log`` option, a more standard and extensible alternative, producing one JSON object per-line, and should cover the same use cases. Please try it out and provide feedback. -The plan is remove the ``--result-log`` option in pytest 6.0 after ``--result-log`` proves satisfactory -to all users and is deemed stable. +The plan is remove the ``--result-log`` option in pytest 6.0 if ``pytest-reportlog`` proves satisfactory +to all users and is deemed stable. The ``pytest-reportlog`` plugin might even be merged into the core +at some point, depending on the plans for the plugins and number of users using it. Removed Features diff --git a/doc/en/report_log.rst b/doc/en/report_log.rst deleted file mode 100644 index 619925180..000000000 --- a/doc/en/report_log.rst +++ /dev/null @@ -1,70 +0,0 @@ -.. _report_log: - -Report files -============ - -.. versionadded:: 5.3 - -The ``--report-log=FILE`` option writes *report logs* into a file as the test session executes. - -Each line of the report log contains a self contained JSON object corresponding to a testing event, -such as a collection or a test result report. The file is guaranteed to be flushed after writing -each line, so systems can read and process events in real-time. - -Each JSON object contains a special key ``$report_type``, which contains a unique identifier for -that kind of report object. For future compatibility, consumers of the file should ignore reports -they don't recognize, as well as ignore unknown properties/keys in JSON objects that they do know, -as future pytest versions might enrich the objects with more properties/keys. - -.. note:: - This option is meant to the replace ``--resultlog``, which is deprecated and meant to be removed - in a future release. If you use ``--resultlog``, please try out ``--report-log`` and - provide feedback. - -Example -------- - -Consider this file: - -.. code-block:: python - - # content of test_report_example.py - - - def test_ok(): - assert 5 + 5 == 10 - - - def test_fail(): - assert 4 + 4 == 1 - - -.. code-block:: pytest - - $ pytest test_report_example.py -q --report-log=log.json - .F [100%] - ================================= FAILURES ================================= - ________________________________ test_fail _________________________________ - - def test_fail(): - > assert 4 + 4 == 1 - E assert (4 + 4) == 1 - - test_report_example.py:8: AssertionError - ------------------- generated report log file: log.json -------------------- - 1 failed, 1 passed in 0.12s - -The generated ``log.json`` will contain a JSON object per line: - -:: - - $ cat log.json - {"pytest_version": "5.2.3.dev90+gd1129cf96.d20191026", "$report_type": "Header"} - {"nodeid": "", "outcome": "passed", "longrepr": null, "result": null, "sections": [], "$report_type": "CollectReport"} - {"nodeid": "test_report_example.py", "outcome": "passed", "longrepr": null, "result": null, "sections": [], "$report_type": "CollectReport"} - {"nodeid": "test_report_example.py::test_ok", "location": ["test_report_example.py", 2, "test_ok"], "keywords": {"report_log.rst-39": 1, "test_report_example.py": 1, "test_ok": 1}, "outcome": "passed", "longrepr": null, "when": "setup", "user_properties": [], "sections": [], "duration": 0.00021314620971679688, "$report_type": "TestReport"} - {"nodeid": "test_report_example.py::test_ok", "location": ["test_report_example.py", 2, "test_ok"], "keywords": {"report_log.rst-39": 1, "test_report_example.py": 1, "test_ok": 1}, "outcome": "passed", "longrepr": null, "when": "call", "user_properties": [], "sections": [], "duration": 0.00014543533325195312, "$report_type": "TestReport"} - {"nodeid": "test_report_example.py::test_ok", "location": ["test_report_example.py", 2, "test_ok"], "keywords": {"report_log.rst-39": 1, "test_report_example.py": 1, "test_ok": 1}, "outcome": "passed", "longrepr": null, "when": "teardown", "user_properties": [], "sections": [], "duration": 0.00016427040100097656, "$report_type": "TestReport"} - {"nodeid": "test_report_example.py::test_fail", "location": ["test_report_example.py", 6, "test_fail"], "keywords": {"test_fail": 1, "test_report_example.py": 1, "report_log.rst-39": 1}, "outcome": "passed", "longrepr": null, "when": "setup", "user_properties": [], "sections": [], "duration": 0.00013589859008789062, "$report_type": "TestReport"} - {"nodeid": "test_report_example.py::test_fail", "location": ["test_report_example.py", 6, "test_fail"], "keywords": {"test_fail": 1, "test_report_example.py": 1, "report_log.rst-39": 1}, "outcome": "failed", "longrepr": {"reprcrash": {"path": "$REGENDOC_TMPDIR/test_report_example.py", "lineno": 8, "message": "assert (4 + 4) == 1"}, "reprtraceback": {"reprentries": [{"type": "ReprEntry", "data": {"lines": [" def test_fail():", "> assert 4 + 4 == 1", "E assert (4 + 4) == 1"], "reprfuncargs": {"args": []}, "reprlocals": null, "reprfileloc": {"path": "test_report_example.py", "lineno": 8, "message": "AssertionError"}, "style": "long"}}], "extraline": null, "style": "long"}, "sections": [], "chain": [[{"reprentries": [{"type": "ReprEntry", "data": {"lines": [" def test_fail():", "> assert 4 + 4 == 1", "E assert (4 + 4) == 1"], "reprfuncargs": {"args": []}, "reprlocals": null, "reprfileloc": {"path": "test_report_example.py", "lineno": 8, "message": "AssertionError"}, "style": "long"}}], "extraline": null, "style": "long"}, {"path": "$REGENDOC_TMPDIR/test_report_example.py", "lineno": 8, "message": "assert (4 + 4) == 1"}, null]]}, "when": "call", "user_properties": [], "sections": [], "duration": 0.00027489662170410156, "$report_type": "TestReport"} - {"nodeid": "test_report_example.py::test_fail", "location": ["test_report_example.py", 6, "test_fail"], "keywords": {"test_fail": 1, "test_report_example.py": 1, "report_log.rst-39": 1}, "outcome": "passed", "longrepr": null, "when": "teardown", "user_properties": [], "sections": [], "duration": 0.00016689300537109375, "$report_type": "TestReport"} diff --git a/doc/en/usage.rst b/doc/en/usage.rst index 3b5919363..ea849c1a7 100644 --- a/doc/en/usage.rst +++ b/doc/en/usage.rst @@ -692,7 +692,7 @@ by the `PyPy-test`_ web page to show test results over several revisions. This option is rarely used and is scheduled for removal in pytest 6.0. - If you use this option, consider using the new :ref:`--result-log `. + If you use this option, consider using the new `pytest-reportlog `__ plugin instead. See `the deprecation docs `__ for more information. diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index d4521adf6..1c98f0266 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -154,7 +154,6 @@ default_plugins = essential_plugins + ( "assertion", "junitxml", "resultlog", - "report_log", "doctest", "cacheprovider", "freeze_support", diff --git a/src/_pytest/deprecated.py b/src/_pytest/deprecated.py index 442f102d1..5a7066041 100644 --- a/src/_pytest/deprecated.py +++ b/src/_pytest/deprecated.py @@ -26,7 +26,7 @@ FUNCARGNAMES = PytestDeprecationWarning( RESULT_LOG = PytestDeprecationWarning( - "--result-log is deprecated and scheduled for removal in pytest 6.0.\n" + "--result-log is deprecated, please try the new pytest-reportlog plugin.\n" "See https://docs.pytest.org/en/latest/deprecations.html#result-log-result-log for more information." ) diff --git a/src/_pytest/report_log.py b/src/_pytest/report_log.py deleted file mode 100644 index b12d0a55d..000000000 --- a/src/_pytest/report_log.py +++ /dev/null @@ -1,72 +0,0 @@ -import json -from pathlib import Path - -import pytest - - -def pytest_addoption(parser): - group = parser.getgroup("terminal reporting", "report-log plugin options") - group.addoption( - "--report-log", - action="store", - metavar="path", - default=None, - help="Path to line-based json objects of test session events.", - ) - - -def pytest_configure(config): - report_log = config.option.report_log - if report_log and not hasattr(config, "slaveinput"): - config._report_log_plugin = ReportLogPlugin(config, Path(report_log)) - config.pluginmanager.register(config._report_log_plugin) - - -def pytest_unconfigure(config): - report_log_plugin = getattr(config, "_report_log_plugin", None) - if report_log_plugin: - report_log_plugin.close() - del config._report_log_plugin - - -class ReportLogPlugin: - def __init__(self, config, log_path: Path): - self._config = config - self._log_path = log_path - - log_path.parent.mkdir(parents=True, exist_ok=True) - self._file = log_path.open("w", buffering=1, encoding="UTF-8") - - def close(self): - if self._file is not None: - self._file.close() - self._file = None - - def _write_json_data(self, data): - self._file.write(json.dumps(data) + "\n") - self._file.flush() - - def pytest_sessionstart(self): - data = {"pytest_version": pytest.__version__, "$report_type": "SessionStart"} - self._write_json_data(data) - - def pytest_sessionfinish(self, exitstatus): - data = {"exitstatus": exitstatus, "$report_type": "SessionFinish"} - self._write_json_data(data) - - def pytest_runtest_logreport(self, report): - data = self._config.hook.pytest_report_to_serializable( - config=self._config, report=report - ) - self._write_json_data(data) - - def pytest_collectreport(self, report): - data = self._config.hook.pytest_report_to_serializable( - config=self._config, report=report - ) - self._write_json_data(data) - - def pytest_terminal_summary(self, terminalreporter): - terminalreporter.write_sep( - "-", "generated report log file: {}".format(self._log_path) - ) diff --git a/testing/deprecated_test.py b/testing/deprecated_test.py index 64ec11b7a..5390d038d 100644 --- a/testing/deprecated_test.py +++ b/testing/deprecated_test.py @@ -16,7 +16,7 @@ def test_resultlog_is_deprecated(testdir): result = testdir.runpytest("--result-log=%s" % testdir.tmpdir.join("result.log")) result.stdout.fnmatch_lines( [ - "*--result-log is deprecated and scheduled for removal in pytest 6.0*", + "*--result-log is deprecated, please try the new pytest-reportlog plugin.", "*See https://docs.pytest.org/en/latest/deprecations.html#result-log-result-log for more information*", ] ) diff --git a/testing/test_report_log.py b/testing/test_report_log.py deleted file mode 100644 index cc2a431ec..000000000 --- a/testing/test_report_log.py +++ /dev/null @@ -1,54 +0,0 @@ -import json - -import pytest -from _pytest.reports import BaseReport - - -def test_basics(testdir, tmp_path, pytestconfig): - """Basic testing of the report log functionality. - - We don't test the test reports extensively because they have been - tested already in ``test_reports``. - """ - testdir.makepyfile( - """ - def test_ok(): - pass - - def test_fail(): - assert 0 - """ - ) - - log_file = tmp_path / "log.json" - - result = testdir.runpytest("--report-log", str(log_file)) - assert result.ret == pytest.ExitCode.TESTS_FAILED - result.stdout.fnmatch_lines(["* generated report log file: {}*".format(log_file)]) - - json_objs = [json.loads(x) for x in log_file.read_text().splitlines()] - assert len(json_objs) == 10 - - # first line should be the session_start - session_start = json_objs[0] - assert session_start == { - "pytest_version": pytest.__version__, - "$report_type": "SessionStart", - } - - # last line should be the session_finish - session_start = json_objs[-1] - assert session_start == { - "exitstatus": pytest.ExitCode.TESTS_FAILED, - "$report_type": "SessionFinish", - } - - # rest of the json objects should be unserialized into report objects; we don't test - # the actual report object extensively because it has been tested in ``test_reports`` - # already. - pm = pytestconfig.pluginmanager - for json_obj in json_objs[1:-1]: - rep = pm.hook.pytest_report_from_serializable( - config=pytestconfig, data=json_obj - ) - assert isinstance(rep, BaseReport)