From a37b902afea21621639b114f087e84f70fb057ba Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 12 Jun 2019 18:49:51 -0300 Subject: [PATCH 1/2] Integrate pytest-faulthandler into the core * Add pytest-faulthandler files unchanged * Adapt imports and tests * Add code to skip registration of the external `pytest_faulthandler` to avoid conflicts Fix #5440 --- changelog/5440.feature.rst | 8 +++ doc/en/usage.rst | 19 +++++- src/_pytest/config/__init__.py | 3 +- src/_pytest/deprecated.py | 8 +++ src/_pytest/faulthandler.py | 102 +++++++++++++++++++++++++++++++++ testing/deprecated_test.py | 23 +++----- testing/test_faulthandler.py | 99 ++++++++++++++++++++++++++++++++ 7 files changed, 245 insertions(+), 17 deletions(-) create mode 100644 changelog/5440.feature.rst create mode 100644 src/_pytest/faulthandler.py create mode 100644 testing/test_faulthandler.py diff --git a/changelog/5440.feature.rst b/changelog/5440.feature.rst new file mode 100644 index 000000000..d3bb95f58 --- /dev/null +++ b/changelog/5440.feature.rst @@ -0,0 +1,8 @@ +The `faulthandler `__ standard library +module is now enabled by default to help users diagnose crashes in C modules. + +This functionality was provided by integrating the external +`pytest-faulthandler `__ plugin into the core, +so users should remove that plugin from their requirements if used. + +For more information see the docs: https://docs.pytest.org/en/latest/usage.html#fault-handler diff --git a/doc/en/usage.rst b/doc/en/usage.rst index acf736f21..c1332706f 100644 --- a/doc/en/usage.rst +++ b/doc/en/usage.rst @@ -408,7 +408,6 @@ Pytest supports the use of ``breakpoint()`` with the following behaviours: Profiling test execution duration ------------------------------------- -.. versionadded: 2.2 To get a list of the slowest 10 test durations: @@ -418,6 +417,24 @@ To get a list of the slowest 10 test durations: By default, pytest will not show test durations that are too small (<0.01s) unless ``-vv`` is passed on the command-line. + +.. _faulthandler: + +Fault Handler +------------- + +.. versionadded:: 5.0 + +The `faulthandler `__ standard module +can be used to dump Python tracebacks on a segfault or after a timeout. + +The module is automatically enabled for pytest runs, unless the ``--no-faulthandler`` is given +on the command-line. + +Also the ``--faulthandler-timeout=X`` can be used to dump the traceback of all threads if a test +takes longer than ``X`` seconds to finish (not available on Windows). + + Creating JUnitXML format files ---------------------------------------------------- diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 1f6ae98f9..74ee4a2bc 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -141,6 +141,7 @@ default_plugins = essential_plugins + ( "warnings", "logging", "reports", + "faulthandler", ) builtin_plugins = set(default_plugins) @@ -299,7 +300,7 @@ class PytestPluginManager(PluginManager): return opts def register(self, plugin, name=None): - if name in ["pytest_catchlog", "pytest_capturelog"]: + if name in _pytest.deprecated.DEPRECATED_EXTERNAL_PLUGINS: warnings.warn( PytestConfigWarning( "{} plugin has been merged into the core, " diff --git a/src/_pytest/deprecated.py b/src/_pytest/deprecated.py index 3feae8b43..1c544fd36 100644 --- a/src/_pytest/deprecated.py +++ b/src/_pytest/deprecated.py @@ -14,6 +14,14 @@ from _pytest.warning_types import UnformattedWarning YIELD_TESTS = "yield tests were removed in pytest 4.0 - {name} will be ignored" +# set of plugins which have been integrated into the core; we use this list to ignore +# them during registration to avoid conflicts +DEPRECATED_EXTERNAL_PLUGINS = { + "pytest_catchlog", + "pytest_capturelog", + "pytest_faulthandler", +} + FIXTURE_FUNCTION_CALL = ( 'Fixture "{name}" called directly. Fixtures are not meant to be called directly,\n' diff --git a/src/_pytest/faulthandler.py b/src/_pytest/faulthandler.py new file mode 100644 index 000000000..48fe0f218 --- /dev/null +++ b/src/_pytest/faulthandler.py @@ -0,0 +1,102 @@ +import io +import os +import sys + +import pytest + + +def pytest_addoption(parser): + group = parser.getgroup("terminal reporting") + group.addoption( + "--no-faulthandler", + action="store_false", + dest="fault_handler", + default=True, + help="Disable faulthandler module.", + ) + + group.addoption( + "--faulthandler-timeout", + type=float, + dest="fault_handler_timeout", + metavar="TIMEOUT", + default=0.0, + help="Dump the traceback of all threads if a test takes " + "more than TIMEOUT seconds to finish.\n" + "Not available on Windows.", + ) + + +def pytest_configure(config): + if config.getoption("fault_handler"): + import faulthandler + + # avoid trying to dup sys.stderr if faulthandler is already enabled + if faulthandler.is_enabled(): + return + + stderr_fd_copy = os.dup(_get_stderr_fileno()) + config.fault_handler_stderr = os.fdopen(stderr_fd_copy, "w") + faulthandler.enable(file=config.fault_handler_stderr) + + +def _get_stderr_fileno(): + try: + return sys.stderr.fileno() + except (AttributeError, io.UnsupportedOperation): + # python-xdist monkeypatches sys.stderr with an object that is not an actual file. + # https://docs.python.org/3/library/faulthandler.html#issue-with-file-descriptors + # This is potentially dangerous, but the best we can do. + return sys.__stderr__.fileno() + + +def pytest_unconfigure(config): + if config.getoption("fault_handler"): + import faulthandler + + faulthandler.disable() + # close our dup file installed during pytest_configure + f = getattr(config, "fault_handler_stderr", None) + if f is not None: + # re-enable the faulthandler, attaching it to the default sys.stderr + # so we can see crashes after pytest has finished, usually during + # garbage collection during interpreter shutdown + config.fault_handler_stderr.close() + del config.fault_handler_stderr + faulthandler.enable(file=_get_stderr_fileno()) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_protocol(item): + enabled = item.config.getoption("fault_handler") + timeout = item.config.getoption("fault_handler_timeout") + if enabled and timeout > 0: + import faulthandler + + stderr = item.config.fault_handler_stderr + faulthandler.dump_traceback_later(timeout, file=stderr) + try: + yield + finally: + faulthandler.cancel_dump_traceback_later() + else: + yield + + +@pytest.hookimpl(tryfirst=True) +def pytest_enter_pdb(): + """Cancel any traceback dumping due to timeout before entering pdb. + """ + import faulthandler + + faulthandler.cancel_dump_traceback_later() + + +@pytest.hookimpl(tryfirst=True) +def pytest_exception_interact(): + """Cancel any traceback dumping due to an interactive exception being + raised. + """ + import faulthandler + + faulthandler.cancel_dump_traceback_later() diff --git a/testing/deprecated_test.py b/testing/deprecated_test.py index 177594c4a..5cbb694b1 100644 --- a/testing/deprecated_test.py +++ b/testing/deprecated_test.py @@ -1,6 +1,7 @@ import os import pytest +from _pytest import deprecated from _pytest.warning_types import PytestDeprecationWarning from _pytest.warnings import SHOW_PYTEST_WARNINGS_ARG @@ -69,22 +70,14 @@ def test_terminal_reporter_writer_attr(pytestconfig): assert terminal_reporter.writer is terminal_reporter._tw -@pytest.mark.parametrize("plugin", ["catchlog", "capturelog"]) +@pytest.mark.parametrize("plugin", deprecated.DEPRECATED_EXTERNAL_PLUGINS) @pytest.mark.filterwarnings("default") -def test_pytest_catchlog_deprecated(testdir, plugin): - testdir.makepyfile( - """ - def test_func(pytestconfig): - pytestconfig.pluginmanager.register(None, 'pytest_{}') - """.format( - plugin - ) - ) - res = testdir.runpytest() - assert res.ret == 0 - res.stdout.fnmatch_lines( - ["*pytest-*log plugin has been merged into the core*", "*1 passed, 1 warnings*"] - ) +def test_external_plugins_integrated(testdir, plugin): + testdir.syspathinsert() + testdir.makepyfile(**{plugin: ""}) + + with pytest.warns(pytest.PytestConfigWarning): + testdir.parseconfig("-p", plugin) def test_raises_message_argument_deprecated(): diff --git a/testing/test_faulthandler.py b/testing/test_faulthandler.py new file mode 100644 index 000000000..d1f2e8b9a --- /dev/null +++ b/testing/test_faulthandler.py @@ -0,0 +1,99 @@ +import sys + +import pytest + + +def test_enabled(testdir): + """Test single crashing test displays a traceback.""" + testdir.makepyfile( + """ + import faulthandler + def test_crash(): + faulthandler._sigabrt() + """ + ) + result = testdir.runpytest_subprocess() + result.stderr.fnmatch_lines(["*Fatal Python error*"]) + assert result.ret != 0 + + +def test_crash_near_exit(testdir): + """Test that fault handler displays crashes that happen even after + pytest is exiting (for example, when the interpreter is shutting down). + """ + testdir.makepyfile( + """ + import faulthandler + import atexit + def test_ok(): + atexit.register(faulthandler._sigabrt) + """ + ) + result = testdir.runpytest_subprocess() + result.stderr.fnmatch_lines(["*Fatal Python error*"]) + assert result.ret != 0 + + +def test_disabled(testdir): + """Test option to disable fault handler in the command line. + """ + testdir.makepyfile( + """ + import faulthandler + def test_disabled(): + assert not faulthandler.is_enabled() + """ + ) + result = testdir.runpytest_subprocess("--no-faulthandler") + result.stdout.fnmatch_lines(["*1 passed*"]) + assert result.ret == 0 + + +@pytest.mark.parametrize("enabled", [True, False]) +def test_timeout(testdir, enabled): + """Test option to dump tracebacks after a certain timeout. + If faulthandler is disabled, no traceback will be dumped. + """ + testdir.makepyfile( + """ + import time + def test_timeout(): + time.sleep(2.0) + """ + ) + args = ["--faulthandler-timeout=1"] + if not enabled: + args.append("--no-faulthandler") + + result = testdir.runpytest_subprocess(*args) + tb_output = "most recent call first" + if sys.version_info[:2] == (3, 3): + tb_output = "Thread" + if enabled: + result.stderr.fnmatch_lines(["*%s*" % tb_output]) + else: + assert tb_output not in result.stderr.str() + result.stdout.fnmatch_lines(["*1 passed*"]) + assert result.ret == 0 + + +@pytest.mark.parametrize("hook_name", ["pytest_enter_pdb", "pytest_exception_interact"]) +def test_cancel_timeout_on_hook(monkeypatch, pytestconfig, hook_name): + """Make sure that we are cancelling any scheduled traceback dumping due + to timeout before entering pdb (pytest-dev/pytest-faulthandler#12) or any other interactive + exception (pytest-dev/pytest-faulthandler#14). + """ + import faulthandler + from _pytest import faulthandler as plugin_module + + called = [] + + monkeypatch.setattr( + faulthandler, "cancel_dump_traceback_later", lambda: called.append(1) + ) + + # call our hook explicitly, we can trust that pytest will call the hook + # for us at the appropriate moment + hook_func = getattr(plugin_module, hook_name) + hook_func() + assert called == [1] From 3ce31b6370fcaa02a63f09c86a2859800d15984a Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 22 Jun 2019 19:22:43 -0300 Subject: [PATCH 2/2] Change pytest-faulthandler for simplification * The --no-faulthandler option is not necessary given that we can use `-p no:faulthandler`. * The `--faulthandler-timeout` command-line option has become an ini option, for the reasons described in https://github.com/pytest-dev/pytest-faulthandler/issues/34 and users can still set it from the command-line. Fix pytest-dev/pytest-faulthandler#34 --- doc/en/reference.rst | 17 ++++++++++ doc/en/usage.rst | 20 +++++++++-- src/_pytest/faulthandler.py | 64 ++++++++++++++---------------------- testing/test_faulthandler.py | 12 ++++--- 4 files changed, 66 insertions(+), 47 deletions(-) diff --git a/doc/en/reference.rst b/doc/en/reference.rst index 0b168eb54..7ad69d4a8 100644 --- a/doc/en/reference.rst +++ b/doc/en/reference.rst @@ -1076,6 +1076,23 @@ passed multiple times. The expected format is ``name=value``. For example:: for more details. +.. confval:: faulthandler_timeout + + Dumps the tracebacks of all threads if a test takes longer than ``X`` seconds to run (including + fixture setup and teardown). Implemented using the `faulthandler.dump_traceback_later`_ function, + so all caveats there apply. + + .. code-block:: ini + + # content of pytest.ini + [pytest] + faulthandler_timeout=5 + + For more information please refer to :ref:`faulthandler`. + +.. _`faulthandler.dump_traceback_later`: https://docs.python.org/3/library/faulthandler.html#faulthandler.dump_traceback_later + + .. confval:: filterwarnings diff --git a/doc/en/usage.rst b/doc/en/usage.rst index c1332706f..a8acc3551 100644 --- a/doc/en/usage.rst +++ b/doc/en/usage.rst @@ -428,11 +428,25 @@ Fault Handler The `faulthandler `__ standard module can be used to dump Python tracebacks on a segfault or after a timeout. -The module is automatically enabled for pytest runs, unless the ``--no-faulthandler`` is given +The module is automatically enabled for pytest runs, unless the ``-p no:faulthandler`` is given on the command-line. -Also the ``--faulthandler-timeout=X`` can be used to dump the traceback of all threads if a test -takes longer than ``X`` seconds to finish (not available on Windows). +Also the :confval:`faulthandler_timeout=X` configuration option can be used +to dump the traceback of all threads if a test takes longer than ``X`` +seconds to finish (not available on Windows). + +.. note:: + + This functionality has been integrated from the external + `pytest-faulthandler `__ plugin, with two + small differences: + + * To disable it, use ``-p no:faulthandler`` instead of ``--no-faulthandler``: the former + can be used with any plugin, so it saves one option. + + * The ``--faulthandler-timeout`` command-line option has become the + :confval:`faulthandler_timeout` configuration option. It can still be configured from + the command-line using ``-o faulthandler_timeout=X``. Creating JUnitXML format files diff --git a/src/_pytest/faulthandler.py b/src/_pytest/faulthandler.py index 48fe0f218..068bec528 100644 --- a/src/_pytest/faulthandler.py +++ b/src/_pytest/faulthandler.py @@ -6,38 +6,24 @@ import pytest def pytest_addoption(parser): - group = parser.getgroup("terminal reporting") - group.addoption( - "--no-faulthandler", - action="store_false", - dest="fault_handler", - default=True, - help="Disable faulthandler module.", - ) - - group.addoption( - "--faulthandler-timeout", - type=float, - dest="fault_handler_timeout", - metavar="TIMEOUT", - default=0.0, - help="Dump the traceback of all threads if a test takes " + help = ( + "Dump the traceback of all threads if a test takes " "more than TIMEOUT seconds to finish.\n" - "Not available on Windows.", + "Not available on Windows." ) + parser.addini("faulthandler_timeout", help, default=0.0) def pytest_configure(config): - if config.getoption("fault_handler"): - import faulthandler + import faulthandler - # avoid trying to dup sys.stderr if faulthandler is already enabled - if faulthandler.is_enabled(): - return + # avoid trying to dup sys.stderr if faulthandler is already enabled + if faulthandler.is_enabled(): + return - stderr_fd_copy = os.dup(_get_stderr_fileno()) - config.fault_handler_stderr = os.fdopen(stderr_fd_copy, "w") - faulthandler.enable(file=config.fault_handler_stderr) + stderr_fd_copy = os.dup(_get_stderr_fileno()) + config.fault_handler_stderr = os.fdopen(stderr_fd_copy, "w") + faulthandler.enable(file=config.fault_handler_stderr) def _get_stderr_fileno(): @@ -51,26 +37,24 @@ def _get_stderr_fileno(): def pytest_unconfigure(config): - if config.getoption("fault_handler"): - import faulthandler + import faulthandler - faulthandler.disable() - # close our dup file installed during pytest_configure - f = getattr(config, "fault_handler_stderr", None) - if f is not None: - # re-enable the faulthandler, attaching it to the default sys.stderr - # so we can see crashes after pytest has finished, usually during - # garbage collection during interpreter shutdown - config.fault_handler_stderr.close() - del config.fault_handler_stderr - faulthandler.enable(file=_get_stderr_fileno()) + faulthandler.disable() + # close our dup file installed during pytest_configure + f = getattr(config, "fault_handler_stderr", None) + if f is not None: + # re-enable the faulthandler, attaching it to the default sys.stderr + # so we can see crashes after pytest has finished, usually during + # garbage collection during interpreter shutdown + config.fault_handler_stderr.close() + del config.fault_handler_stderr + faulthandler.enable(file=_get_stderr_fileno()) @pytest.hookimpl(hookwrapper=True) def pytest_runtest_protocol(item): - enabled = item.config.getoption("fault_handler") - timeout = item.config.getoption("fault_handler_timeout") - if enabled and timeout > 0: + timeout = float(item.config.getini("faulthandler_timeout") or 0.0) + if timeout > 0: import faulthandler stderr = item.config.fault_handler_stderr diff --git a/testing/test_faulthandler.py b/testing/test_faulthandler.py index d1f2e8b9a..a0cf1d8c1 100644 --- a/testing/test_faulthandler.py +++ b/testing/test_faulthandler.py @@ -44,7 +44,7 @@ def test_disabled(testdir): assert not faulthandler.is_enabled() """ ) - result = testdir.runpytest_subprocess("--no-faulthandler") + result = testdir.runpytest_subprocess("-p", "no:faulthandler") result.stdout.fnmatch_lines(["*1 passed*"]) assert result.ret == 0 @@ -61,9 +61,13 @@ def test_timeout(testdir, enabled): time.sleep(2.0) """ ) - args = ["--faulthandler-timeout=1"] - if not enabled: - args.append("--no-faulthandler") + testdir.makeini( + """ + [pytest] + faulthandler_timeout = 1 + """ + ) + args = ["-p", "no:faulthandler"] if not enabled else [] result = testdir.runpytest_subprocess(*args) tb_output = "most recent call first"