From 006a901b861e9de28daf11ab4b10b87bed18aba1 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 27 Oct 2016 21:15:05 -0200 Subject: [PATCH] Properly handle exceptions in multiprocessing tasks Fix #1984 --- CHANGELOG.rst | 11 ++++++++- _pytest/_code/code.py | 15 ++++++++---- testing/code/test_excinfo.py | 44 ++++++++++++++++++++++++++++++++++++ testing/test_assertion.py | 31 +++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 210359fb4..44a7f9a39 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,7 +6,7 @@ * Import errors when collecting test modules now display the full traceback (`#1976`_). Thanks `@cwitty`_ for the report and `@nicoddemus`_ for the PR. -* Fix confusing command-line help message for custom options with two or more `metavar` properties (`#2004`_). +* Fix confusing command-line help message for custom options with two or more ``metavar`` properties (`#2004`_). Thanks `@okulynyak`_ and `@davehunt`_ for the report and `@nicoddemus`_ for the PR. * When loading plugins, import errors which contain non-ascii messages are now properly handled in Python 2 (`#1998`_). @@ -23,9 +23,17 @@ * Fix teardown error message in generated xUnit XML. Thanks `@gdyuldin`_ or the PR. +* Properly handle exceptions in ``multiprocessing`` tasks (`#1984`_). + Thanks `@adborden`_ for the report and `@nicoddemus`_ for the PR. + +* + +* + * +.. _@adborden: https://github.com/adborden .. _@cwitty: https://github.com/cwitty .. _@okulynyak: https://github.com/okulynyak .. _@matclab: https://github.com/matclab @@ -33,6 +41,7 @@ .. _#442: https://github.com/pytest-dev/pytest/issues/442 .. _#1976: https://github.com/pytest-dev/pytest/issues/1976 +.. _#1984: https://github.com/pytest-dev/pytest/issues/1984 .. _#1998: https://github.com/pytest-dev/pytest/issues/1998 .. _#2004: https://github.com/pytest-dev/pytest/issues/2004 .. _#2005: https://github.com/pytest-dev/pytest/issues/2005 diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py index 30e12940b..416ee0b1b 100644 --- a/_pytest/_code/code.py +++ b/_pytest/_code/code.py @@ -623,16 +623,23 @@ class FormattedExcinfo(object): e = excinfo.value descr = None while e is not None: - reprtraceback = self.repr_traceback(excinfo) - reprcrash = excinfo._getreprcrash() + if excinfo: + reprtraceback = self.repr_traceback(excinfo) + reprcrash = excinfo._getreprcrash() + else: + # fallback to native repr if the exception doesn't have a traceback: + # ExceptionInfo objects require a full traceback to work + reprtraceback = ReprTracebackNative(py.std.traceback.format_exception(type(e), e, None)) + reprcrash = None + repr_chain += [(reprtraceback, reprcrash, descr)] if e.__cause__ is not None: e = e.__cause__ - excinfo = ExceptionInfo((type(e), e, e.__traceback__)) + excinfo = ExceptionInfo((type(e), e, e.__traceback__)) if e.__traceback__ else None descr = 'The above exception was the direct cause of the following exception:' elif e.__context__ is not None: e = e.__context__ - excinfo = ExceptionInfo((type(e), e, e.__traceback__)) + excinfo = ExceptionInfo((type(e), e, e.__traceback__)) if e.__traceback__ else None descr = 'During handling of the above exception, another exception occurred:' else: e = None diff --git a/testing/code/test_excinfo.py b/testing/code/test_excinfo.py index 283f8eb76..3aae9c71c 100644 --- a/testing/code/test_excinfo.py +++ b/testing/code/test_excinfo.py @@ -1050,6 +1050,50 @@ raise ValueError() assert line.endswith('mod.py') assert tw.lines[47] == ":15: AttributeError" + @pytest.mark.skipif("sys.version_info[0] < 3") + @pytest.mark.parametrize('reason, description', [ + ('cause', 'The above exception was the direct cause of the following exception:'), + ('context', 'During handling of the above exception, another exception occurred:'), + ]) + def test_exc_chain_repr_without_traceback(self, importasmod, reason, description): + """ + Handle representation of exception chains where one of the exceptions doesn't have a + real traceback, such as those raised in a subprocess submitted by the multiprocessing + module (#1984). + """ + from _pytest.pytester import LineMatcher + exc_handling_code = ' from e' if reason == 'cause' else '' + mod = importasmod(""" + def f(): + try: + g() + except Exception as e: + raise RuntimeError('runtime problem'){exc_handling_code} + def g(): + raise ValueError('invalid value') + """.format(exc_handling_code=exc_handling_code)) + + with pytest.raises(RuntimeError) as excinfo: + mod.f() + + # emulate the issue described in #1984 + attr = '__%s__' % reason + getattr(excinfo.value, attr).__traceback__ = None + + r = excinfo.getrepr() + tw = py.io.TerminalWriter(stringio=True) + tw.hasmarkup = False + r.toterminal(tw) + + matcher = LineMatcher(tw.stringio.getvalue().splitlines()) + matcher.fnmatch_lines([ + "ValueError: invalid value", + description, + "* except Exception as e:", + "> * raise RuntimeError('runtime problem')" + exc_handling_code, + "E *RuntimeError: runtime problem", + ]) + @pytest.mark.parametrize("style", ["short", "long"]) @pytest.mark.parametrize("encoding", [None, "utf8", "utf16"]) diff --git a/testing/test_assertion.py b/testing/test_assertion.py index 2d4761431..48cd26f02 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -749,6 +749,37 @@ def test_traceback_failure(testdir): "*test_traceback_failure.py:4: AssertionError" ]) + +@pytest.mark.skipif(sys.version_info[:2] <= (3, 3), reason='Python 3.4+ shows chained exceptions on multiprocess') +def test_exception_handling_no_traceback(testdir): + """ + Handle chain exceptions in tasks submitted by the multiprocess module (#1984). + """ + p1 = testdir.makepyfile(""" + from multiprocessing import Pool + + def process_task(n): + assert n == 10 + + def multitask_job(): + tasks = [1] + with Pool(processes=1) as pool: + pool.map(process_task, tasks) + + def test_multitask_job(): + multitask_job() + """) + result = testdir.runpytest(p1, "--tb=long") + result.stdout.fnmatch_lines([ + "====* FAILURES *====", + "*multiprocessing.pool.RemoteTraceback:*", + "Traceback (most recent call last):", + "*assert n == 10", + "The above exception was the direct cause of the following exception:", + "> * multitask_job()", + ]) + + @pytest.mark.skipif("'__pypy__' in sys.builtin_module_names or sys.platform.startswith('java')" ) def test_warn_missing(testdir): testdir.makepyfile("")