From aac371cf077b1b5094ba4fbdd3506f2bdaea96d8 Mon Sep 17 00:00:00 2001 From: Eric Hunsberger Date: Tue, 28 Jul 2015 19:04:46 -0400 Subject: [PATCH 1/2] Added myself to AUTHORS --- AUTHORS | 1 + CONTRIBUTING.rst | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/AUTHORS b/AUTHORS index f5cb30bb9..f8b46f94c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -28,6 +28,7 @@ Eduardo Schettino Eric Siegerman Florian Bruhin Edison Gustavo Muenz +Eric Hunsberger Floris Bruynooghe Graham Horler Grig Gheorghiu diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 7b24cd9e7..be33e4fb4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -191,6 +191,10 @@ but here is a simple overview: $ git commit -a -m "" $ git push -u + Make sure you add a CHANGELOG message, and add yourself to AUTHORS. If you + are unsure about either of these steps, submit your pull request and we'll + help you fix it up. + #. Finally, submit a pull request through the GitHub website: .. image:: img/pullrequest.png From 52b4eb6c46e6b8b3f4be6163198f34aef5a30366 Mon Sep 17 00:00:00 2001 From: Eric Hunsberger Date: Tue, 28 Jul 2015 19:01:11 -0400 Subject: [PATCH 2/2] Added `warns` to assert warnings are thrown Works in a similar manner to `raises`, but for warnings instead of exceptions. Also refactored `recwarn.py` so that all the warning recording and checking use the same core code. --- CHANGELOG | 3 + _pytest/python.py | 29 +++--- _pytest/recwarn.py | 204 ++++++++++++++++++++++++++++----------- doc/en/assert.txt | 10 ++ doc/en/recwarn.txt | 110 +++++++++++++++++---- testing/python/raises.py | 1 - testing/test_recwarn.py | 176 ++++++++++++++++++++++++--------- 7 files changed, 394 insertions(+), 139 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 6ae0a7864..af4cf08c4 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,9 @@ 2.8.0.dev (compared to 2.7.X) ----------------------------- +- Add 'warns' to assert that warnings are thrown (like 'raises'). + Thanks to Eric Hunsberger for the PR. + - Fix #683: Do not apply an already applied mark. Thanks ojake for the PR. - Deal with capturing failures better so fewer exceptions get lost to diff --git a/_pytest/python.py b/_pytest/python.py index 42da4d073..fe93c938e 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1052,8 +1052,8 @@ def getlocation(function, curdir): # builtin pytest.raises helper -def raises(ExpectedException, *args, **kwargs): - """ assert that a code block/function call raises @ExpectedException +def raises(expected_exception, *args, **kwargs): + """ assert that a code block/function call raises @expected_exception and raise a failure exception otherwise. This helper produces a ``py.code.ExceptionInfo()`` object. @@ -1101,23 +1101,23 @@ def raises(ExpectedException, *args, **kwargs): """ __tracebackhide__ = True - if ExpectedException is AssertionError: + if expected_exception is AssertionError: # we want to catch a AssertionError # replace our subclass with the builtin one # see https://github.com/pytest-dev/pytest/issues/176 from _pytest.assertion.util import BuiltinAssertionError \ - as ExpectedException + as expected_exception msg = ("exceptions must be old-style classes or" " derived from BaseException, not %s") - if isinstance(ExpectedException, tuple): - for exc in ExpectedException: + if isinstance(expected_exception, tuple): + for exc in expected_exception: if not inspect.isclass(exc): raise TypeError(msg % type(exc)) - elif not inspect.isclass(ExpectedException): - raise TypeError(msg % type(ExpectedException)) + elif not inspect.isclass(expected_exception): + raise TypeError(msg % type(expected_exception)) if not args: - return RaisesContext(ExpectedException) + return RaisesContext(expected_exception) elif isinstance(args[0], str): code, = args assert isinstance(code, str) @@ -1130,19 +1130,19 @@ def raises(ExpectedException, *args, **kwargs): py.builtin.exec_(code, frame.f_globals, loc) # XXX didn'T mean f_globals == f_locals something special? # this is destroyed here ... - except ExpectedException: + except expected_exception: return py.code.ExceptionInfo() else: func = args[0] try: func(*args[1:], **kwargs) - except ExpectedException: + except expected_exception: return py.code.ExceptionInfo() pytest.fail("DID NOT RAISE") class RaisesContext(object): - def __init__(self, ExpectedException): - self.ExpectedException = ExpectedException + def __init__(self, expected_exception): + self.expected_exception = expected_exception self.excinfo = None def __enter__(self): @@ -1161,7 +1161,7 @@ class RaisesContext(object): exc_type, value, traceback = tp tp = exc_type, exc_type(value), traceback self.excinfo.__init__(tp) - return issubclass(self.excinfo.type, self.ExpectedException) + return issubclass(self.excinfo.type, self.expected_exception) # # the basic pytest Function item @@ -2123,4 +2123,3 @@ def get_scope_node(node, scope): return node.session raise ValueError("unknown scope") return node.getparent(cls) - diff --git a/_pytest/recwarn.py b/_pytest/recwarn.py index 875cb510e..bddb89ea8 100644 --- a/_pytest/recwarn.py +++ b/_pytest/recwarn.py @@ -1,10 +1,14 @@ """ recording warnings during test function execution. """ +import inspect +import py import sys import warnings +import pytest -def pytest_funcarg__recwarn(request): +@pytest.yield_fixture +def recwarn(request): """Return a WarningsRecorder instance that provides these methods: * ``pop(category=None)``: return last warning matching the category. @@ -13,83 +17,169 @@ def pytest_funcarg__recwarn(request): See http://docs.python.org/library/warnings.html for information on warning categories. """ - if sys.version_info >= (2,7): - oldfilters = warnings.filters[:] - warnings.simplefilter('default') - def reset_filters(): - warnings.filters[:] = oldfilters - request.addfinalizer(reset_filters) wrec = WarningsRecorder() - request.addfinalizer(wrec.finalize) - return wrec + with wrec: + warnings.simplefilter('default') + yield wrec + def pytest_namespace(): - return {'deprecated_call': deprecated_call} + return {'deprecated_call': deprecated_call, + 'warns': warns} + def deprecated_call(func, *args, **kwargs): - """ assert that calling ``func(*args, **kwargs)`` - triggers a DeprecationWarning. + """Assert that ``func(*args, **kwargs)`` triggers a DeprecationWarning. """ - l = [] - oldwarn_explicit = getattr(warnings, 'warn_explicit') - def warn_explicit(*args, **kwargs): - l.append(args) - oldwarn_explicit(*args, **kwargs) - oldwarn = getattr(warnings, 'warn') - def warn(*args, **kwargs): - l.append(args) - oldwarn(*args, **kwargs) - - warnings.warn_explicit = warn_explicit - warnings.warn = warn - try: + wrec = WarningsRecorder() + with wrec: + warnings.simplefilter('always') # ensure all warnings are triggered ret = func(*args, **kwargs) - finally: - warnings.warn_explicit = oldwarn_explicit - warnings.warn = oldwarn - if not l: + + if not any(r.category is DeprecationWarning for r in wrec): __tracebackhide__ = True - raise AssertionError("%r did not produce DeprecationWarning" %(func,)) + raise AssertionError("%r did not produce DeprecationWarning" % (func,)) + return ret -class RecordedWarning: - def __init__(self, message, category, filename, lineno, line): +def warns(expected_warning, *args, **kwargs): + """Assert that code raises a particular class of warning. + + Specifically, the input @expected_warning can be a warning class or + tuple of warning classes, and the code must return that warning + (if a single class) or one of those warnings (if a tuple). + + This helper produces a list of ``warnings.WarningMessage`` objects, + one for each warning raised. + + This function can be used as a context manager, or any of the other ways + ``pytest.raises`` can be used:: + + >>> with warns(RuntimeWarning): + ... warnings.warn("my warning", RuntimeWarning) + """ + wcheck = WarningsChecker(expected_warning) + if not args: + return wcheck + elif isinstance(args[0], str): + code, = args + assert isinstance(code, str) + frame = sys._getframe(1) + loc = frame.f_locals.copy() + loc.update(kwargs) + + with wcheck: + code = py.code.Source(code).compile() + py.builtin.exec_(code, frame.f_globals, loc) + else: + func = args[0] + with wcheck: + return func(*args[1:], **kwargs) + + +class RecordedWarning(object): + def __init__(self, message, category, filename, lineno, file, line): self.message = message self.category = category self.filename = filename self.lineno = lineno + self.file = file self.line = line -class WarningsRecorder: - def __init__(self): - self.list = [] - def showwarning(message, category, filename, lineno, line=0): - self.list.append(RecordedWarning( - message, category, filename, lineno, line)) - try: - self.old_showwarning(message, category, - filename, lineno, line=line) - except TypeError: - # < python2.6 - self.old_showwarning(message, category, filename, lineno) - self.old_showwarning = warnings.showwarning - warnings.showwarning = showwarning + +class WarningsRecorder(object): + """A context manager to record raised warnings. + + Adapted from `warnings.catch_warnings`. + """ + + def __init__(self, module=None): + self._module = sys.modules['warnings'] if module is None else module + self._entered = False + self._list = [] + + @property + def list(self): + """The list of recorded warnings.""" + return self._list + + def __getitem__(self, i): + """Get a recorded warning by index.""" + return self._list[i] + + def __iter__(self): + """Iterate through the recorded warnings.""" + return iter(self._list) + + def __len__(self): + """The number of recorded warnings.""" + return len(self._list) def pop(self, cls=Warning): - """ pop the first recorded warning, raise exception if not exists.""" - for i, w in enumerate(self.list): + """Pop the first recorded warning, raise exception if not exists.""" + for i, w in enumerate(self._list): if issubclass(w.category, cls): - return self.list.pop(i) + return self._list.pop(i) __tracebackhide__ = True - assert 0, "%r not found in %r" %(cls, self.list) - - #def resetregistry(self): - # warnings.onceregistry.clear() - # warnings.__warningregistry__.clear() + raise AssertionError("%r not found in warning list" % cls) def clear(self): - self.list[:] = [] + """Clear the list of recorded warnings.""" + self._list[:] = [] - def finalize(self): - warnings.showwarning = self.old_showwarning + def __enter__(self): + if self._entered: + __tracebackhide__ = True + raise RuntimeError("Cannot enter %r twice" % self) + self._entered = True + self._filters = self._module.filters + self._module.filters = self._filters[:] + self._showwarning = self._module.showwarning + + def showwarning(message, category, filename, lineno, + file=None, line=None): + self._list.append(RecordedWarning( + message, category, filename, lineno, file, line)) + + # still perform old showwarning functionality + self._showwarning(message, category, filename, lineno, + file=file, line=line) + + self._module.showwarning = showwarning + return self + + def __exit__(self, *exc_info): + if not self._entered: + __tracebackhide__ = True + raise RuntimeError("Cannot exit %r without entering first" % self) + self._module.filters = self._filters + self._module.showwarning = self._showwarning + + +class WarningsChecker(WarningsRecorder): + def __init__(self, expected_warning=None, module=None): + super(WarningsChecker, self).__init__(module=module) + + msg = ("exceptions must be old-style classes or " + "derived from Warning, not %s") + if isinstance(expected_warning, tuple): + for exc in expected_warning: + if not inspect.isclass(exc): + raise TypeError(msg % type(exc)) + elif inspect.isclass(expected_warning): + expected_warning = (expected_warning,) + elif expected_warning is not None: + raise TypeError(msg % type(expected_warning)) + + self.expected_warning = expected_warning + + def __exit__(self, *exc_info): + super(WarningsChecker, self).__exit__(*exc_info) + + # only check if we're not currently handling an exception + if all(a is None for a in exc_info): + if self.expected_warning is not None: + if not any(r.category in self.expected_warning for r in self): + __tracebackhide__ = True + pytest.fail("DID NOT WARN") diff --git a/doc/en/assert.txt b/doc/en/assert.txt index aed43803f..57a748485 100644 --- a/doc/en/assert.txt +++ b/doc/en/assert.txt @@ -114,6 +114,16 @@ like documenting unfixed bugs (where the test describes what "should" happen) or bugs in dependencies. +.. _`assertwarns`: + +Assertions about expected warnings +----------------------------------------- + +.. versionadded:: 2.8 + +You can check that code raises a particular warning using +:ref:`pytest.warns `. + .. _newreport: diff --git a/doc/en/recwarn.txt b/doc/en/recwarn.txt index c07a2cbe7..c2a1e65fa 100644 --- a/doc/en/recwarn.txt +++ b/doc/en/recwarn.txt @@ -1,35 +1,91 @@ -Asserting deprecation and other warnings +Asserting Warnings ===================================================== -.. _function_argument: +.. _warns: -The recwarn function argument ------------------------------------- +Asserting warnings with the warns function +----------------------------------------------- -You can use the ``recwarn`` funcarg to assert that code triggers -warnings through the Python warnings system. Here is a simple -self-contained test:: +.. versionadded:: 2.8 + +You can check that code raises a particular warning using ``pytest.warns``, +which works in a similar manner to :ref:`raises `:: + + import warnings + import pytest + + def test_warning(): + with pytest.warns(UserWarning): + warnings.warn("my warning", UserWarning) + +The test will fail if the warning in question is not raised. + +You can also call ``pytest.warns`` on a function or code string:: + + pytest.warns(expected_warning, func, *args, **kwargs) + pytest.warns(expected_warning, "func(*args, **kwargs)") + +The function also returns a list of all raised warnings (as +``warnings.WarningMessage`` objects), which you can query for +additional information:: + + with pytest.warns(RuntimeWarning) as record: + warnings.warn("another warning", RuntimeWarning) + + # check that only one warning was raised + assert len(record) == 1 + # check that the message matches + assert record[0].message.args[0] == "another warning" + +Alternatively, you can examine raised warnings in detail using the +:ref:`recwarn ` fixture (see below). + +.. _recwarn: + +Recording warnings +------------------------ + +You can record raised warnings either using ``pytest.warns`` or with +the ``recwarn`` fixture. + +To record with ``pytest.warns`` without asserting anything about the warnings, +pass ``None`` as the expected warning type:: + + with pytest.warns(None) as record: + warnings.warn("user", UserWarning) + warnings.warn("runtime", RuntimeWarning) + + assert len(record) == 2 + assert str(record[0].message) == "user" + assert str(record[1].message) == "runtime" + +The ``recwarn`` fixture will record warnings for the whole function:: + + import warnings - # content of test_recwarn.py def test_hello(recwarn): - from warnings import warn - warn("hello", DeprecationWarning) - w = recwarn.pop(DeprecationWarning) - assert issubclass(w.category, DeprecationWarning) - assert 'hello' in str(w.message) + warnings.warn("hello", UserWarning) + assert len(recwarn) == 1 + w = recwarn.pop(UserWarning) + assert issubclass(w.category, UserWarning) + assert str(w.message) == "hello" assert w.filename assert w.lineno -The ``recwarn`` function argument provides these methods: +Both ``recwarn`` and ``pytest.warns`` return the same interface for recorded +warnings: a WarningsRecorder instance. To view the recorded warnings, you can +iterate over this instance, call ``len`` on it to get the number of recorded +warnings, or index into it to get a particular recorded warning. It also +provides these methods: -.. method:: pop(category=None) +.. autoclass:: _pytest.recwarn.WarningsRecorder() + :members: - Return last warning matching the category. - -.. method:: clear() - - Clear list of warnings +Each recorded warning has the attributes ``message``, ``category``, +``filename``, ``lineno``, ``file``, and ``line``. The ``category`` is the +class of the warning. The ``message`` is the warning itself; calling +``str(message)`` will return the actual message of the warning. .. _ensuring_function_triggers: @@ -44,3 +100,17 @@ that a certain function call triggers a ``DeprecationWarning``:: def test_global(): pytest.deprecated_call(myfunction, 17) + +By default, deprecation warnings will not be caught when using ``pytest.warns`` +or ``recwarn``, since the default Python warnings filters hide +DeprecationWarnings. If you wish to record them in your own code, use the +command ``warnings.simplefilter('always')``:: + + import warnings + import pytest + + def test_deprecation(recwarn): + warnings.simplefilter('always') + warnings.warn("deprecated", DeprecationWarning) + assert len(recwarn) == 1 + assert recwarn.pop(DeprecationWarning) diff --git a/testing/python/raises.py b/testing/python/raises.py index 5ba56bb71..edeb52226 100644 --- a/testing/python/raises.py +++ b/testing/python/raises.py @@ -34,7 +34,6 @@ class TestRaises: raise BuiltinAssertionError """) - @pytest.mark.skipif('sys.version < "2.5"') def test_raises_as_contextmanager(self, testdir): testdir.makepyfile(""" from __future__ import with_statement diff --git a/testing/test_recwarn.py b/testing/test_recwarn.py index 21909ae2a..ad89c9cd4 100644 --- a/testing/test_recwarn.py +++ b/testing/test_recwarn.py @@ -1,24 +1,8 @@ -import py, pytest -from _pytest.recwarn import WarningsRecorder +import warnings +import py +import pytest +from _pytest.recwarn import WarningsChecker, WarningsRecorder -def test_WarningRecorder(recwarn): - showwarning = py.std.warnings.showwarning - rec = WarningsRecorder() - assert py.std.warnings.showwarning != showwarning - assert not rec.list - py.std.warnings.warn_explicit("hello", UserWarning, "xyz", 13) - assert len(rec.list) == 1 - py.std.warnings.warn(DeprecationWarning("hello")) - assert len(rec.list) == 2 - warn = rec.pop() - assert str(warn.message) == "hello" - l = rec.list - rec.clear() - assert len(rec.list) == 0 - assert l is rec.list - pytest.raises(AssertionError, "rec.pop()") - rec.finalize() - assert showwarning == py.std.warnings.showwarning def test_recwarn_functional(testdir): reprec = testdir.inline_runsource(""" @@ -35,6 +19,49 @@ def test_recwarn_functional(testdir): res = reprec.countoutcomes() assert tuple(res) == (2, 0, 0), res + +class TestWarningsRecorderChecker(object): + def test_recording(self, recwarn): + showwarning = py.std.warnings.showwarning + rec = WarningsRecorder() + with rec: + assert py.std.warnings.showwarning != showwarning + assert not rec.list + py.std.warnings.warn_explicit("hello", UserWarning, "xyz", 13) + assert len(rec.list) == 1 + py.std.warnings.warn(DeprecationWarning("hello")) + assert len(rec.list) == 2 + warn = rec.pop() + assert str(warn.message) == "hello" + l = rec.list + rec.clear() + assert len(rec.list) == 0 + assert l is rec.list + pytest.raises(AssertionError, "rec.pop()") + + assert showwarning == py.std.warnings.showwarning + + def test_typechecking(self): + with pytest.raises(TypeError): + WarningsChecker(5) + with pytest.raises(TypeError): + WarningsChecker(('hi', RuntimeWarning)) + with pytest.raises(TypeError): + WarningsChecker([DeprecationWarning, RuntimeWarning]) + + def test_invalid_enter_exit(self): + # wrap this test in WarningsRecorder to ensure warning state gets reset + with WarningsRecorder(): + with pytest.raises(RuntimeError): + rec = WarningsRecorder() + rec.__exit__(None, None, None) # can't exit before entering + + with pytest.raises(RuntimeError): + rec = WarningsRecorder() + with rec: + with rec: + pass # can't enter twice + # # ============ test pytest.deprecated_call() ============== # @@ -50,35 +77,92 @@ def dep_explicit(i): py.std.warnings.warn_explicit("dep_explicit", category=DeprecationWarning, filename="hello", lineno=3) -def test_deprecated_call_raises(): - excinfo = pytest.raises(AssertionError, - "pytest.deprecated_call(dep, 3)") - assert str(excinfo).find("did not produce") != -1 +class TestDeprecatedCall(object): + def test_deprecated_call_raises(self): + excinfo = pytest.raises(AssertionError, + "pytest.deprecated_call(dep, 3)") + assert str(excinfo).find("did not produce") != -1 -def test_deprecated_call(): - pytest.deprecated_call(dep, 0) + def test_deprecated_call(self): + pytest.deprecated_call(dep, 0) -def test_deprecated_call_ret(): - ret = pytest.deprecated_call(dep, 0) - assert ret == 42 + def test_deprecated_call_ret(self): + ret = pytest.deprecated_call(dep, 0) + assert ret == 42 -def test_deprecated_call_preserves(): - onceregistry = py.std.warnings.onceregistry.copy() - filters = py.std.warnings.filters[:] - warn = py.std.warnings.warn - warn_explicit = py.std.warnings.warn_explicit - test_deprecated_call_raises() - test_deprecated_call() - assert onceregistry == py.std.warnings.onceregistry - assert filters == py.std.warnings.filters - assert warn is py.std.warnings.warn - assert warn_explicit is py.std.warnings.warn_explicit + def test_deprecated_call_preserves(self): + onceregistry = py.std.warnings.onceregistry.copy() + filters = py.std.warnings.filters[:] + warn = py.std.warnings.warn + warn_explicit = py.std.warnings.warn_explicit + self.test_deprecated_call_raises() + self.test_deprecated_call() + assert onceregistry == py.std.warnings.onceregistry + assert filters == py.std.warnings.filters + assert warn is py.std.warnings.warn + assert warn_explicit is py.std.warnings.warn_explicit -def test_deprecated_explicit_call_raises(): - pytest.raises(AssertionError, - "pytest.deprecated_call(dep_explicit, 3)") + def test_deprecated_explicit_call_raises(self): + pytest.raises(AssertionError, + "pytest.deprecated_call(dep_explicit, 3)") -def test_deprecated_explicit_call(): - pytest.deprecated_call(dep_explicit, 0) - pytest.deprecated_call(dep_explicit, 0) + def test_deprecated_explicit_call(self): + pytest.deprecated_call(dep_explicit, 0) + pytest.deprecated_call(dep_explicit, 0) + +class TestWarns(object): + def test_strings(self): + # different messages, b/c Python suppresses multiple identical warnings + source1 = "warnings.warn('w1', RuntimeWarning)" + source2 = "warnings.warn('w2', RuntimeWarning)" + source3 = "warnings.warn('w3', RuntimeWarning)" + pytest.warns(RuntimeWarning, source1) + pytest.raises(pytest.fail.Exception, + lambda: pytest.warns(UserWarning, source2)) + pytest.warns(RuntimeWarning, source3) + + def test_function(self): + pytest.warns(SyntaxWarning, + lambda msg: warnings.warn(msg, SyntaxWarning), "syntax") + + def test_warning_tuple(self): + pytest.warns((RuntimeWarning, SyntaxWarning), + lambda: warnings.warn('w1', RuntimeWarning)) + pytest.warns((RuntimeWarning, SyntaxWarning), + lambda: warnings.warn('w2', SyntaxWarning)) + pytest.raises(pytest.fail.Exception, + lambda: pytest.warns( + (RuntimeWarning, SyntaxWarning), + lambda: warnings.warn('w3', UserWarning))) + + def test_as_contextmanager(self): + with pytest.warns(RuntimeWarning): + warnings.warn("runtime", RuntimeWarning) + + with pytest.raises(pytest.fail.Exception): + with pytest.warns(RuntimeWarning): + warnings.warn("user", UserWarning) + + with pytest.raises(pytest.fail.Exception): + with pytest.warns(UserWarning): + warnings.warn("runtime", RuntimeWarning) + + with pytest.warns(UserWarning): + warnings.warn("user", UserWarning) + + def test_record(self): + with pytest.warns(UserWarning) as record: + warnings.warn("user", UserWarning) + + assert len(record) == 1 + assert str(record[0].message) == "user" + + def test_record_only(self): + with pytest.warns(None) as record: + warnings.warn("user", UserWarning) + warnings.warn("runtime", RuntimeWarning) + + assert len(record) == 2 + assert str(record[0].message) == "user" + assert str(record[1].message) == "runtime"