Merge pull request #1586 from nicoddemus/issue-1461-merge-yield-fixture

Make normal fixtures work with "yield"

CHANGELOG.rst

@@ -36,6 +36,12 @@
 
 **Changes**
 
+* Fixtures marked with ``@pytest.fixture`` can now use ``yield`` statements exactly like
+  those marked with the ``@pytest.yield_fixture`` decorator. This change renders
+  ``@pytest.yield_fixture`` deprecated and makes ``@pytest.fixture`` with ``yield`` statements
+  the preferred way to write teardown code (`#1461`_).
+  Thanks `@csaftoiu`_ for bringing this to attention and `@nicoddemus`_ for the PR.
+
 * Fix (`#1351`_):
   explicitly passed parametrize ids do not get escaped to ascii.
   Thanks `@ceridwen`_ for the PR.
@@ -58,6 +64,7 @@
 *
 
 .. _@milliams: https://github.com/milliams
+.. _@csaftoiu: https://github.com/csaftoiu
 .. _@novas0x2a: https://github.com/novas0x2a
 .. _@kalekundert: https://github.com/kalekundert
 .. _@tareqalayan: https://github.com/tareqalayan
@@ -72,6 +79,7 @@
 .. _#1441: https://github.com/pytest-dev/pytest/pull/1441
 .. _#1454: https://github.com/pytest-dev/pytest/pull/1454
 .. _#1351: https://github.com/pytest-dev/pytest/issues/1351
+.. _#1461: https://github.com/pytest-dev/pytest/pull/1461
 .. _#1468: https://github.com/pytest-dev/pytest/pull/1468
 .. _#1474: https://github.com/pytest-dev/pytest/pull/1474
 .. _#1502: https://github.com/pytest-dev/pytest/pull/1502

_pytest/python.py

@@ -116,12 +116,10 @@ def safe_getattr(object, name, default):
 
 
 class FixtureFunctionMarker:
-    def __init__(self, scope, params,
+    def __init__(self, scope, params, autouse=False, ids=None, name=None):
-                 autouse=False, yieldctx=False, ids=None, name=None):
         self.scope = scope
         self.params = params
         self.autouse = autouse
-        self.yieldctx = yieldctx
         self.ids = ids
         self.name = name
 
@@ -166,6 +164,10 @@ def fixture(scope="function", params=None, autouse=False, ids=None, name=None):
                to resolve this is to name the decorated function
                ``fixture_<fixturename>`` and then use
                ``@pytest.fixture(name='<fixturename>')``.
+
+    Fixtures can optionally provide their values to test functions using a ``yield`` statement,
+    instead of ``return``. In this case, the code block after the ``yield`` statement is executed
+    as teardown code regardless of the test outcome. A fixture function must yield exactly once.
     """
     if callable(scope) and params is None and autouse == False:
         # direct decoration
@@ -175,22 +177,19 @@ def fixture(scope="function", params=None, autouse=False, ids=None, name=None):
         params = list(params)
     return FixtureFunctionMarker(scope, params, autouse, ids=ids, name=name)
 
-def yield_fixture(scope="function", params=None, autouse=False, ids=None):
-    """ (return a) decorator to mark a yield-fixture factory function
-    (EXPERIMENTAL).
 
-    This takes the same arguments as :py:func:`pytest.fixture` but
+def yield_fixture(scope="function", params=None, autouse=False, ids=None, name=None):
-    expects a fixture function to use a ``yield`` instead of a ``return``
+    """ (return a) decorator to mark a yield-fixture factory function.
-    statement to provide a fixture.  See
+
-    http://pytest.org/en/latest/yieldfixture.html for more info.
+    .. deprecated:: 1.10
+        Use :py:func:`pytest.fixture` directly instead.
     """
-    if callable(scope) and params is None and autouse == False:
+    if callable(scope) and params is None and not autouse:
         # direct decoration
         return FixtureFunctionMarker(
-                "function", params, autouse, yieldctx=True)(scope)
+                "function", params, autouse, ids=ids, name=name)(scope)
     else:
-        return FixtureFunctionMarker(scope, params, autouse,
+        return FixtureFunctionMarker(scope, params, autouse, ids=ids, name=name)
-                                     yieldctx=True, ids=ids)
 
 defaultfuncargprefixmarker = fixture()
 
@@ -2287,7 +2286,6 @@ class FixtureManager:
                 assert not name.startswith(self._argprefix)
             fixturedef = FixtureDef(self, nodeid, name, obj,
                                     marker.scope, marker.params,
-                                    yieldctx=marker.yieldctx,
                                     unittest=unittest, ids=marker.ids)
             faclist = self._arg2fixturedefs.setdefault(name, [])
             if fixturedef.has_location:
@@ -2325,38 +2323,30 @@ def fail_fixturefunc(fixturefunc, msg):
     pytest.fail(msg + ":\n\n" + str(source.indent()) + "\n" + location,
                 pytrace=False)
 
-def call_fixture_func(fixturefunc, request, kwargs, yieldctx):
+def call_fixture_func(fixturefunc, request, kwargs):
+    yieldctx = is_generator(fixturefunc)
     if yieldctx:
-        if not is_generator(fixturefunc):
+        it = fixturefunc(**kwargs)
-            fail_fixturefunc(fixturefunc,
+        res = next(it)
-                msg="yield_fixture requires yield statement in function")
+
-        iter = fixturefunc(**kwargs)
-        next = getattr(iter, "__next__", None)
-        if next is None:
-            next = getattr(iter, "next")
-        res = next()
         def teardown():
             try:
-                next()
+                next(it)
             except StopIteration:
                 pass
             else:
                 fail_fixturefunc(fixturefunc,
                     "yield_fixture function has more than one 'yield'")
+
         request.addfinalizer(teardown)
     else:
-        if is_generator(fixturefunc):
-            fail_fixturefunc(fixturefunc,
-                msg="pytest.fixture functions cannot use ``yield``. "
-                    "Instead write and return an inner function/generator "
-                    "and let the consumer call and iterate over it.")
         res = fixturefunc(**kwargs)
     return res
 
 class FixtureDef:
     """ A container for a factory definition. """
     def __init__(self, fixturemanager, baseid, argname, func, scope, params,
-                 yieldctx, unittest=False, ids=None):
+                 unittest=False, ids=None):
         self._fixturemanager = fixturemanager
         self.baseid = baseid or ''
         self.has_location = baseid is not None
@@ -2367,7 +2357,6 @@ class FixtureDef:
         self.params = params
         startindex = unittest and 1 or None
         self.argnames = getfuncargnames(func, startindex=startindex)
-        self.yieldctx = yieldctx
         self.unittest = unittest
         self.ids = ids
         self._finalizer = []
@@ -2428,8 +2417,7 @@ class FixtureDef:
                     fixturefunc = fixturefunc.__get__(request.instance)
 
         try:
-            result = call_fixture_func(fixturefunc, request, kwargs,
+            result = call_fixture_func(fixturefunc, request, kwargs)
-                                       self.yieldctx)
         except Exception:
             self.cached_result = (None, my_cache_key, sys.exc_info())
             raise

doc/en/example/costlysetup/conftest.py

@@ -4,8 +4,8 @@ import pytest
 @pytest.fixture("session")
 def setup(request):
     setup = CostlySetup()
-    request.addfinalizer(setup.finalize)
+    yield setup
-    return setup
+    setup.finalize()
 
 class CostlySetup:
     def __init__(self):

doc/en/example/simple.rst

@@ -648,15 +648,14 @@ here is a little example implemented via a local plugin::
 
     @pytest.fixture
     def something(request):
-        def fin():
+        yield
-            # request.node is an "item" because we use the default
+        # request.node is an "item" because we use the default
-            # "function" scope
+        # "function" scope
-            if request.node.rep_setup.failed:
+        if request.node.rep_setup.failed:
-                print ("setting up a test failed!", request.node.nodeid)
+            print ("setting up a test failed!", request.node.nodeid)
-            elif request.node.rep_setup.passed:
+        elif request.node.rep_setup.passed:
-                if request.node.rep_call.failed:
+            if request.node.rep_call.failed:
-                    print ("executing test failed", request.node.nodeid)
+                print ("executing test failed", request.node.nodeid)
-        request.addfinalizer(fin)
 
 
 if you then have failing tests::

doc/en/fixture.rst

@@ -34,11 +34,6 @@ both styles, moving incrementally from classic to new style, as you
 prefer.  You can also start out from existing :ref:`unittest.TestCase
 style <unittest.TestCase>` or :ref:`nose based <nosestyle>` projects.
 
-.. note::
-
-    pytest-2.4 introduced an additional :ref:`yield fixture mechanism
-    <yieldfixture>` for easier context manager integration and more linear
-    writing of teardown code.
 
 .. _`funcargs`:
 .. _`funcarg mechanism`:
@@ -247,9 +242,8 @@ Fixture finalization / executing teardown code
 -------------------------------------------------------------
 
 pytest supports execution of fixture specific finalization code
-when the fixture goes out of scope.  By accepting a ``request`` object
+when the fixture goes out of scope.  By using a ``yield`` statement instead of ``return``, all
-into your fixture function you can call its ``request.addfinalizer`` one
+the code after the *yield* statement serves as the teardown code.::
-or multiple times::
 
     # content of conftest.py
 
@@ -259,14 +253,12 @@ or multiple times::
     @pytest.fixture(scope="module")
     def smtp(request):
         smtp = smtplib.SMTP("smtp.gmail.com")
-        def fin():
+        yield smtp  # provide the fixture value
-            print ("teardown smtp")
+        print("teardown smtp")
-            smtp.close()
+        smtp.close()
-        request.addfinalizer(fin)
-        return smtp  # provide the fixture value
 
-The ``fin`` function will execute when the last test using
+The ``print`` and ``smtp.close()`` statements will execute when the last test using
-the fixture in the module has finished execution.
+the fixture in the module has finished execution, regardless of the exception status of the tests.
 
 Let's execute it::
 
@@ -282,14 +274,55 @@ occur around each single test.  In either case the test
 module itself does not need to change or know about these details
 of fixture setup.
 
+Note that we can also seamlessly use the ``yield`` syntax with ``with`` statements::
 
-Finalization/teardown with yield fixtures
+    # content of test_yield2.py
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Another alternative to the *request.addfinalizer()* method is to use *yield
+    import pytest
-fixtures*. All the code after the *yield* statement serves as the teardown
-code. See the :ref:`yield fixture documentation <yieldfixture>`.
 
+    @pytest.fixture
+    def passwd():
+        with open("/etc/passwd") as f:
+            yield f.readlines()
+
+    def test_has_lines(passwd):
+        assert len(passwd) >= 1
+
+The file ``f`` will be closed after the test finished execution
+because the Python ``file`` object supports finalization when
+the ``with`` statement ends.
+
+
+.. note::
+    Prior to version 2.10, in order to use a ``yield`` statement to execute teardown code one
+    had to mark a fixture using the ``yield_fixture`` marker. From 2.10 onward, normal
+    fixtures can use ``yield`` directly so the ``yield_fixture`` decorator is no longer needed
+    and considered deprecated.
+
+.. note::
+    As historical note, another way to write teardown code is
+    by accepting a ``request`` object into your fixture function and can call its
+    ``request.addfinalizer`` one or multiple times::
+
+        # content of conftest.py
+
+        import smtplib
+        import pytest
+
+        @pytest.fixture(scope="module")
+        def smtp(request):
+            smtp = smtplib.SMTP("smtp.gmail.com")
+            def fin():
+                print ("teardown smtp")
+                smtp.close()
+            request.addfinalizer(fin)
+            return smtp  # provide the fixture value
+
+    The ``fin`` function will execute when the last test using
+    the fixture in the module has finished execution.
+
+    This method is still fully supported, but ``yield`` is recommended from 2.10 onward because
+    it is considered simpler and better describes the natural code flow.
 
 .. _`request-context`:
 
@@ -309,12 +342,9 @@ read an optional server URL from the test module which uses our fixture::
     def smtp(request):
         server = getattr(request.module, "smtpserver", "smtp.gmail.com")
         smtp = smtplib.SMTP(server)
-
+        yield smtp
-        def fin():
+        print ("finalizing %s (%s)" % (smtp, server))
-            print ("finalizing %s (%s)" % (smtp, server))
+        smtp.close()
-            smtp.close()
-        request.addfinalizer(fin)
-        return smtp
 
 We use the ``request.module`` attribute to optionally obtain an
 ``smtpserver`` attribute from the test module.  If we just execute
@@ -351,7 +381,7 @@ from the module namespace.
 
 .. _`fixture-parametrize`:
 
-Parametrizing a fixture
+Parametrizing fixtures
 -----------------------------------------------------------------
 
 Fixture functions can be parametrized in which case they will be called
@@ -374,11 +404,9 @@ through the special :py:class:`request <FixtureRequest>` object::
                     params=["smtp.gmail.com", "mail.python.org"])
     def smtp(request):
         smtp = smtplib.SMTP(request.param)
-        def fin():
+        yield smtp
-            print ("finalizing %s" % smtp)
+        print ("finalizing %s" % smtp)
-            smtp.close()
+        smtp.close()
-        request.addfinalizer(fin)
-        return smtp
 
 The main change is the declaration of ``params`` with
 :py:func:`@pytest.fixture <_pytest.python.fixture>`, a list of values
@@ -586,19 +614,15 @@ to show the setup/teardown flow::
     def modarg(request):
         param = request.param
         print ("  SETUP modarg %s" % param)
-        def fin():
+        yield param
-            print ("  TEARDOWN modarg %s" % param)
+        print ("  TEARDOWN modarg %s" % param)
-        request.addfinalizer(fin)
-        return param
 
     @pytest.fixture(scope="function", params=[1,2])
     def otherarg(request):
         param = request.param
         print ("  SETUP otherarg %s" % param)
-        def fin():
+        yield param
-            print ("  TEARDOWN otherarg %s" % param)
+        print ("  TEARDOWN otherarg %s" % param)
-        request.addfinalizer(fin)
-        return param
 
     def test_0(otherarg):
         print ("  RUN test0 with otherarg %s" % otherarg)
@@ -777,7 +801,8 @@ self-contained implementation of this idea::
         @pytest.fixture(autouse=True)
         def transact(self, request, db):
             db.begin(request.function.__name__)
-            request.addfinalizer(db.rollback)
+            yield
+            db.rollback()
 
         def test_method1(self, db):
             assert db.intransaction == ["test_method1"]
@@ -817,10 +842,11 @@ active.  The canonical way to do that is to put the transact definition
 into a conftest.py file **without** using ``autouse``::
 
     # content of conftest.py
-    @pytest.fixture()
+    @pytest.fixture
     def transact(self, request, db):
         db.begin()
-        request.addfinalizer(db.rollback)
+        yield
+        db.rollback()
 
 and then e.g. have a TestClass using it by declaring the need::
 

doc/en/yieldfixture.rst

@@ -1,100 +1,17 @@
 .. _yieldfixture:
 
-Fixture functions using "yield" / context manager integration
+"yield_fixture" functions
 ---------------------------------------------------------------
 
+.. deprecated:: 2.10
+
 .. versionadded:: 2.4
 
-.. regendoc:wipe
+.. important::
+    Since pytest-2.10, fixtures using the normal ``fixture`` decorator can use a ``yield``
+    statement to provide fixture values and execute teardown code, exactly like ``yield_fixture``
+    in previous versions.
 
-pytest-2.4 allows fixture functions to seamlessly use a ``yield`` instead 
+    Marking functions as ``yield_fixture`` is still supported, but deprecated and should not
-of a ``return`` statement to provide a fixture value while otherwise
+    be used in new code.
-fully supporting all other fixture features.
 
-Let's look at a simple standalone-example using the ``yield`` syntax::
-
-    # content of test_yield.py
-    
-    import pytest
-
-    @pytest.yield_fixture
-    def passwd():
-        print ("\nsetup before yield")
-        f = open("/etc/passwd")
-        yield f.readlines()
-        print ("teardown after yield")
-        f.close()
-
-    def test_has_lines(passwd):
-        print ("test called")
-        assert passwd
-
-In contrast to :ref:`finalization through registering callbacks
-<finalization>`, our fixture function used a ``yield``
-statement to provide the lines of the ``/etc/passwd`` file.  
-The code after the ``yield`` statement serves as the teardown code, 
-avoiding the indirection of registering a teardown callback function.   
-
-Let's run it with output capturing disabled::
-
-    $ py.test -q -s test_yield.py
-    
-    setup before yield
-    test called
-    .teardown after yield
-    
-    1 passed in 0.12 seconds
-
-We can also seamlessly use the new syntax with ``with`` statements.
-Let's simplify the above ``passwd`` fixture::
-
-    # content of test_yield2.py
-    
-    import pytest
-
-    @pytest.yield_fixture
-    def passwd():
-        with open("/etc/passwd") as f:
-            yield f.readlines()
-
-    def test_has_lines(passwd):
-        assert len(passwd) >= 1
-
-The file ``f`` will be closed after the test finished execution
-because the Python ``file`` object supports finalization when
-the ``with`` statement ends. 
-
-Note that the yield fixture form supports all other fixture
-features such as ``scope``, ``params``, etc., thus changing existing
-fixture functions to use ``yield`` is straightforward.
-
-.. note::
-
-    While the ``yield`` syntax is similar to what
-    :py:func:`contextlib.contextmanager` decorated functions
-    provide, with pytest fixture functions the part after the
-    "yield" will always be invoked, independently from the
-    exception status of the test function which uses the fixture.
-    This behaviour makes sense if you consider that many different
-    test functions might use a module or session scoped fixture.
-
-
-Discussion and future considerations / feedback
-++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-There are some topics that are worth mentioning:
-
-- usually ``yield`` is used for producing multiple values.
-  But fixture functions can only yield exactly one value.
-  Yielding a second fixture value will get you an error.
-  It's possible we can evolve pytest to allow for producing
-  multiple values as an alternative to current parametrization.
-  For now, you can just use the normal
-  :ref:`fixture parametrization <fixture-parametrize>`
-  mechanisms together with ``yield``-style fixtures.
-
-- lastly ``yield`` introduces more than one way to write
-  fixture functions, so what's the obvious way to a newcomer?
-
-If you want to feedback or participate in discussion of the above
-topics, please join our :ref:`contact channels`, you are most welcome.

testing/python/fixture.py

@@ -2597,11 +2597,13 @@ class TestShowFixtures:
         ''')
 
 
+@pytest.mark.parametrize('flavor', ['fixture', 'yield_fixture'])
 class TestContextManagerFixtureFuncs:
-    def test_simple(self, testdir):
+
+    def test_simple(self, testdir, flavor):
         testdir.makepyfile("""
             import pytest
-            @pytest.yield_fixture
+            @pytest.{flavor}
             def arg1():
                 print ("setup")
                 yield 1
@@ -2611,7 +2613,7 @@ class TestContextManagerFixtureFuncs:
             def test_2(arg1):
                 print ("test2 %s" % arg1)
                 assert 0
-        """)
+        """.format(flavor=flavor))
         result = testdir.runpytest("-s")
         result.stdout.fnmatch_lines("""
             *setup*
@@ -2622,10 +2624,10 @@ class TestContextManagerFixtureFuncs:
             *teardown*
         """)
 
-    def test_scoped(self, testdir):
+    def test_scoped(self, testdir, flavor):
         testdir.makepyfile("""
             import pytest
-            @pytest.yield_fixture(scope="module")
+            @pytest.{flavor}(scope="module")
             def arg1():
                 print ("setup")
                 yield 1
@@ -2634,7 +2636,7 @@ class TestContextManagerFixtureFuncs:
                 print ("test1 %s" % arg1)
             def test_2(arg1):
                 print ("test2 %s" % arg1)
-        """)
+        """.format(flavor=flavor))
         result = testdir.runpytest("-s")
         result.stdout.fnmatch_lines("""
             *setup*
@@ -2643,94 +2645,62 @@ class TestContextManagerFixtureFuncs:
             *teardown*
         """)
 
-    def test_setup_exception(self, testdir):
+    def test_setup_exception(self, testdir, flavor):
         testdir.makepyfile("""
             import pytest
-            @pytest.yield_fixture(scope="module")
+            @pytest.{flavor}(scope="module")
             def arg1():
                 pytest.fail("setup")
                 yield 1
             def test_1(arg1):
                 pass
-        """)
+        """.format(flavor=flavor))
         result = testdir.runpytest("-s")
         result.stdout.fnmatch_lines("""
             *pytest.fail*setup*
             *1 error*
         """)
 
-    def test_teardown_exception(self, testdir):
+    def test_teardown_exception(self, testdir, flavor):
         testdir.makepyfile("""
             import pytest
-            @pytest.yield_fixture(scope="module")
+            @pytest.{flavor}(scope="module")
             def arg1():
                 yield 1
                 pytest.fail("teardown")
             def test_1(arg1):
                 pass
-        """)
+        """.format(flavor=flavor))
         result = testdir.runpytest("-s")
         result.stdout.fnmatch_lines("""
             *pytest.fail*teardown*
             *1 passed*1 error*
         """)
 
-    def test_yields_more_than_one(self, testdir):
+    def test_yields_more_than_one(self, testdir, flavor):
         testdir.makepyfile("""
             import pytest
-            @pytest.yield_fixture(scope="module")
+            @pytest.{flavor}(scope="module")
             def arg1():
                 yield 1
                 yield 2
             def test_1(arg1):
                 pass
-        """)
+        """.format(flavor=flavor))
         result = testdir.runpytest("-s")
         result.stdout.fnmatch_lines("""
             *fixture function*
             *test_yields*:2*
         """)
 
-
+    def test_custom_name(self, testdir, flavor):
-    def test_no_yield(self, testdir):
         testdir.makepyfile("""
             import pytest
-            @pytest.yield_fixture(scope="module")
+            @pytest.{flavor}(name='meow')
-            def arg1():
-                return 1
-            def test_1(arg1):
-                pass
-        """)
-        result = testdir.runpytest("-s")
-        result.stdout.fnmatch_lines("""
-            *yield_fixture*requires*yield*
-            *yield_fixture*
-            *def arg1*
-        """)
-
-    def test_yield_not_allowed_in_non_yield(self, testdir):
-        testdir.makepyfile("""
-            import pytest
-            @pytest.fixture(scope="module")
-            def arg1():
-                yield 1
-            def test_1(arg1):
-                pass
-        """)
-        result = testdir.runpytest("-s")
-        result.stdout.fnmatch_lines("""
-            *fixture*cannot use*yield*
-            *def arg1*
-        """)
-
-    def test_custom_name(self, testdir):
-        testdir.makepyfile("""
-            import pytest
-            @pytest.fixture(name='meow')
             def arg1():
                 return 'mew'
             def test_1(meow):
                 print(meow)
-        """)
+        """.format(flavor=flavor))
         result = testdir.runpytest("-s")
         result.stdout.fnmatch_lines("*mew*")