Fixed #6527 -- Provided repeatable content access

in HttpResponses instantiated with iterators.

django/http/response.py

@@ -246,8 +246,18 @@ class HttpResponse(HttpResponseBase):
     else:
         __str__ = serialize
 
+    def _consume_content(self):
+        # If the response was instantiated with an iterator, when its content
+        # is accessed, the iterator is going be exhausted and the content
+        # loaded in memory. At this point, it's better to abandon the original
+        # iterator and save the content for later reuse. This is a temporary
+        # solution. See the comment in __iter__ below for the long term plan.
+        if self._base_content_is_iter:
+            self.content = b''.join(self.make_bytes(e) for e in self._container)
+
     @property
     def content(self):
+        self._consume_content()
         return b''.join(self.make_bytes(e) for e in self._container)
 
     @content.setter
@@ -262,6 +272,17 @@ class HttpResponse(HttpResponseBase):
             self._base_content_is_iter = False
 
     def __iter__(self):
+        # Raise a deprecation warning only if the content wasn't consumed yet,
+        # because the response may be intended to be streamed.
+        # Once the deprecation completes, iterators should be consumed upon
+        # assignment rather than upon access. The _consume_content method
+        # should be removed. See #6527.
+        if self._base_content_is_iter:
+            warnings.warn(
+                'Creating streaming responses with `HttpResponse` is '
+                'deprecated. Use `StreamingHttpResponse` instead '
+                'if you need the streaming behavior.',
+                PendingDeprecationWarning, stacklevel=2)
         self._iterator = iter(self._container)
         return self
 
@@ -277,14 +298,12 @@ class HttpResponse(HttpResponseBase):
     next = __next__             # Python 2 compatibility
 
     def write(self, content):
-        if self._base_content_is_iter:
+        self._consume_content()
-            raise Exception("This %s instance is not writable" % self.__class__.__name__)
         self._container.append(content)
 
     def tell(self):
-        if self._base_content_is_iter:
+        self._consume_content()
-            raise Exception("This %s instance cannot tell its position" % self.__class__.__name__)
+        return sum(len(chunk) for chunk in self)
-        return sum([len(chunk) for chunk in self])
 
 
 class StreamingHttpResponse(HttpResponseBase):
@@ -389,6 +408,7 @@ class HttpResponseNotModified(HttpResponse):
         if value:
             raise AttributeError("You cannot set content to a 304 (Not Modified) response")
         self._container = []
+        self._base_content_is_iter = False
 
 
 class HttpResponseBadRequest(HttpResponse):

docs/internals/deprecation.txt

@@ -286,6 +286,10 @@ these changes.
 * The ``mimetype`` argument to :class:`~django.http.HttpResponse` ``__init__``
   will be removed (``content_type`` should be used instead).
 
+* When :class:`~django.http.HttpResponse` is instantiated with an iterator,
+  or when :attr:`~django.http.HttpResponse.content` is set to an iterator,
+  that iterator will be immediately consumed.
+
 * The ``AUTH_PROFILE_MODULE`` setting, and the ``get_profile()`` method on
   the User model, will be removed.
 

docs/ref/request-response.txt

@@ -569,18 +569,25 @@ Passing iterators
 Finally, you can pass ``HttpResponse`` an iterator rather than strings. If you
 use this technique, the iterator should return strings.
 
+Passing an iterator as content to :class:`HttpResponse` creates a
+streaming response if (and only if) no middleware accesses the
+:attr:`HttpResponse.content` attribute before the response is returned.
+
 .. versionchanged:: 1.5
 
-    Passing an iterator as content to :class:`HttpResponse` creates a
+This technique is fragile and was deprecated in Django 1.5. If you need the
-    streaming response if (and only if) no middleware accesses the
+response to be streamed from the iterator to the client, you should use the
-    :attr:`HttpResponse.content` attribute before the response is returned.
+:class:`StreamingHttpResponse` class instead.
 
-    If you want to guarantee that your response will stream to the client, you
+As of Django 1.7, when :class:`HttpResponse` is instantiated with an
-    should use the new :class:`StreamingHttpResponse` class instead.
+iterator, it will consume it immediately, store the response content as a
+string, and discard the iterator.
 
-If an :class:`HttpResponse` instance has been initialized with an iterator as
+.. versionchanged:: 1.5
-its content, you can't use it as a file-like object. Doing so will raise an
+
-exception.
+You can now use :class:`HttpResponse` as a file-like object even if it was
+instantiated with an iterator. Django will consume and save the content of
+the iterator on first access.
 
 Setting headers
 ~~~~~~~~~~~~~~~

docs/releases/1.5.txt

@@ -84,6 +84,8 @@ For one-to-one relationships, both sides can be cached. For many-to-one
 relationships, only the single side of the relationship can be cached. This
 is particularly helpful in combination with ``prefetch_related``.
 
+.. _explicit-streaming-responses:
+
 Explicit support for streaming responses
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -98,7 +100,7 @@ You can now explicitly generate a streaming response with the new
 is an iterator.
 
 Since :class:`~django.http.StreamingHttpResponse` does not have a ``content``
-attribute, middleware that need access to the response content must test for
+attribute, middleware that needs access to the response content must test for
 streaming responses and behave accordingly. See :ref:`response-middleware` for
 more information.
 
@@ -483,6 +485,30 @@ Features deprecated in 1.5
 
 .. _simplejson-deprecation:
 
+:setting:`AUTH_PROFILE_MODULE`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+With the introduction of :ref:`custom User models <auth-custom-user>`, there is
+no longer any need for a built-in mechanism to store user profile data.
+
+You can still define user profiles models that have a one-to-one relation with
+the User model - in fact, for many applications needing to associate data with
+a User account, this will be an appropriate design pattern to follow. However,
+the :setting:`AUTH_PROFILE_MODULE` setting, and the
+:meth:`~django.contrib.auth.models.User.get_profile()` method for accessing
+the user profile model, should not be used any longer.
+
+Streaming behavior of :class:`HttpResponse`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Django 1.5 deprecates the ability to stream a response by passing an iterator
+to :class:`~django.http.HttpResponse`. If you rely on this behavior, switch to
+:class:`~django.http.StreamingHttpResponse`. See :ref:`explicit-streaming-
+responses` above.
+
+In Django 1.7 and above, the iterator will be consumed immediately by
+:class:`~django.http.HttpResponse`.
+
 ``django.utils.simplejson``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -497,12 +523,6 @@ incompatibilities between versions of :mod:`simplejson` -- see the
 If you rely on features added to :mod:`simplejson` after it became Python's
 :mod:`json`, you should import :mod:`simplejson` explicitly.
 
-``itercompat.product``
-~~~~~~~~~~~~~~~~~~~~~~
-
-The :func:`~django.utils.itercompat.product` function has been deprecated. Use
-the built-in :func:`itertools.product` instead.
-
 ``django.utils.encoding.StrAndUnicode``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -510,6 +530,13 @@ The :class:`~django.utils.encoding.StrAndUnicode` mix-in has been deprecated.
 Define a ``__str__`` method and apply the
 :func:`~django.utils.encoding.python_2_unicode_compatible` decorator instead.
 
+``django.utils.itercompat.product``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :func:`~django.utils.itercompat.product` function has been deprecated. Use
+the built-in :func:`itertools.product` instead.
+
+
 ``django.utils.markup``
 ~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -517,16 +544,3 @@ The markup contrib module has been deprecated and will follow an accelerated
 deprecation schedule. Direct use of python markup libraries or 3rd party tag
 libraries is preferred to Django maintaining this functionality in the
 framework.
-
-:setting:`AUTH_PROFILE_MODULE`
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-With the introduction of :ref:`custom User models <auth-custom-user>`, there is
-no longer any need for a built-in mechanism to store user profile data.
-
-You can still define user profiles models that have a one-to-one relation with
-the User model - in fact, for many applications needing to associate data with
-a User account, this will be an appropriate design pattern to follow. However,
-the :setting:`AUTH_PROFILE_MODULE` setting, and the
-:meth:`~django.contrib.auth.models.User.get_profile()` method for accessing
-the user profile model, should not be used any longer.

tests/regressiontests/httpwrappers/tests.py

@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import copy
 import os
 import pickle
+import warnings
 
 from django.core.exceptions import SuspiciousOperation
 from django.http import (QueryDict, HttpResponse, HttpResponseRedirect,
@@ -313,11 +314,17 @@ class HttpResponseTests(unittest.TestCase):
         r.content = [1, 2, 3]
         self.assertEqual(r.content, b'123')
 
-        #test retrieval explicitly using iter and odd inputs
+        #test retrieval explicitly using iter (deprecated) and odd inputs
         r = HttpResponse()
         r.content = ['1', '2', 3, '\u079e']
-        my_iter = r.__iter__()
+        with warnings.catch_warnings(record=True) as w:
-        result = list(my_iter)
+            warnings.simplefilter("always", PendingDeprecationWarning)
+            my_iter = iter(r)
+            self.assertEqual(w[0].category, PendingDeprecationWarning)
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always", PendingDeprecationWarning)
+            result = list(my_iter)
+            self.assertEqual(w[0].category, PendingDeprecationWarning)
         #'\xde\x9e' == unichr(1950).encode('utf-8')
         self.assertEqual(result, [b'1', b'2', b'3', b'\xde\x9e'])
         self.assertEqual(r.content, b'123\xde\x9e')
@@ -330,6 +337,16 @@ class HttpResponseTests(unittest.TestCase):
         self.assertRaises(UnicodeEncodeError,
                           getattr, r, 'content')
 
+        # content can safely be accessed multiple times.
+        r = HttpResponse(iter(['hello', 'world']))
+        self.assertEqual(r.content, r.content)
+        self.assertEqual(r.content, b'helloworld')
+
+        # additional content can be written to the response.
+        r.write('!')
+        self.assertEqual(r.content, b'helloworld!')
+
+
     def test_file_interface(self):
         r = HttpResponse()
         r.write(b"hello")
@@ -338,7 +355,9 @@ class HttpResponseTests(unittest.TestCase):
         self.assertEqual(r.tell(), 17)
 
         r = HttpResponse(['abc'])
-        self.assertRaises(Exception, r.write, 'def')
+        r.write('def')
+        self.assertEqual(r.tell(), 6)
+        self.assertEqual(r.content, b'abcdef')
 
     def test_unsafe_redirect(self):
         bad_urls = [
@@ -447,7 +466,9 @@ class FileCloseTests(TestCase):
         file1 = open(filename)
         r = HttpResponse(file1)
         self.assertFalse(file1.closed)
-        list(r)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", PendingDeprecationWarning)
+            list(r)
         self.assertFalse(file1.closed)
         r.close()
         self.assertTrue(file1.closed)