[1.7.x] Prevented views.static.serve() from using large memory on large files.

This is a security fix. Disclosure following shortly.

django/views/static.py

@@ -17,6 +17,8 @@ from django.utils.http import http_date, parse_http_date
 from django.utils.six.moves.urllib.parse import unquote
 from django.utils.translation import ugettext as _, ugettext_lazy
 
+STREAM_CHUNK_SIZE = 4096
+
 
 def serve(request, path, document_root=None, show_indexes=False):
     """
@@ -61,7 +63,8 @@ def serve(request, path, document_root=None, show_indexes=False):
         return HttpResponseNotModified()
     content_type, encoding = mimetypes.guess_type(fullpath)
     content_type = content_type or 'application/octet-stream'
-    response = StreamingHttpResponse(open(fullpath, 'rb'),
+    f = open(fullpath, 'rb')
+    response = StreamingHttpResponse(iter(lambda: f.read(STREAM_CHUNK_SIZE), b''),
                                      content_type=content_type)
     response["Last-Modified"] = http_date(statobj.st_mtime)
     if stat.S_ISREG(statobj.st_mode):

docs/releases/1.4.18.txt

@@ -45,6 +45,21 @@ from a XSS attack. This bug doesn't affect Django currently, since we only put
 this URL into the ``Location`` response header and browsers seem to ignore
 JavaScript there.
 
+Denial-of-service attack against ``django.views.static.serve``
+==============================================================
+
+In older versions of Django, the :func:`django.views.static.serve` view read
+the files it served one line at a time. Therefore, a big file with no newlines
+would result in memory usage equal to the size of that file. An attacker could
+exploit this and launch a denial-of-service attack by simultaneously requesting
+many large files. This view now reads the file in chunks to prevent large
+memory usage.
+
+Note, however, that this view has always carried a warning that it is not
+hardened for production use and should be used only as a development aid. Now
+may be a good time to audit your project and serve your files in production
+using a real front-end web server if you are not doing so.
+
 Bugfixes
 ========
 

docs/releases/1.6.10.txt

@@ -43,3 +43,18 @@ provide safe redirect targets and put such a URL into a link, they could suffer
 from a XSS attack. This bug doesn't affect Django currently, since we only put
 this URL into the ``Location`` response header and browsers seem to ignore
 JavaScript there.
+
+Denial-of-service attack against ``django.views.static.serve``
+==============================================================
+
+In older versions of Django, the :func:`django.views.static.serve` view read
+the files it served one line at a time. Therefore, a big file with no newlines
+would result in memory usage equal to the size of that file. An attacker could
+exploit this and launch a denial-of-service attack by simultaneously requesting
+many large files. This view now reads the file in chunks to prevent large
+memory usage.
+
+Note, however, that this view has always carried a warning that it is not
+hardened for production use and should be used only as a development aid. Now
+may be a good time to audit your project and serve your files in production
+using a real front-end web server if you are not doing so.

docs/releases/1.7.3.txt

@@ -44,6 +44,21 @@ from a XSS attack. This bug doesn't affect Django currently, since we only put
 this URL into the ``Location`` response header and browsers seem to ignore
 JavaScript there.
 
+Denial-of-service attack against ``django.views.static.serve``
+==============================================================
+
+In older versions of Django, the :func:`django.views.static.serve` view read
+the files it served one line at a time. Therefore, a big file with no newlines
+would result in memory usage equal to the size of that file. An attacker could
+exploit this and launch a denial-of-service attack by simultaneously requesting
+many large files. This view now reads the file in chunks to prevent large
+memory usage.
+
+Note, however, that this view has always carried a warning that it is not
+hardened for production use and should be used only as a development aid. Now
+may be a good time to audit your project and serve your files in production
+using a real front-end web server if you are not doing so.
+
 Bugfixes
 ========
 

tests/view_tests/tests/test_static.py

@@ -8,7 +8,7 @@ from django.conf.urls.static import static
 from django.http import HttpResponseNotModified
 from django.test import SimpleTestCase, override_settings
 from django.utils.http import http_date
-from django.views.static import was_modified_since
+from django.views.static import was_modified_since, STREAM_CHUNK_SIZE
 
 from .. import urls
 from ..urls import media_dir
@@ -33,6 +33,14 @@ class StaticTests(SimpleTestCase):
             self.assertEqual(len(response_content), int(response['Content-Length']))
             self.assertEqual(mimetypes.guess_type(file_path)[1], response.get('Content-Encoding', None))
 
+    def test_chunked(self):
+        "The static view should stream files in chunks to avoid large memory usage"
+        response = self.client.get('/%s/%s' % (self.prefix, 'long-line.txt'))
+        first_chunk = next(response.streaming_content)
+        self.assertEqual(len(first_chunk), STREAM_CHUNK_SIZE)
+        second_chunk = next(response.streaming_content)
+        self.assertEqual(len(second_chunk), 1451)
+
     def test_unknown_mime_type(self):
         response = self.client.get('/%s/file.unknown' % self.prefix)
         self.assertEqual('application/octet-stream', response['Content-Type'])