p15693087
/
django
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed #21231 -- Enforced a max size for GET/POST values read into memory. 

Thanks Tom Christie for review.
This commit is contained in:
Andre Cruz 2015-01-07 18:41:29 +00:00 committed by Tim Graham
parent 4065f429f5
commit 929684d6ee
9 changed files with 387 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
django/conf/global_settings.py
View File

@ -285,6 +285,14 @@ FILE_UPLOAD_HANDLERS = [
# file system instead of into memory.
FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440 # i.e. 2.5 MB
# Maximum size in bytes of request data (excluding file uploads) that will be
# read before a SuspiciousOperation (RequestDataTooBig) is raised.
DATA_UPLOAD_MAX_MEMORY_SIZE = 2621440 # i.e. 2.5 MB
# Maximum number of GET/POST parameters that will be read before a
# SuspiciousOperation (TooManyFieldsSent) is raised.
DATA_UPLOAD_MAX_NUMBER_FIELDS = 1000
# Directory in which upload streamed files will be temporarily saved. A value of
# `None` will make Django use the operating system's default temporary directory
# (i.e. "/tmp" on *nix systems).

16
django/core/exceptions.py
View File

@ -53,6 +53,22 @@ class DisallowedRedirect(SuspiciousOperation):
pass
class TooManyFieldsSent(SuspiciousOperation):
"""
The number of fields in a POST request exceeded
settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.
"""
pass
class RequestDataTooBig(SuspiciousOperation):
"""
The size of the request (excluding any file uploads) exceeded
settings.DATA_UPLOAD_MAX_MEMORY_SIZE.
"""
pass
class PermissionDenied(Exception):
"""The user did not have permission to do that"""
pass

37
django/http/multipartparser.py
View File

@ -12,7 +12,9 @@ import cgi
import sys
from django.conf import settings
from django.core.exceptions import SuspiciousMultipartForm
from django.core.exceptions import (
RequestDataTooBig, SuspiciousMultipartForm, TooManyFieldsSent,
)
from django.core.files.uploadhandler import (
SkipFile, StopFutureHandlers, StopUpload,
)
@ -145,6 +147,13 @@ class MultiPartParser(object):
old_field_name = None
counters = [0] * len(handlers)
# Number of bytes that have been read.
num_bytes_read = 0
# To count the number of keys in the request.
num_post_keys = 0
# To limit the amount of data read from the request.
read_size = None
try:
for item_type, meta_data, field_stream in Parser(stream, self._boundary):
if old_field_name:
@ -166,15 +175,37 @@ class MultiPartParser(object):
field_name = force_text(field_name, encoding, errors='replace')
if item_type == FIELD:
# Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS.
num_post_keys += 1
if (settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None and
settings.DATA_UPLOAD_MAX_NUMBER_FIELDS < num_post_keys):
raise TooManyFieldsSent(
'The number of GET/POST parameters exceeded '
'settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.'
)
# Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE.
if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None:
read_size = settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read
# This is a post field, we can just set it in the post
if transfer_encoding == 'base64':
raw_data = field_stream.read()
raw_data = field_stream.read(size=read_size)
num_bytes_read += len(raw_data)
try:
data = base64.b64decode(raw_data)
except _BASE64_DECODE_ERROR:
data = raw_data
else:
data = field_stream.read()
data = field_stream.read(size=read_size)
num_bytes_read += len(data)
# Add two here to make the check consistent with the
# x-www-form-urlencoded check that includes '&='.
num_bytes_read += len(field_name) + 2
if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and
num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE):
raise RequestDataTooBig('Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.')
self._post.appendlist(field_name,
force_text(data, encoding, errors='replace'))

27
django/http/request.py
View File

@ -8,7 +8,9 @@ from itertools import chain
from django.conf import settings
from django.core import signing
from django.core.exceptions import DisallowedHost, ImproperlyConfigured
from django.core.exceptions import (
DisallowedHost, ImproperlyConfigured, RequestDataTooBig,
)
from django.core.files import uploadhandler
from django.http.multipartparser import MultiPartParser, MultiPartParserError
from django.utils import six
@ -16,9 +18,9 @@ from django.utils.datastructures import ImmutableList, MultiValueDict
from django.utils.encoding import (
escape_uri_path, force_bytes, force_str, force_text, iri_to_uri,
)
from django.utils.http import is_same_domain
from django.utils.http import is_same_domain, limited_parse_qsl
from django.utils.six.moves.urllib.parse import (
parse_qsl, quote, urlencode, urljoin, urlsplit,
quote, urlencode, urljoin, urlsplit,
)
RAISE_ERROR = object()
@ -259,6 +261,12 @@ class HttpRequest(object):
if not hasattr(self, '_body'):
if self._read_started:
raise RawPostDataException("You cannot access body after reading from request's data stream")
# Limit the maximum request data size that will be handled in-memory.
if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and
int(self.META.get('CONTENT_LENGTH', 0)) > settings.DATA_UPLOAD_MAX_MEMORY_SIZE):
raise RequestDataTooBig('Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.')
try:
self._body = self.read()
except IOError as e:
@ -368,6 +376,12 @@ class QueryDict(MultiValueDict):
if not encoding:
encoding = settings.DEFAULT_CHARSET
self.encoding = encoding
query_string = query_string or ''
parse_qsl_kwargs = {
'keep_blank_values': True,
'fields_limit': settings.DATA_UPLOAD_MAX_NUMBER_FIELDS,
'encoding': encoding,
}
if six.PY3:
if isinstance(query_string, bytes):
# query_string normally contains URL-encoded data, a subset of ASCII.
@ -376,13 +390,10 @@ class QueryDict(MultiValueDict):
except UnicodeDecodeError:
# ... but some user agents are misbehaving :-(
query_string = query_string.decode('iso-8859-1')
for key, value in parse_qsl(query_string or '',
keep_blank_values=True,
encoding=encoding):
for key, value in limited_parse_qsl(query_string, **parse_qsl_kwargs):
self.appendlist(key, value)
else:
for key, value in parse_qsl(query_string or '',
keep_blank_values=True):
for key, value in limited_parse_qsl(query_string, **parse_qsl_kwargs):
try:
value = value.decode(encoding)
except UnicodeDecodeError:

60
django/utils/http.py
View File

@ -9,6 +9,7 @@ import unicodedata
from binascii import Error as BinasciiError
from email.utils import formatdate
from django.core.exceptions import TooManyFieldsSent
from django.utils import six
from django.utils.datastructures import MultiValueDict
from django.utils.encoding import force_bytes, force_str, force_text
@ -34,6 +35,8 @@ ASCTIME_DATE = re.compile(r'^\w{3} %s %s %s %s$' % (__M, __D2, __T, __Y))
RFC3986_GENDELIMS = str(":/?#[]@")
RFC3986_SUBDELIMS = str("!$&'()*+,;=")
FIELDS_MATCH = re.compile('[&;]')
@keep_lazy_text
def urlquote(url, safe='/'):
@ -314,3 +317,60 @@ def _is_safe_url(url, host):
return False
return ((not url_info.netloc or url_info.netloc == host) and
(not url_info.scheme or url_info.scheme in ['http', 'https']))
def limited_parse_qsl(qs, keep_blank_values=False, encoding='utf-8',
errors='replace', fields_limit=None):
"""
Return a list of key/value tuples parsed from query string.
Copied from urlparse with an additional "fields_limit" argument.
Copyright (C) 2013 Python Software Foundation (see LICENSE.python).
Arguments:
qs: percent-encoded query string to be parsed
keep_blank_values: flag indicating whether blank values in
percent-encoded queries should be treated as blank strings. A
true value indicates that blanks should be retained as blank
strings. The default false value indicates that blank values
are to be ignored and treated as if they were not included.
encoding and errors: specify how to decode percent-encoded sequences
into Unicode characters, as accepted by the bytes.decode() method.
fields_limit: maximum number of fields parsed or an exception
is raised. None means no limit and is the default.
"""
if fields_limit:
pairs = FIELDS_MATCH.split(qs, fields_limit)
if len(pairs) > fields_limit:
raise TooManyFieldsSent(
'The number of GET/POST parameters exceeded '
'settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.'
)
else:
pairs = FIELDS_MATCH.split(qs)
r = []
for name_value in pairs:
if not name_value:
continue
nv = name_value.split(str('='), 1)
if len(nv) != 2:
# Handle case of a control-name with no equal sign
if keep_blank_values:
nv.append('')
else:
continue
if len(nv[1]) or keep_blank_values:
if six.PY3:
name = nv[0].replace('+', ' ')
name = unquote(name, encoding=encoding, errors=errors)
value = nv[1].replace('+', ' ')
value = unquote(value, encoding=encoding, errors=errors)
else:
name = unquote(nv[0].replace(b'+', b' '))
value = unquote(nv[1].replace(b'+', b' '))
r.append((name, value))
return r

2
docs/ref/exceptions.txt
View File

@ -61,9 +61,11 @@ Django core exception classes are defined in ``django.core.exceptions``.
* ``DisallowedModelAdminToField``
* ``DisallowedRedirect``
* ``InvalidSessionKey``
* ``RequestDataTooBig``
* ``SuspiciousFileOperation``
* ``SuspiciousMultipartForm``
* ``SuspiciousSession``
* ``TooManyFieldsSent``
If a ``SuspiciousOperation`` exception reaches the WSGI handler level it is
logged at the ``Error`` level and results in

49
docs/ref/settings.txt
View File

@ -880,6 +880,51 @@ This is an Oracle-specific setting.
The maximum size that the DATAFILE_TMP is allowed to grow to.
.. setting:: DATA_UPLOAD_MAX_MEMORY_SIZE
DATA_UPLOAD_MAX_MEMORY_SIZE
---------------------------
.. versionadded:: 1.10
Default: ``2621440`` (i.e. 2.5 MB).
The maximum size in bytes that a request body may be before a
:exc:`~django.core.exceptions.SuspiciousOperation` (``RequestDataTooBig``) is
raised. The check is done when accessing ``request.body`` or ``request.POST``
and is calculated against the total request size excluding any file upload
data. You can set this to ``None`` to disable the check. Applications that are
expected to receive unusually large form posts should tune this setting.
The amount of request data is correlated to the amount of memory needed to
process the request and populate the GET and POST dictionaries. Large requests
could be used as a denial-of-service attack vector if left unchecked. Since web
servers don't typically perform deep request inspection, it's not possible to
perform a similar check at that level.
See also :setting:`FILE_UPLOAD_MAX_MEMORY_SIZE`.
.. setting:: DATA_UPLOAD_MAX_NUMBER_FIELDS
DATA_UPLOAD_MAX_NUMBER_FIELDS
-----------------------------
.. versionadded:: 1.10
Default: ``1000``
The maximum number of parameters that may be received via GET or POST before a
:exc:`~django.core.exceptions.SuspiciousOperation` (``TooManyFields``) is
raised. You can set this to ``None`` to disable the check. Applications that
are expected to receive an unusually large number of form fields should tune
this setting.
The number of request parameters is correlated to the amount of time needed to
process the request and populate the GET and POST dictionaries. Large requests
could be used as a denial-of-service attack vector if left unchecked. Since web
servers don't typically perform deep request inspection, it's not possible to
perform a similar check at that level.
.. setting:: DATABASE_ROUTERS
``DATABASE_ROUTERS``
@ -1325,6 +1370,8 @@ Default: ``2621440`` (i.e. 2.5 MB).
The maximum size (in bytes) that an upload will be before it gets streamed to
the file system. See :doc:`/topics/files` for details.
See also :setting:`DATA_UPLOAD_MAX_MEMORY_SIZE`.
.. setting:: FILE_UPLOAD_DIRECTORY_PERMISSIONS
``FILE_UPLOAD_DIRECTORY_PERMISSIONS``
@ -3258,6 +3305,8 @@ Globalization (``i18n``/``l10n``)
HTTP
----
* :setting:`DATA_UPLOAD_MAX_MEMORY_SIZE`
* :setting:`DATA_UPLOAD_MAX_NUMBER_FIELDS`
* :setting:`DEFAULT_CHARSET`
* :setting:`DEFAULT_CONTENT_TYPE`
* :setting:`DISALLOWED_USER_AGENTS`

13
docs/releases/1.10.txt
View File

@ -719,6 +719,19 @@ custom lookup for it. For example::
``POINT (23.0000000000000000 5.5000000000000000)``, you'll get
``POINT (23 5.5)``.
Maximum size of a request body and the number of GET/POST parameters is limited
-------------------------------------------------------------------------------
Two new settings help mitigate denial-of-service attacks via large requests:
* :setting:`DATA_UPLOAD_MAX_MEMORY_SIZE` limits the size that a request body
may be. File uploads don't count towards this limit.
* :setting:`DATA_UPLOAD_MAX_NUMBER_FIELDS` limits the number of GET/POST
parameters that are parsed.
Applications that receive unusually large form posts may need to tune these
settings.
Miscellaneous
-------------

186
tests/requests/test_data_upload_settings.py Normal file
View File

@ -0,0 +1,186 @@
from io import BytesIO
from django.core.exceptions import RequestDataTooBig, TooManyFieldsSent
from django.core.handlers.wsgi import WSGIRequest
from django.test import SimpleTestCase
from django.test.client import FakePayload
TOO_MANY_FIELDS_MSG = 'The number of GET/POST parameters exceeded settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.'
TOO_MUCH_DATA_MSG = 'Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.'
class DataUploadMaxMemorySizeFormPostTests(SimpleTestCase):
def setUp(self):
payload = FakePayload('a=1&a=2;a=3\r\n')
self.request = WSGIRequest({
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': 'application/x-www-form-urlencoded',
'CONTENT_LENGTH': len(payload),
'wsgi.input': payload,
})
def test_size_exceeded(self):
with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=12):
with self.assertRaisesMessage(RequestDataTooBig, TOO_MUCH_DATA_MSG):
self.request._load_post_and_files()
def test_size_not_exceeded(self):
with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=13):
self.request._load_post_and_files()
def test_no_limit(self):
with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=None):
self.request._load_post_and_files()
class DataUploadMaxMemorySizeMultipartPostTests(SimpleTestCase):
def setUp(self):
payload = FakePayload("\r\n".join([
'--boundary',
'Content-Disposition: form-data; name="name"',
'',
'value',
'--boundary--'
''
]))
self.request = WSGIRequest({
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': 'multipart/form-data; boundary=boundary',
'CONTENT_LENGTH': len(payload),
'wsgi.input': payload,
})
def test_size_exceeded(self):
with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=10):
with self.assertRaisesMessage(RequestDataTooBig, TOO_MUCH_DATA_MSG):
self.request._load_post_and_files()
def test_size_not_exceeded(self):
with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=11):
self.request._load_post_and_files()
def test_no_limit(self):
with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=None):
self.request._load_post_and_files()
def test_file_passes(self):
payload = FakePayload("\r\n".join([
'--boundary',
'Content-Disposition: form-data; name="file1"; filename="test.file"',
'',
'value',
'--boundary--'
''
]))
request = WSGIRequest({
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': 'multipart/form-data; boundary=boundary',
'CONTENT_LENGTH': len(payload),
'wsgi.input': payload,
})
with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=1):
request._load_post_and_files()
self.assertIn('file1', request.FILES, "Upload file not present")
class DataUploadMaxMemorySizeGetTests(SimpleTestCase):
def setUp(self):
self.request = WSGIRequest({
'REQUEST_METHOD': 'GET',
'wsgi.input': BytesIO(b''),
'CONTENT_LENGTH': 3,
})
def test_data_upload_max_memory_size_exceeded(self):
with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=2):
with self.assertRaisesMessage(RequestDataTooBig, TOO_MUCH_DATA_MSG):
self.request.body
def test_size_not_exceeded(self):
with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=3):
self.request.body
def test_no_limit(self):
with self.settings(DATA_UPLOAD_MAX_MEMORY_SIZE=None):
self.request.body
class DataUploadMaxNumberOfFieldsGet(SimpleTestCase):
def test_get_max_fields_exceeded(self):
with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=1):
with self.assertRaisesMessage(TooManyFieldsSent, TOO_MANY_FIELDS_MSG):
request = WSGIRequest({
'REQUEST_METHOD': 'GET',
'wsgi.input': BytesIO(b''),
'QUERY_STRING': 'a=1&a=2;a=3',
})
request.GET['a']
def test_get_max_fields_not_exceeded(self):
with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=3):
request = WSGIRequest({
'REQUEST_METHOD': 'GET',
'wsgi.input': BytesIO(b''),
'QUERY_STRING': 'a=1&a=2;a=3',
})
request.GET['a']
class DataUploadMaxNumberOfFieldsMultipartPost(SimpleTestCase):
def setUp(self):
payload = FakePayload("\r\n".join([
'--boundary',
'Content-Disposition: form-data; name="name1"',
'',
'value1',
'--boundary',
'Content-Disposition: form-data; name="name2"',
'',
'value2',
'--boundary--'
''
]))
self.request = WSGIRequest({
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': 'multipart/form-data; boundary=boundary',
'CONTENT_LENGTH': len(payload),
'wsgi.input': payload,
})
def test_number_exceeded(self):
with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=1):
with self.assertRaisesMessage(TooManyFieldsSent, TOO_MANY_FIELDS_MSG):
self.request._load_post_and_files()
def test_number_not_exceeded(self):
with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=2):
self.request._load_post_and_files()
def test_no_limit(self):
with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=None):
self.request._load_post_and_files()
class DataUploadMaxNumberOfFieldsFormPost(SimpleTestCase):
def setUp(self):
payload = FakePayload("\r\n".join(['a=1&a=2;a=3', '']))
self.request = WSGIRequest({
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': 'application/x-www-form-urlencoded',
'CONTENT_LENGTH': len(payload),
'wsgi.input': payload,
})
def test_number_exceeded(self):
with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=2):
with self.assertRaisesMessage(TooManyFieldsSent, TOO_MANY_FIELDS_MSG):
self.request._load_post_and_files()
def test_number_not_exceeded(self):
with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=3):
self.request._load_post_and_files()
def test_no_limit(self):
with self.settings(DATA_UPLOAD_MAX_NUMBER_FIELDS=None):
self.request._load_post_and_files()