From b0953dc91385fb2823294a76d3c99e01c7b7e4ee Mon Sep 17 00:00:00 2001 From: Benjamin Kagia Date: Fri, 19 Apr 2013 20:20:23 +0300 Subject: [PATCH] Fixed #13721 -- Added UploadedFile.content_type_extra. Thanks Waldemar Kornewald and mvschaik for work on the patch. --- django/core/files/uploadedfile.py | 13 ++++----- django/core/files/uploadhandler.py | 9 ++++--- django/http/multipartparser.py | 12 ++++----- django/test/client.py | 6 ++++- docs/releases/1.7.txt | 16 ++++++++++++ docs/topics/http/file-uploads.txt | 42 +++++++++++++++++++++++++----- tests/file_uploads/tests.py | 21 +++++++++++++++ tests/file_uploads/urls.py | 1 + tests/file_uploads/views.py | 13 ++++++++- 9 files changed, 109 insertions(+), 24 deletions(-) diff --git a/django/core/files/uploadedfile.py b/django/core/files/uploadedfile.py index 39b99ff78f..9f948ca03b 100644 --- a/django/core/files/uploadedfile.py +++ b/django/core/files/uploadedfile.py @@ -23,11 +23,12 @@ class UploadedFile(File): """ DEFAULT_CHUNK_SIZE = 64 * 2**10 - def __init__(self, file=None, name=None, content_type=None, size=None, charset=None): + def __init__(self, file=None, name=None, content_type=None, size=None, charset=None, content_type_extra=None): super(UploadedFile, self).__init__(file, name) self.size = size self.content_type = content_type self.charset = charset + self.content_type_extra = content_type_extra def __repr__(self): return force_str("<%s: %s (%s)>" % ( @@ -55,13 +56,13 @@ class TemporaryUploadedFile(UploadedFile): """ A file uploaded to a temporary location (i.e. stream-to-disk). """ - def __init__(self, name, content_type, size, charset): + def __init__(self, name, content_type, size, charset, content_type_extra): if settings.FILE_UPLOAD_TEMP_DIR: file = tempfile.NamedTemporaryFile(suffix='.upload', dir=settings.FILE_UPLOAD_TEMP_DIR) else: file = tempfile.NamedTemporaryFile(suffix='.upload') - super(TemporaryUploadedFile, self).__init__(file, name, content_type, size, charset) + super(TemporaryUploadedFile, self).__init__(file, name, content_type, size, charset, content_type_extra) def temporary_file_path(self): """ @@ -83,8 +84,8 @@ class InMemoryUploadedFile(UploadedFile): """ A file uploaded into memory (i.e. stream-to-memory). """ - def __init__(self, file, field_name, name, content_type, size, charset): - super(InMemoryUploadedFile, self).__init__(file, name, content_type, size, charset) + def __init__(self, file, field_name, name, content_type, size, charset, content_type_extra): + super(InMemoryUploadedFile, self).__init__(file, name, content_type, size, charset, content_type_extra) self.field_name = field_name def open(self, mode=None): @@ -109,7 +110,7 @@ class SimpleUploadedFile(InMemoryUploadedFile): def __init__(self, name, content, content_type='text/plain'): content = content or b'' super(SimpleUploadedFile, self).__init__(BytesIO(content), None, name, - content_type, len(content), None) + content_type, len(content), None, None) def from_dict(cls, file_dict): """ diff --git a/django/core/files/uploadhandler.py b/django/core/files/uploadhandler.py index f5e95cf2fd..6739b26e0c 100644 --- a/django/core/files/uploadhandler.py +++ b/django/core/files/uploadhandler.py @@ -64,6 +64,7 @@ class FileUploadHandler(object): self.content_type = None self.content_length = None self.charset = None + self.content_type_extra = None self.request = request def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None): @@ -84,7 +85,7 @@ class FileUploadHandler(object): """ pass - def new_file(self, field_name, file_name, content_type, content_length, charset=None): + def new_file(self, field_name, file_name, content_type, content_length, charset=None, content_type_extra=None): """ Signal that a new file has been started. @@ -96,6 +97,7 @@ class FileUploadHandler(object): self.content_type = content_type self.content_length = content_length self.charset = charset + self.content_type_extra = content_type_extra def receive_data_chunk(self, raw_data, start): """ @@ -132,7 +134,7 @@ class TemporaryFileUploadHandler(FileUploadHandler): Create the file object to append to as data is coming in. """ super(TemporaryFileUploadHandler, self).new_file(file_name, *args, **kwargs) - self.file = TemporaryUploadedFile(self.file_name, self.content_type, 0, self.charset) + self.file = TemporaryUploadedFile(self.file_name, self.content_type, 0, self.charset, self.content_type_extra) def receive_data_chunk(self, raw_data, start): self.file.write(raw_data) @@ -187,7 +189,8 @@ class MemoryFileUploadHandler(FileUploadHandler): name = self.file_name, content_type = self.content_type, size = file_size, - charset = self.charset + charset = self.charset, + content_type_extra = self.content_type_extra ) diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index eeb435fa57..9a5193e49b 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -50,7 +50,7 @@ class MultiPartParser(object): The raw post data, as a file-like object. :upload_handlers: A list of UploadHandler instances that perform operations on the uploaded - data. + data. :encoding: The encoding with which to treat the incoming data. """ @@ -177,11 +177,9 @@ class MultiPartParser(object): file_name = force_text(file_name, encoding, errors='replace') file_name = self.IE_sanitize(unescape_entities(file_name)) - content_type = meta_data.get('content-type', ('',))[0].strip() - try: - charset = meta_data.get('content-type', (0, {}))[1].get('charset', None) - except: - charset = None + content_type, content_type_extra = meta_data.get('content-type', ('', {})) + content_type = content_type.strip() + charset = content_type_extra.get('charset') try: content_length = int(meta_data.get('content-length')[0]) @@ -194,7 +192,7 @@ class MultiPartParser(object): try: handler.new_file(field_name, file_name, content_type, content_length, - charset) + charset, content_type_extra) except StopFutureHandlers: break diff --git a/django/test/client.py b/django/test/client.py index 94cfada725..8dbd33ccf1 100644 --- a/django/test/client.py +++ b/django/test/client.py @@ -180,7 +180,11 @@ def encode_multipart(boundary, data): def encode_file(boundary, key, file): to_bytes = lambda s: force_bytes(s, settings.DEFAULT_CHARSET) - content_type = mimetypes.guess_type(file.name)[0] + if hasattr(file, 'content_type'): + content_type = file.content_type + else: + content_type = mimetypes.guess_type(file.name)[0] + if content_type is None: content_type = 'application/octet-stream' return [ diff --git a/docs/releases/1.7.txt b/docs/releases/1.7.txt index b5e0b27508..8d967ff469 100644 --- a/docs/releases/1.7.txt +++ b/docs/releases/1.7.txt @@ -30,6 +30,14 @@ In addition, the widgets now display a help message when the browser and server time zone are different, to clarify how the value inserted in the field will be interpreted. +Minor features +~~~~~~~~~~~~~~ + +* The new :attr:`UploadedFile.content_type_extra + ` attribute + contains extra parameters passed to the ``content-type`` header on a file + upload. + Backwards incompatible changes in 1.7 ===================================== @@ -41,6 +49,14 @@ Backwards incompatible changes in 1.7 deprecation timeline for a given feature, its removal may appear as a backwards incompatible change. +Miscellaneous +~~~~~~~~~~~~~ + +* The :meth:`django.core.files.uploadhandler.FileUploadHandler.new_file()` + method is now passed an additional ``content_type_extra`` parameter. If you + have a custom :class:`~django.core.files.uploadhandler.FileUploadHandler` + that implements ``new_file()``, be sure it accepts this new parameter. + Features deprecated in 1.7 ========================== diff --git a/docs/topics/http/file-uploads.txt b/docs/topics/http/file-uploads.txt index f6fa27e27c..2cdab9ea9b 100644 --- a/docs/topics/http/file-uploads.txt +++ b/docs/topics/http/file-uploads.txt @@ -240,6 +240,18 @@ In addition to those inherited from :class:`~django.core.files.File`, all need to validate that the file contains the content that the content-type header claims -- "trust but verify." +.. attribute:: UploadedFile.content_type_extra + + .. versionadded:: 1.7 + + A dictionary containing extra parameters passed to the ``content-type`` + header. This is typically provided by services, such as Google App Engine, + that intercept and handle file uploads on your behalf. As a result your + handler may not receive the uploaded file content, but instead a URL or + other pointer to the file. (see `RFC 2388`_ section 5.3). + + .. _RFC 2388: http://www.ietf.org/rfc/rfc2388.txt + .. attribute:: UploadedFile.charset For :mimetype:`text/*` content-types, the character set (i.e. ``utf8``) @@ -350,6 +362,10 @@ list:: Writing custom upload handlers ------------------------------ +.. currentmodule:: django.core.files.uploadhandler + +.. class:: FileUploadHandler + All file upload handlers should be subclasses of ``django.core.files.uploadhandler.FileUploadHandler``. You can define upload handlers wherever you wish. @@ -359,7 +375,8 @@ Required methods Custom file upload handlers **must** define the following methods: -``FileUploadHandler.receive_data_chunk(self, raw_data, start)`` +.. method:: FileUploadHandler.receive_data_chunk(self, raw_data, start) + Receives a "chunk" of data from the file upload. ``raw_data`` is a byte string containing the uploaded data. @@ -379,7 +396,8 @@ Custom file upload handlers **must** define the following methods: If you raise a ``StopUpload`` or a ``SkipFile`` exception, the upload will abort or the file will be completely skipped. -``FileUploadHandler.file_complete(self, file_size)`` +.. method:: FileUploadHandler.file_complete(self, file_size) + Called when a file has finished uploading. The handler should return an ``UploadedFile`` object that will be stored @@ -392,7 +410,8 @@ Optional methods Custom upload handlers may also define any of the following optional methods or attributes: -``FileUploadHandler.chunk_size`` +.. attribute:: FileUploadHandler.chunk_size + Size, in bytes, of the "chunks" Django should store into memory and feed into the handler. That is, this attribute controls the size of chunks fed into ``FileUploadHandler.receive_data_chunk``. @@ -404,7 +423,8 @@ attributes: The default is 64*2\ :sup:`10` bytes, or 64 KB. -``FileUploadHandler.new_file(self, field_name, file_name, content_type, content_length, charset)`` +.. method:: FileUploadHandler.new_file(self, field_name, file_name, content_type, content_length, charset, content_type_extra) + Callback signaling that a new file upload is starting. This is called before any data has been fed to any upload handlers. @@ -421,13 +441,23 @@ attributes: ``charset`` is the character set (i.e. ``utf8``) given by the browser. Like ``content_length``, this sometimes won't be provided. + ``content_type_extra`` is extra information about the file from the + ``content-type`` header. See :attr:`UploadedFile.content_type_extra + `. + This method may raise a ``StopFutureHandlers`` exception to prevent future handlers from handling this file. -``FileUploadHandler.upload_complete(self)`` + .. versionadded:: 1.7 + + The ``content_type_extra`` parameter was added. + +.. method:: FileUploadHandler.upload_complete(self) + Callback signaling that the entire upload (all files) has completed. -``FileUploadHandler.handle_raw_input(self, input_data, META, content_length, boundary, encoding)`` +.. method:: FileUploadHandler.handle_raw_input(self, input_data, META, content_length, boundary, encoding) + Allows the handler to completely override the parsing of the raw HTTP input. diff --git a/tests/file_uploads/tests.py b/tests/file_uploads/tests.py index 66bc4465a2..f5a9c10be4 100644 --- a/tests/file_uploads/tests.py +++ b/tests/file_uploads/tests.py @@ -187,6 +187,27 @@ class FileUploadTests(TestCase): got = json.loads(self.client.request(**r).content.decode('utf-8')) self.assertTrue(len(got['file']) < 256, "Got a long file name (%s characters)." % len(got['file'])) + def test_content_type_extra(self): + """Uploaded files may have content type parameters available.""" + tdir = tempfile.gettempdir() + + no_content_type = tempfile.NamedTemporaryFile(suffix=".ctype_extra", dir=tdir) + no_content_type.write(b'something') + no_content_type.seek(0) + + simple_file = tempfile.NamedTemporaryFile(suffix=".ctype_extra", dir=tdir) + simple_file.write(b'something') + simple_file.seek(0) + simple_file.content_type = 'text/plain; test-key=test_value' + + response = self.client.post('/file_uploads/echo_content_type_extra/', { + 'no_content_type': no_content_type, + 'simple_file': simple_file, + }) + received = json.loads(response.content.decode('utf-8')) + self.assertEqual(received['no_content_type'], {}) + self.assertEqual(received['simple_file'], {'test-key': 'test_value'}) + def test_truncated_multipart_handled_gracefully(self): """ If passed an incomplete multipart message, MultiPartParser does not diff --git a/tests/file_uploads/urls.py b/tests/file_uploads/urls.py index fc5576828f..96efaaa5d8 100644 --- a/tests/file_uploads/urls.py +++ b/tests/file_uploads/urls.py @@ -10,6 +10,7 @@ urlpatterns = patterns('', (r'^verify/$', views.file_upload_view_verify), (r'^unicode_name/$', views.file_upload_unicode_name), (r'^echo/$', views.file_upload_echo), + (r'^echo_content_type_extra/$', views.file_upload_content_type_extra), (r'^echo_content/$', views.file_upload_echo_content), (r'^quota/$', views.file_upload_quota), (r'^quota/broken/$', views.file_upload_quota_broken), diff --git a/tests/file_uploads/views.py b/tests/file_uploads/views.py index eb7b654c09..d1bd2cf44e 100644 --- a/tests/file_uploads/views.py +++ b/tests/file_uploads/views.py @@ -7,7 +7,7 @@ import os from django.core.files.uploadedfile import UploadedFile from django.http import HttpResponse, HttpResponseServerError from django.utils import six -from django.utils.encoding import force_bytes +from django.utils.encoding import force_bytes, smart_str from .models import FileModel from .tests import UNICODE_FILENAME, UPLOAD_TO @@ -136,3 +136,14 @@ def file_upload_filename_case_view(request): obj = FileModel() obj.testfile.save(file.name, file) return HttpResponse('%d' % obj.pk) + +def file_upload_content_type_extra(request): + """ + Simple view to echo back extra content-type parameters. + """ + params = {} + for file_name, uploadedfile in request.FILES.items(): + params[file_name] = dict([ + (k, smart_str(v)) for k, v in uploadedfile.content_type_extra.items() + ]) + return HttpResponse(json.dumps(params))