innov
/
django1
1
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

Fixed #7614: the quickening has come, and there now is only one UploadedFile. On top of that, UploadedFile's interface has been improved: 

* The API now more closely matches a proper file API. This unfortunately means a few backwards-incompatible renamings; see BackwardsIncompatibleChanges. This refs #7593.
  * While we were at it, renamed chunk() to chunks() to clarify that it's an iterator.
  * Temporary uploaded files now property use the tempfile library behind the scenes which should ensure better cleanup of tempfiles (refs #7593 again).

Thanks to Mike Axiak for the bulk of this patch.


git-svn-id: http://code.djangoproject.com/svn/django/trunk@7859 bcc190cf-cafb-0310-a4f2-bffc1f526a37
This commit is contained in:
Jacob Kaplan-Moss 2008-07-07 23:16:00 +00:00
parent 9dabd1f8ff
commit a28b75b0ba
12 changed files with 165 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
django/core/files/uploadedfile.py
View File

@ -3,12 +3,40 @@ Classes representing uploaded files.
"""
import os
import tempfile
import warnings
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile')
from django.conf import settings
__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile', 'SimpleUploadedFile')
# Because we fooled around with it a bunch, UploadedFile has a bunch
# of deprecated properties. This little shortcut helps define 'em
# without too much code duplication.
def deprecated_property(old, new, readonly=False):
def issue_warning():
warnings.warn(
message = "UploadedFile.%s is deprecated; use UploadedFile.%s instead." % (old, new),
category = DeprecationWarning,
stacklevel = 3
)
def getter(self):
issue_warning()
return getattr(self, new)
def setter(self, value):
issue_warning()
setattr(self, new, value)
if readonly:
return property(getter)
else:
return property(getter, setter)
class UploadedFile(object):
"""
@ -20,16 +48,19 @@ class UploadedFile(object):
"""
DEFAULT_CHUNK_SIZE = 64 * 2**10
def __init__(self, file_name=None, content_type=None, file_size=None, charset=None):
self.file_name = file_name
self.file_size = file_size
def __init__(self, name=None, content_type=None, size=None, charset=None):
self.name = name
self.size = size
self.content_type = content_type
self.charset = charset
def __repr__(self):
return "<%s: %s (%s)>" % (self.__class__.__name__, self.file_name, self.content_type)
return "<%s: %s (%s)>" % (self.__class__.__name__, self.name, self.content_type)
def _set_file_name(self, name):
def _get_name(self):
return self._name
def _set_name(self, name):
# Sanitize the file name so that it can't be dangerous.
if name is not None:
# Just use the basename of the file -- anything else is dangerous.
@ -40,14 +71,11 @@ class UploadedFile(object):
name, ext = os.path.splitext(name)
name = name[:255 - len(ext)] + ext
self._file_name = name
self._name = name
def _get_file_name(self):
return self._file_name
name = property(_get_name, _set_name)
file_name = property(_get_file_name, _set_file_name)
def chunk(self, chunk_size=None):
def chunks(self, chunk_size=None):
"""
Read the file and yield chucks of ``chunk_size`` bytes (defaults to
``UploadedFile.DEFAULT_CHUNK_SIZE``).
@ -58,12 +86,18 @@ class UploadedFile(object):
if hasattr(self, 'seek'):
self.seek(0)
# Assume the pointer is at zero...
counter = self.file_size
counter = self.size
while counter > 0:
yield self.read(chunk_size)
counter -= chunk_size
# Deprecated properties
file_name = deprecated_property(old="file_name", new="name")
file_size = deprecated_property(old="file_size", new="size")
data = deprecated_property(old="data", new="read", readonly=True)
chunk = deprecated_property(old="chunk", new="chunks", readonly=True)
def multiple_chunks(self, chunk_size=None):
"""
Returns ``True`` if you can expect multiple chunks.
@ -74,9 +108,9 @@ class UploadedFile(object):
"""
if not chunk_size:
chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
return self.file_size < chunk_size
return self.size > chunk_size
# Abstract methods; subclasses *must* default read() and probably should
# Abstract methods; subclasses *must* define read() and probably should
# define open/close.
def read(self, num_bytes=None):
raise NotImplementedError()
@ -87,23 +121,49 @@ class UploadedFile(object):
def close(self):
pass
def xreadlines(self):
return self
def readlines(self):
return list(self.xreadlines())
def __iter__(self):
# Iterate over this file-like object by newlines
buffer_ = None
for chunk in self.chunks():
chunk_buffer = StringIO(chunk)
for line in chunk_buffer:
if buffer_:
line = buffer_ + line
buffer_ = None
# If this is the end of a line, yield
# otherwise, wait for the next round
if line[-1] in ('\n', '\r'):
yield line
else:
buffer_ = line
if buffer_ is not None:
yield buffer_
# Backwards-compatible support for uploaded-files-as-dictionaries.
def __getitem__(self, key):
import warnings
warnings.warn(
message = "The dictionary access of uploaded file objects is deprecated. Use the new object interface instead.",
category = DeprecationWarning,
stacklevel = 2
)
backwards_translate = {
'filename': 'file_name',
'filename': 'name',
'content-type': 'content_type',
}
if key == 'content':
return self.read()
elif key == 'filename':
return self.file_name
return self.name
elif key == 'content-type':
return self.content_type
else:
@ -113,34 +173,36 @@ class TemporaryUploadedFile(UploadedFile):
"""
A file uploaded to a temporary location (i.e. stream-to-disk).
"""
def __init__(self, file, file_name, content_type, file_size, charset):
super(TemporaryUploadedFile, self).__init__(file_name, content_type, file_size, charset)
self.file = file
self.path = file.name
self.file.seek(0)
def __init__(self, name, content_type, size, charset):
super(TemporaryUploadedFile, self).__init__(name, content_type, size, charset)
if settings.FILE_UPLOAD_TEMP_DIR:
self._file = tempfile.NamedTemporaryFile(suffix='.upload', dir=settings.FILE_UPLOAD_TEMP_DIR)
else:
self._file = tempfile.NamedTemporaryFile(suffix='.upload')
def temporary_file_path(self):
"""
Returns the full path of this file.
"""
return self.path
return self.name
def read(self, *args, **kwargs):
return self.file.read(*args, **kwargs)
def open(self):
self.seek(0)
def seek(self, *args, **kwargs):
self.file.seek(*args, **kwargs)
# Most methods on this object get proxied to NamedTemporaryFile.
# We can't directly subclass because NamedTemporaryFile is actually a
# factory function
def read(self, *args): return self._file.read(*args)
def seek(self, offset): return self._file.seek(offset)
def write(self, s): return self._file.write(s)
def close(self): return self._file.close()
def __iter__(self): return iter(self._file)
def readlines(self, size=None): return self._file.readlines(size)
def xreadlines(self): return self._file.xreadlines()
class InMemoryUploadedFile(UploadedFile):
"""
A file uploaded into memory (i.e. stream-to-memory).
"""
def __init__(self, file, field_name, file_name, content_type, file_size, charset):
super(InMemoryUploadedFile, self).__init__(file_name, content_type, file_size, charset)
def __init__(self, file, field_name, name, content_type, size, charset):
super(InMemoryUploadedFile, self).__init__(name, content_type, size, charset)
self.file = file
self.field_name = field_name
self.file.seek(0)
@ -154,7 +216,7 @@ class InMemoryUploadedFile(UploadedFile):
def read(self, *args, **kwargs):
return self.file.read(*args, **kwargs)
def chunk(self, chunk_size=None):
def chunks(self, chunk_size=None):
self.file.seek(0)
yield self.read()
@ -168,9 +230,9 @@ class SimpleUploadedFile(InMemoryUploadedFile):
"""
def __init__(self, name, content, content_type='text/plain'):
self.file = StringIO(content or '')
self.file_name = name
self.name = name
self.field_name = None
self.file_size = len(content or '')
self.size = len(content or '')
self.content_type = content_type
self.charset = None
self.file.seek(0)

43
django/core/files/uploadhandler.py
View File

@ -132,21 +132,15 @@ class TemporaryFileUploadHandler(FileUploadHandler):
Create the file object to append to as data is coming in.
"""
super(TemporaryFileUploadHandler, self).new_file(file_name, *args, **kwargs)
self.file = TemporaryFile(settings.FILE_UPLOAD_TEMP_DIR)
self.write = self.file.write
self.file = TemporaryUploadedFile(self.file_name, self.content_type, 0, self.charset)
def receive_data_chunk(self, raw_data, start):
self.write(raw_data)
self.file.write(raw_data)
def file_complete(self, file_size):
self.file.seek(0)
return TemporaryUploadedFile(
file = self.file,
file_name = self.file_name,
content_type = self.content_type,
file_size = file_size,
charset = self.charset
)
self.file.size = file_size
return self.file
class MemoryFileUploadHandler(FileUploadHandler):
"""
@ -189,37 +183,12 @@ class MemoryFileUploadHandler(FileUploadHandler):
return InMemoryUploadedFile(
file = self.file,
field_name = self.field_name,
file_name = self.file_name,
name = self.file_name,
content_type = self.content_type,
file_size = file_size,
size = file_size,
charset = self.charset
)
class TemporaryFile(object):
"""
A temporary file that tries to delete itself when garbage collected.
"""
def __init__(self, dir):
if not dir:
dir = tempfile.gettempdir()
try:
(fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir)
self.file = os.fdopen(fd, 'w+b')
except (OSError, IOError):
raise OSError("Could not create temporary file for uploading, have you set settings.FILE_UPLOAD_TEMP_DIR correctly?")
self.name = name
def __getattr__(self, name):
a = getattr(self.__dict__['file'], name)
if type(a) != type(0):
setattr(self, name, a)
return a
def __del__(self):
try:
os.unlink(self.name)
except OSError:
pass
def load_handler(path, *args, **kwargs):
"""

2
django/db/models/base.py
View File

@ -536,7 +536,7 @@ class Model(object):
# This is a normal uploadedfile that we can stream.
fp = open(full_filename, 'wb')
locks.lock(fp, locks.LOCK_EX)
for chunk in raw_field.chunk():
for chunk in raw_field.chunks():
fp.write(chunk)
locks.unlock(fp)
fp.close()

11
django/db/models/fields/__init__.py
View File

@ -766,8 +766,11 @@ class FileField(Field):
def get_db_prep_save(self, value):
"Returns field's value prepared for saving into a database."
# Need to convert UploadedFile objects provided via a form to unicode for database insertion
if value is None:
if hasattr(value, 'name'):
return value.name
elif value is None:
return None
else:
return unicode(value)
def get_manipulator_fields(self, opts, manipulator, change, name_prefix='', rel=False, follow=True):
@ -842,7 +845,7 @@ class FileField(Field):
# We don't need to raise a warning because Model._save_FIELD_file will
# do so for us.
try:
file_name = file.file_name
file_name = file.name
except AttributeError:
file_name = file['filename']
@ -857,9 +860,9 @@ class FileField(Field):
return os.path.normpath(f)
def save_form_data(self, instance, data):
from django.newforms.fields import UploadedFile
from django.core.files.uploadedfile import UploadedFile
if data and isinstance(data, UploadedFile):
getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False)
getattr(instance, "save_%s_file" % self.name)(data.name, data, save=False)
def formfield(self, **kwargs):
defaults = {'form_class': forms.FileField}

25
django/newforms/fields.py
View File

@ -27,7 +27,7 @@ from django.utils.encoding import StrAndUnicode, smart_unicode, smart_str
from util import ErrorList, ValidationError
from widgets import TextInput, PasswordInput, HiddenInput, MultipleHiddenInput, FileInput, CheckboxInput, Select, NullBooleanSelect, SelectMultiple, DateTimeInput
from django.core.files.uploadedfile import SimpleUploadedFile as UploadedFile
__all__ = (
'Field', 'CharField', 'IntegerField',
@ -419,18 +419,6 @@ except ImportError:
# It's OK if Django settings aren't configured.
URL_VALIDATOR_USER_AGENT = 'Django (http://www.djangoproject.com/)'
class UploadedFile(StrAndUnicode):
"A wrapper for files uploaded in a FileField"
def __init__(self, filename, data):
self.filename = filename
self.data = data
def __unicode__(self):
"""
The unicode representation is the filename, so that the pre-database-insertion
logic can use UploadedFile objects
"""
return self.filename
class FileField(Field):
widget = FileInput
@ -460,15 +448,12 @@ class FileField(Field):
category = DeprecationWarning,
stacklevel = 2
)
data = UploadedFile(data['filename'], data['content'])
try:
file_name = data.file_name
file_size = data.file_size
file_name = data.name
file_size = data.size
except AttributeError:
try:
file_name = data.get('filename')
file_size = bool(data['content'])
except (AttributeError, KeyError):
raise ValidationError(self.error_messages['invalid'])
if not file_name:
@ -476,7 +461,7 @@ class FileField(Field):
if not file_size:
raise ValidationError(self.error_messages['empty'])
return UploadedFile(file_name, data)
return data
class ImageField(FileField):
default_error_messages = {

2
django/oldforms/__init__.py
View File

@ -686,7 +686,7 @@ class FileUploadField(FormField):
if upload_errors:
raise validators.CriticalValidationError, upload_errors
try:
file_size = new_data.file_size
file_size = new_data.size
except AttributeError:
file_size = len(new_data['content'])
if not file_size:

15
docs/newforms.txt
View File

@ -1331,23 +1331,12 @@ given length.
* Validates that non-empty file data has been bound to the form.
* Error message keys: ``required``, ``invalid``, ``missing``, ``empty``
An ``UploadedFile`` object has two attributes:
====================== ====================================================
Attribute Description
====================== ====================================================
``filename`` The name of the file, provided by the uploading
client.
``content`` The array of bytes comprising the file content.
====================== ====================================================
The string representation of an ``UploadedFile`` is the same as the filename
attribute.
To learn more about the ``UploadedFile`` object, see the `file uploads documentation`_.
When you use a ``FileField`` in a form, you must also remember to
`bind the file data to the form`_.
.. _file uploads documentation: ../upload_handling/
.. _`bind the file data to the form`: `Binding uploaded files to a form`_
``FilePathField``

2
docs/settings.txt
View File

@ -279,7 +279,7 @@ Default: ``''`` (Empty string)
The database backend to use. The build-in database backends are
``'postgresql_psycopg2'``, ``'postgresql'``, ``'mysql'``, ``'mysql_old'``,
``'sqlite3'``, ``'oracle'``, and ``'oracle'``.
``'sqlite3'``, and ``'oracle'``.
In the Django development version, you can use a database backend that doesn't
ship with Django by setting ``DATABASE_ENGINE`` to a fully-qualified path (i.e.

12
docs/upload_handling.txt
View File

@ -64,7 +64,7 @@ methods to access the uploaded content:
``UploadedFile.read()``
Read the entire uploaded data from the file. Be careful with this
method: if the uploaded file is huge it can overwhelm your system if you
try to read it into memory. You'll probably want to use ``chunk()``
try to read it into memory. You'll probably want to use ``chunks()``
instead; see below.
``UploadedFile.multiple_chunks()``
@ -161,13 +161,13 @@ All ``UploadedFile`` objects define the following methods/attributes:
Returns a byte string of length ``num_bytes``, or the complete file if
``num_bytes`` is ``None``.
``UploadedFile.chunk(self, chunk_size=None)``
``UploadedFile.chunks(self, chunk_size=None)``
A generator yielding small chunks from the file. If ``chunk_size`` isn't
given, chunks will be 64 kb.
given, chunks will be 64 KB.
``UploadedFile.multiple_chunks(self, chunk_size=None)``
Returns ``True`` if you can expect more than one chunk when calling
``UploadedFile.chunk(self, chunk_size)``.
``UploadedFile.chunks(self, chunk_size)``.
``UploadedFile.file_size``
The size, in bytes, of the uploaded file.
@ -186,10 +186,14 @@ All ``UploadedFile`` objects define the following methods/attributes:
For ``text/*`` content-types, the character set (i.e. ``utf8``) supplied
by the browser. Again, "trust but verify" is the best policy here.
``UploadedFile.__iter__()``
Iterates over the lines in the file.
``UploadedFile.temporary_file_path()``
Only files uploaded onto disk will have this method; it returns the full
path to the temporary uploaded file.
Upload Handlers
===============

8
tests/modeltests/model_forms/models.py
View File

@ -803,7 +803,7 @@ False
>>> f.is_valid()
True
>>> type(f.cleaned_data['file'])
<class 'django.newforms.fields.UploadedFile'>
<class 'django.core.files.uploadedfile.SimpleUploadedFile'>
>>> instance = f.save()
>>> instance.file
u'...test1.txt'
@ -814,7 +814,7 @@ u'...test1.txt'
>>> f.is_valid()
True
>>> type(f.cleaned_data['file'])
<class 'django.newforms.fields.UploadedFile'>
<class 'django.core.files.uploadedfile.SimpleUploadedFile'>
>>> instance = f.save()
>>> instance.file
u'...test1.txt'
@ -906,7 +906,7 @@ u'...test3.txt'
>>> f.is_valid()
True
>>> type(f.cleaned_data['image'])
<class 'django.newforms.fields.UploadedFile'>
<class 'django.core.files.uploadedfile.SimpleUploadedFile'>
>>> instance = f.save()
>>> instance.image
u'...test.png'
@ -918,7 +918,7 @@ u'...test.png'
>>> f.is_valid()
True
>>> type(f.cleaned_data['image'])
<class 'django.newforms.fields.UploadedFile'>
<class 'django.core.files.uploadedfile.SimpleUploadedFile'>
>>> instance = f.save()
>>> instance.image
u'...test.png'

6
tests/regressiontests/file_uploads/views.py
View File

@ -15,7 +15,7 @@ def file_upload_view(request):
if isinstance(form_data.get('file_field'), UploadedFile) and isinstance(form_data['name'], unicode):
# If a file is posted, the dummy client should only post the file name,
# not the full path.
if os.path.dirname(form_data['file_field'].file_name) != '':
if os.path.dirname(form_data['file_field'].name) != '':
return HttpResponseServerError()
return HttpResponse('')
else:
@ -29,7 +29,7 @@ def file_upload_view_verify(request):
form_data.update(request.FILES)
# Check to see if unicode names worked out.
if not request.FILES['file_unicode'].file_name.endswith(u'test_\u4e2d\u6587_Orl\xe9ans.jpg'):
if not request.FILES['file_unicode'].name.endswith(u'test_\u4e2d\u6587_Orl\xe9ans.jpg'):
return HttpResponseServerError()
for key, value in form_data.items():
@ -51,7 +51,7 @@ def file_upload_echo(request):
"""
Simple view to echo back info about uploaded files for tests.
"""
r = dict([(k, f.file_name) for k, f in request.FILES.items()])
r = dict([(k, f.name) for k, f in request.FILES.items()])
return HttpResponse(simplejson.dumps(r))
def file_upload_quota(request):

4
tests/regressiontests/forms/fields.py
View File

@ -800,10 +800,10 @@ Traceback (most recent call last):
ValidationError: [u'The submitted file is empty.']
>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content')))
<class 'django.newforms.fields.UploadedFile'>
<class 'django.core.files.uploadedfile.SimpleUploadedFile'>
>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'), 'files/test4.pdf'))
<class 'django.newforms.fields.UploadedFile'>
<class 'django.core.files.uploadedfile.SimpleUploadedFile'>
# URLField ##################################################################