Fixed #26029 -- Allowed configuring custom file storage backends.

This commit is contained in:
Jarosław Wygoda 2023-01-11 10:48:57 +01:00 committed by Mariusz Felisiak
parent d02a9f0cee
commit 1ec3f0961f
12 changed files with 209 additions and 11 deletions

View File

@ -280,6 +280,8 @@ SECRET_KEY_FALLBACKS = []
# Default file storage mechanism that holds media.
DEFAULT_FILE_STORAGE = "django.core.files.storage.FileSystemStorage"
STORAGES = {}
# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
MEDIA_ROOT = ""

View File

@ -4,6 +4,7 @@ from django.utils.module_loading import import_string
from .base import Storage
from .filesystem import FileSystemStorage
from .handler import InvalidStorageError, StorageHandler
from .memory import InMemoryStorage
__all__ = (
@ -13,6 +14,9 @@ __all__ = (
"DefaultStorage",
"default_storage",
"get_storage_class",
"InvalidStorageError",
"StorageHandler",
"storages",
)
@ -25,4 +29,5 @@ class DefaultStorage(LazyObject):
self._wrapped = get_storage_class()()
storages = StorageHandler()
default_storage = DefaultStorage()

View File

@ -0,0 +1,46 @@
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.utils.functional import cached_property
from django.utils.module_loading import import_string
class InvalidStorageError(ImproperlyConfigured):
pass
class StorageHandler:
def __init__(self, backends=None):
# backends is an optional dict of storage backend definitions
# (structured like settings.STORAGES).
self._backends = backends
self._storages = {}
@cached_property
def backends(self):
if self._backends is None:
self._backends = settings.STORAGES.copy()
return self._backends
def __getitem__(self, alias):
try:
return self._storages[alias]
except KeyError:
try:
params = self.backends[alias]
except KeyError:
raise InvalidStorageError(
f"Could not find config for '{alias}' in settings.STORAGES."
)
storage = self.create_storage(params)
self._storages[alias] = storage
return storage
def create_storage(self, params):
params = params.copy()
backend = params.pop("BACKEND")
options = params.pop("OPTIONS", {})
try:
storage_cls = import_string(backend)
except ImportError as e:
raise InvalidStorageError(f"Could not find backend {backend!r}: {e}") from e
return storage_cls(**options)

View File

@ -111,6 +111,23 @@ def reset_template_engines(*, setting, **kwargs):
get_default_renderer.cache_clear()
@receiver(setting_changed)
def storages_changed(*, setting, **kwargs):
from django.core.files.storage import storages
if setting in (
"STORAGES",
"STATIC_ROOT",
"STATIC_URL",
):
try:
del storages.backends
except AttributeError:
pass
storages._backends = None
storages._storages = {}
@receiver(setting_changed)
def clear_serializers_cache(*, setting, **kwargs):
if setting == "SERIALIZATION_MODULES":

View File

@ -116,3 +116,23 @@ free unique filename cannot be found, a :exc:`SuspiciousFileOperation
If a file with ``name`` already exists, ``get_alternative_name()`` is called to
obtain an alternative name.
.. _using-custom-storage-engine:
Use your custom storage engine
==============================
.. versionadded:: 4.2
The first step to using your custom storage with Django is to tell Django about
the file storage backend you'll be using. This is done using the
:setting:`STORAGES` setting. This setting maps storage aliases, which are a way
to refer to a specific storage throughout Django, to a dictionary of settings
for that specific storage backend. The settings in the inner dictionaries are
described fully in the :setting:`STORAGES` documentation.
Storages are then accessed by alias from from the
:data:`django.core.files.storage.storages` dictionary::
from django.core.files.storage import storages
example_storage = storages["example"]

View File

@ -23,6 +23,7 @@ Settings
See :ref:`staticfiles settings <settings-staticfiles>` for details on the
following settings:
* :setting:`STORAGES`
* :setting:`STATIC_ROOT`
* :setting:`STATIC_URL`
* :setting:`STATICFILES_DIRS`

View File

@ -9,6 +9,12 @@ Getting the default storage class
Django provides convenient ways to access the default storage class:
.. data:: storages
.. versionadded:: 4.2
Storage instances as defined by :setting:`STORAGES`.
.. class:: DefaultStorage
:class:`~django.core.files.storage.DefaultStorage` provides

View File

@ -2606,6 +2606,43 @@ Silenced checks will not be output to the console.
See also the :doc:`/ref/checks` documentation.
.. setting:: STORAGES
``STORAGES``
------------
.. versionadded:: 4.2
Default::
{}
A dictionary containing the settings for all storages to be used with Django.
It is a nested dictionary whose contents map a storage alias to a dictionary
containing the options for an individual storage.
Storages can have any alias you choose.
The following is an example ``settings.py`` snippet defining a custom file
storage called ``example``::
STORAGES = {
# ...
"example": {
"BACKEND": "django.core.files.storage.FileSystemStorage",
"OPTIONS": {
"location": "/example",
"base_url": "/example/",
},
},
}
``OPTIONS`` are passed to the ``BACKEND`` on initialization in ``**kwargs``.
A ready-to-use instance of the storage backends can be retrieved from
:data:`django.core.files.storage.storages`. Use a key corresponding to the
backend definition in :setting:`STORAGES`.
.. setting:: TEMPLATES
``TEMPLATES``
@ -3663,6 +3700,7 @@ File uploads
* :setting:`FILE_UPLOAD_TEMP_DIR`
* :setting:`MEDIA_ROOT`
* :setting:`MEDIA_URL`
* :setting:`STORAGES`
Forms
-----

View File

@ -91,6 +91,12 @@ In-memory file storage
The new ``django.core.files.storage.InMemoryStorage`` class provides a
non-persistent storage useful for speeding up tests by avoiding disk access.
Custom file storages
--------------------
The new :setting:`STORAGES` setting allows configuring multiple custom file
storage backends.
Minor features
--------------

View File

@ -239,3 +239,15 @@ For example::
class MyModel(models.Model):
my_file = models.FileField(storage=select_storage)
In order to set a storage defined in the :setting:`STORAGES` setting you can
use a lambda function::
from django.core.files.storage import storages
class MyModel(models.Model):
upload = models.FileField(storage=lambda: storages["custom_storage"])
.. versionchanged:: 4.2
Support for ``storages`` was added.

View File

@ -1441,15 +1441,16 @@ when settings are changed.
Django itself uses this signal to reset various data:
================================ ========================
Overridden settings Data reset
================================ ========================
USE_TZ, TIME_ZONE Databases timezone
TEMPLATES Template engines
SERIALIZATION_MODULES Serializers cache
LOCALE_PATHS, LANGUAGE_CODE Default translation and loaded translations
MEDIA_ROOT, DEFAULT_FILE_STORAGE Default file storage
================================ ========================
================================= ========================
Overridden settings Data reset
================================= ========================
USE_TZ, TIME_ZONE Databases timezone
TEMPLATES Template engines
SERIALIZATION_MODULES Serializers cache
LOCALE_PATHS, LANGUAGE_CODE Default translation and loaded translations
MEDIA_ROOT, DEFAULT_FILE_STORAGE Default file storage
STATIC_ROOT, STATIC_URL, STORAGES Storages configuration
================================= ========================
Isolating apps
--------------

View File

@ -14,9 +14,14 @@ from urllib.request import urlopen
from django.core.cache import cache
from django.core.exceptions import SuspiciousFileOperation
from django.core.files.base import ContentFile, File
from django.core.files.storage import FileSystemStorage
from django.core.files.storage import FileSystemStorage, InvalidStorageError
from django.core.files.storage import Storage as BaseStorage
from django.core.files.storage import default_storage, get_storage_class
from django.core.files.storage import (
StorageHandler,
default_storage,
get_storage_class,
storages,
)
from django.core.files.uploadedfile import (
InMemoryUploadedFile,
SimpleUploadedFile,
@ -1157,3 +1162,42 @@ class FileLikeObjectTestCase(LiveServerTestCase):
remote_file = urlopen(self.live_server_url + "/")
with self.storage.open(stored_filename) as stored_file:
self.assertEqual(stored_file.read(), remote_file.read())
class StorageHandlerTests(SimpleTestCase):
@override_settings(
STORAGES={
"custom_storage": {
"BACKEND": "django.core.files.storage.FileSystemStorage",
},
}
)
def test_same_instance(self):
cache1 = storages["custom_storage"]
cache2 = storages["custom_storage"]
self.assertIs(cache1, cache2)
def test_defaults(self):
storages = StorageHandler()
self.assertEqual(storages.backends, {})
def test_nonexistent_alias(self):
msg = "Could not find config for 'nonexistent' in settings.STORAGES."
storages = StorageHandler()
with self.assertRaisesMessage(InvalidStorageError, msg):
storages["nonexistent"]
def test_nonexistent_backend(self):
test_storages = StorageHandler(
{
"invalid_backend": {
"BACKEND": "django.nonexistent.NonexistentBackend",
},
}
)
msg = (
"Could not find backend 'django.nonexistent.NonexistentBackend': "
"No module named 'django.nonexistent'"
)
with self.assertRaisesMessage(InvalidStorageError, msg):
test_storages["invalid_backend"]