Fixed #26029 -- Allowed configuring custom file storage backends.

django/conf/global_settings.py

@@ -280,6 +280,8 @@ SECRET_KEY_FALLBACKS = []
 # Default file storage mechanism that holds media.
 DEFAULT_FILE_STORAGE = "django.core.files.storage.FileSystemStorage"
 
+STORAGES = {}
+
 # Absolute filesystem path to the directory that will hold user-uploaded files.
 # Example: "/var/www/example.com/media/"
 MEDIA_ROOT = ""

django/core/files/storage/__init__.py

@@ -4,6 +4,7 @@ from django.utils.module_loading import import_string
 
 from .base import Storage
 from .filesystem import FileSystemStorage
+from .handler import InvalidStorageError, StorageHandler
 from .memory import InMemoryStorage
 
 __all__ = (
@@ -13,6 +14,9 @@ __all__ = (
     "DefaultStorage",
     "default_storage",
     "get_storage_class",
+    "InvalidStorageError",
+    "StorageHandler",
+    "storages",
 )
 
 
@@ -25,4 +29,5 @@ class DefaultStorage(LazyObject):
         self._wrapped = get_storage_class()()
 
 
+storages = StorageHandler()
 default_storage = DefaultStorage()

django/core/files/storage/handler.py

@@ -0,0 +1,46 @@
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+from django.utils.functional import cached_property
+from django.utils.module_loading import import_string
+
+
+class InvalidStorageError(ImproperlyConfigured):
+    pass
+
+
+class StorageHandler:
+    def __init__(self, backends=None):
+        # backends is an optional dict of storage backend definitions
+        # (structured like settings.STORAGES).
+        self._backends = backends
+        self._storages = {}
+
+    @cached_property
+    def backends(self):
+        if self._backends is None:
+            self._backends = settings.STORAGES.copy()
+        return self._backends
+
+    def __getitem__(self, alias):
+        try:
+            return self._storages[alias]
+        except KeyError:
+            try:
+                params = self.backends[alias]
+            except KeyError:
+                raise InvalidStorageError(
+                    f"Could not find config for '{alias}' in settings.STORAGES."
+                )
+            storage = self.create_storage(params)
+            self._storages[alias] = storage
+            return storage
+
+    def create_storage(self, params):
+        params = params.copy()
+        backend = params.pop("BACKEND")
+        options = params.pop("OPTIONS", {})
+        try:
+            storage_cls = import_string(backend)
+        except ImportError as e:
+            raise InvalidStorageError(f"Could not find backend {backend!r}: {e}") from e
+        return storage_cls(**options)

django/test/signals.py

@@ -111,6 +111,23 @@ def reset_template_engines(*, setting, **kwargs):
         get_default_renderer.cache_clear()
 
 
+@receiver(setting_changed)
+def storages_changed(*, setting, **kwargs):
+    from django.core.files.storage import storages
+
+    if setting in (
+        "STORAGES",
+        "STATIC_ROOT",
+        "STATIC_URL",
+    ):
+        try:
+            del storages.backends
+        except AttributeError:
+            pass
+        storages._backends = None
+        storages._storages = {}
+
+
 @receiver(setting_changed)
 def clear_serializers_cache(*, setting, **kwargs):
     if setting == "SERIALIZATION_MODULES":

docs/howto/custom-file-storage.txt

@@ -116,3 +116,23 @@ free unique filename cannot be found, a :exc:`SuspiciousFileOperation
 
 If a file with ``name`` already exists, ``get_alternative_name()`` is called to
 obtain an alternative name.
+
+.. _using-custom-storage-engine:
+
+Use your custom storage engine
+==============================
+
+.. versionadded:: 4.2
+
+The first step to using your custom storage with Django is to tell Django about
+the file storage backend you'll be using. This is done using the
+:setting:`STORAGES` setting. This setting maps storage aliases, which are a way
+to refer to a specific storage throughout Django, to a dictionary of settings
+for that specific storage backend. The settings in the inner dictionaries are
+described fully in the :setting:`STORAGES` documentation.
+
+Storages are then accessed by alias from from the
+:data:`django.core.files.storage.storages` dictionary::
+
+    from django.core.files.storage import storages
+    example_storage = storages["example"]

docs/ref/contrib/staticfiles.txt

@@ -23,6 +23,7 @@ Settings
 See :ref:`staticfiles settings <settings-staticfiles>` for details on the
 following settings:
 
+* :setting:`STORAGES`
 * :setting:`STATIC_ROOT`
 * :setting:`STATIC_URL`
 * :setting:`STATICFILES_DIRS`

docs/ref/files/storage.txt

@@ -9,6 +9,12 @@ Getting the default storage class
 
 Django provides convenient ways to access the default storage class:
 
+.. data:: storages
+
+    .. versionadded:: 4.2
+
+    Storage instances as defined by :setting:`STORAGES`.
+
 .. class:: DefaultStorage
 
     :class:`~django.core.files.storage.DefaultStorage` provides

docs/ref/settings.txt

@@ -2606,6 +2606,43 @@ Silenced checks will not be output to the console.
 
 See also the :doc:`/ref/checks` documentation.
 
+.. setting:: STORAGES
+
+``STORAGES``
+------------
+
+.. versionadded:: 4.2
+
+Default::
+
+    {}
+
+A dictionary containing the settings for all storages to be used with Django.
+It is a nested dictionary whose contents map a storage alias to a dictionary
+containing the options for an individual storage.
+
+Storages can have any alias you choose.
+
+The following is an example ``settings.py`` snippet defining a custom file
+storage called ``example``::
+
+    STORAGES = {
+        # ...
+        "example": {
+            "BACKEND": "django.core.files.storage.FileSystemStorage",
+            "OPTIONS": {
+                "location": "/example",
+                "base_url": "/example/",
+            },
+        },
+    }
+
+``OPTIONS`` are passed to the ``BACKEND`` on initialization in ``**kwargs``.
+
+A ready-to-use instance of the storage backends can be retrieved from
+:data:`django.core.files.storage.storages`. Use a key corresponding to the
+backend definition in :setting:`STORAGES`.
+
 .. setting:: TEMPLATES
 
 ``TEMPLATES``
@@ -3663,6 +3700,7 @@ File uploads
 * :setting:`FILE_UPLOAD_TEMP_DIR`
 * :setting:`MEDIA_ROOT`
 * :setting:`MEDIA_URL`
+* :setting:`STORAGES`
 
 Forms
 -----

docs/releases/4.2.txt

@@ -91,6 +91,12 @@ In-memory file storage
 The new ``django.core.files.storage.InMemoryStorage`` class provides a
 non-persistent storage useful for speeding up tests by avoiding disk access.
 
+Custom file storages
+--------------------
+
+The new :setting:`STORAGES` setting allows configuring multiple custom file
+storage backends.
+
 Minor features
 --------------
 

docs/topics/files.txt

@@ -239,3 +239,15 @@ For example::
 
     class MyModel(models.Model):
         my_file = models.FileField(storage=select_storage)
+
+In order to set a storage defined in the :setting:`STORAGES` setting you can
+use a lambda function::
+
+    from django.core.files.storage import storages
+
+    class MyModel(models.Model):
+        upload = models.FileField(storage=lambda: storages["custom_storage"])
+
+.. versionchanged:: 4.2
+
+    Support for ``storages`` was added.

docs/topics/testing/tools.txt

@@ -1441,15 +1441,16 @@ when settings are changed.
 
 Django itself uses this signal to reset various data:
 
-================================ ========================
+================================= ========================
 Overridden settings               Data reset
-================================ ========================
+================================= ========================
 USE_TZ, TIME_ZONE                 Databases timezone
 TEMPLATES                         Template engines
 SERIALIZATION_MODULES             Serializers cache
 LOCALE_PATHS, LANGUAGE_CODE       Default translation and loaded translations
 MEDIA_ROOT, DEFAULT_FILE_STORAGE  Default file storage
-================================ ========================
+STATIC_ROOT, STATIC_URL, STORAGES Storages configuration
+================================= ========================
 
 Isolating apps
 --------------

tests/file_storage/tests.py

@@ -14,9 +14,14 @@ from urllib.request import urlopen
 from django.core.cache import cache
 from django.core.exceptions import SuspiciousFileOperation
 from django.core.files.base import ContentFile, File
-from django.core.files.storage import FileSystemStorage
+from django.core.files.storage import FileSystemStorage, InvalidStorageError
 from django.core.files.storage import Storage as BaseStorage
-from django.core.files.storage import default_storage, get_storage_class
+from django.core.files.storage import (
+    StorageHandler,
+    default_storage,
+    get_storage_class,
+    storages,
+)
 from django.core.files.uploadedfile import (
     InMemoryUploadedFile,
     SimpleUploadedFile,
@@ -1157,3 +1162,42 @@ class FileLikeObjectTestCase(LiveServerTestCase):
         remote_file = urlopen(self.live_server_url + "/")
         with self.storage.open(stored_filename) as stored_file:
             self.assertEqual(stored_file.read(), remote_file.read())
+
+
+class StorageHandlerTests(SimpleTestCase):
+    @override_settings(
+        STORAGES={
+            "custom_storage": {
+                "BACKEND": "django.core.files.storage.FileSystemStorage",
+            },
+        }
+    )
+    def test_same_instance(self):
+        cache1 = storages["custom_storage"]
+        cache2 = storages["custom_storage"]
+        self.assertIs(cache1, cache2)
+
+    def test_defaults(self):
+        storages = StorageHandler()
+        self.assertEqual(storages.backends, {})
+
+    def test_nonexistent_alias(self):
+        msg = "Could not find config for 'nonexistent' in settings.STORAGES."
+        storages = StorageHandler()
+        with self.assertRaisesMessage(InvalidStorageError, msg):
+            storages["nonexistent"]
+
+    def test_nonexistent_backend(self):
+        test_storages = StorageHandler(
+            {
+                "invalid_backend": {
+                    "BACKEND": "django.nonexistent.NonexistentBackend",
+                },
+            }
+        )
+        msg = (
+            "Could not find backend 'django.nonexistent.NonexistentBackend': "
+            "No module named 'django.nonexistent'"
+        )
+        with self.assertRaisesMessage(InvalidStorageError, msg):
+            test_storages["invalid_backend"]