diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index aff2a4d24d2..d0ec97e5251 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -280,6 +280,8 @@ SECRET_KEY_FALLBACKS = [] # Default file storage mechanism that holds media. DEFAULT_FILE_STORAGE = "django.core.files.storage.FileSystemStorage" +STORAGES = {} + # Absolute filesystem path to the directory that will hold user-uploaded files. # Example: "/var/www/example.com/media/" MEDIA_ROOT = "" diff --git a/django/core/files/storage/__init__.py b/django/core/files/storage/__init__.py index b315e57dc01..ebc887336ad 100644 --- a/django/core/files/storage/__init__.py +++ b/django/core/files/storage/__init__.py @@ -4,6 +4,7 @@ from django.utils.module_loading import import_string from .base import Storage from .filesystem import FileSystemStorage +from .handler import InvalidStorageError, StorageHandler from .memory import InMemoryStorage __all__ = ( @@ -13,6 +14,9 @@ __all__ = ( "DefaultStorage", "default_storage", "get_storage_class", + "InvalidStorageError", + "StorageHandler", + "storages", ) @@ -25,4 +29,5 @@ class DefaultStorage(LazyObject): self._wrapped = get_storage_class()() +storages = StorageHandler() default_storage = DefaultStorage() diff --git a/django/core/files/storage/handler.py b/django/core/files/storage/handler.py new file mode 100644 index 00000000000..ca379c9f5f2 --- /dev/null +++ b/django/core/files/storage/handler.py @@ -0,0 +1,46 @@ +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.utils.functional import cached_property +from django.utils.module_loading import import_string + + +class InvalidStorageError(ImproperlyConfigured): + pass + + +class StorageHandler: + def __init__(self, backends=None): + # backends is an optional dict of storage backend definitions + # (structured like settings.STORAGES). + self._backends = backends + self._storages = {} + + @cached_property + def backends(self): + if self._backends is None: + self._backends = settings.STORAGES.copy() + return self._backends + + def __getitem__(self, alias): + try: + return self._storages[alias] + except KeyError: + try: + params = self.backends[alias] + except KeyError: + raise InvalidStorageError( + f"Could not find config for '{alias}' in settings.STORAGES." + ) + storage = self.create_storage(params) + self._storages[alias] = storage + return storage + + def create_storage(self, params): + params = params.copy() + backend = params.pop("BACKEND") + options = params.pop("OPTIONS", {}) + try: + storage_cls = import_string(backend) + except ImportError as e: + raise InvalidStorageError(f"Could not find backend {backend!r}: {e}") from e + return storage_cls(**options) diff --git a/django/test/signals.py b/django/test/signals.py index 75b6339a155..4b270d99fc5 100644 --- a/django/test/signals.py +++ b/django/test/signals.py @@ -111,6 +111,23 @@ def reset_template_engines(*, setting, **kwargs): get_default_renderer.cache_clear() +@receiver(setting_changed) +def storages_changed(*, setting, **kwargs): + from django.core.files.storage import storages + + if setting in ( + "STORAGES", + "STATIC_ROOT", + "STATIC_URL", + ): + try: + del storages.backends + except AttributeError: + pass + storages._backends = None + storages._storages = {} + + @receiver(setting_changed) def clear_serializers_cache(*, setting, **kwargs): if setting == "SERIALIZATION_MODULES": diff --git a/docs/howto/custom-file-storage.txt b/docs/howto/custom-file-storage.txt index 9974c30452b..47abe3e7fd4 100644 --- a/docs/howto/custom-file-storage.txt +++ b/docs/howto/custom-file-storage.txt @@ -116,3 +116,23 @@ free unique filename cannot be found, a :exc:`SuspiciousFileOperation If a file with ``name`` already exists, ``get_alternative_name()`` is called to obtain an alternative name. + +.. _using-custom-storage-engine: + +Use your custom storage engine +============================== + +.. versionadded:: 4.2 + +The first step to using your custom storage with Django is to tell Django about +the file storage backend you'll be using. This is done using the +:setting:`STORAGES` setting. This setting maps storage aliases, which are a way +to refer to a specific storage throughout Django, to a dictionary of settings +for that specific storage backend. The settings in the inner dictionaries are +described fully in the :setting:`STORAGES` documentation. + +Storages are then accessed by alias from from the +:data:`django.core.files.storage.storages` dictionary:: + + from django.core.files.storage import storages + example_storage = storages["example"] diff --git a/docs/ref/contrib/staticfiles.txt b/docs/ref/contrib/staticfiles.txt index 7ca3584c33c..08fc23bdb12 100644 --- a/docs/ref/contrib/staticfiles.txt +++ b/docs/ref/contrib/staticfiles.txt @@ -23,6 +23,7 @@ Settings See :ref:`staticfiles settings ` for details on the following settings: +* :setting:`STORAGES` * :setting:`STATIC_ROOT` * :setting:`STATIC_URL` * :setting:`STATICFILES_DIRS` diff --git a/docs/ref/files/storage.txt b/docs/ref/files/storage.txt index fa79a4f91a5..d5daccf8344 100644 --- a/docs/ref/files/storage.txt +++ b/docs/ref/files/storage.txt @@ -9,6 +9,12 @@ Getting the default storage class Django provides convenient ways to access the default storage class: +.. data:: storages + + .. versionadded:: 4.2 + + Storage instances as defined by :setting:`STORAGES`. + .. class:: DefaultStorage :class:`~django.core.files.storage.DefaultStorage` provides diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index b1a8e2444d2..d1b638f4b6c 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -2606,6 +2606,43 @@ Silenced checks will not be output to the console. See also the :doc:`/ref/checks` documentation. +.. setting:: STORAGES + +``STORAGES`` +------------ + +.. versionadded:: 4.2 + +Default:: + + {} + +A dictionary containing the settings for all storages to be used with Django. +It is a nested dictionary whose contents map a storage alias to a dictionary +containing the options for an individual storage. + +Storages can have any alias you choose. + +The following is an example ``settings.py`` snippet defining a custom file +storage called ``example``:: + + STORAGES = { + # ... + "example": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + "OPTIONS": { + "location": "/example", + "base_url": "/example/", + }, + }, + } + +``OPTIONS`` are passed to the ``BACKEND`` on initialization in ``**kwargs``. + +A ready-to-use instance of the storage backends can be retrieved from +:data:`django.core.files.storage.storages`. Use a key corresponding to the +backend definition in :setting:`STORAGES`. + .. setting:: TEMPLATES ``TEMPLATES`` @@ -3663,6 +3700,7 @@ File uploads * :setting:`FILE_UPLOAD_TEMP_DIR` * :setting:`MEDIA_ROOT` * :setting:`MEDIA_URL` +* :setting:`STORAGES` Forms ----- diff --git a/docs/releases/4.2.txt b/docs/releases/4.2.txt index aa2deed932e..ecc9a06a870 100644 --- a/docs/releases/4.2.txt +++ b/docs/releases/4.2.txt @@ -91,6 +91,12 @@ In-memory file storage The new ``django.core.files.storage.InMemoryStorage`` class provides a non-persistent storage useful for speeding up tests by avoiding disk access. +Custom file storages +-------------------- + +The new :setting:`STORAGES` setting allows configuring multiple custom file +storage backends. + Minor features -------------- diff --git a/docs/topics/files.txt b/docs/topics/files.txt index 6f7f9c21e2e..eb4e655cfa9 100644 --- a/docs/topics/files.txt +++ b/docs/topics/files.txt @@ -239,3 +239,15 @@ For example:: class MyModel(models.Model): my_file = models.FileField(storage=select_storage) + +In order to set a storage defined in the :setting:`STORAGES` setting you can +use a lambda function:: + + from django.core.files.storage import storages + + class MyModel(models.Model): + upload = models.FileField(storage=lambda: storages["custom_storage"]) + +.. versionchanged:: 4.2 + + Support for ``storages`` was added. diff --git a/docs/topics/testing/tools.txt b/docs/topics/testing/tools.txt index 139d66e64b1..524fc855844 100644 --- a/docs/topics/testing/tools.txt +++ b/docs/topics/testing/tools.txt @@ -1441,15 +1441,16 @@ when settings are changed. Django itself uses this signal to reset various data: -================================ ======================== -Overridden settings Data reset -================================ ======================== -USE_TZ, TIME_ZONE Databases timezone -TEMPLATES Template engines -SERIALIZATION_MODULES Serializers cache -LOCALE_PATHS, LANGUAGE_CODE Default translation and loaded translations -MEDIA_ROOT, DEFAULT_FILE_STORAGE Default file storage -================================ ======================== +================================= ======================== +Overridden settings Data reset +================================= ======================== +USE_TZ, TIME_ZONE Databases timezone +TEMPLATES Template engines +SERIALIZATION_MODULES Serializers cache +LOCALE_PATHS, LANGUAGE_CODE Default translation and loaded translations +MEDIA_ROOT, DEFAULT_FILE_STORAGE Default file storage +STATIC_ROOT, STATIC_URL, STORAGES Storages configuration +================================= ======================== Isolating apps -------------- diff --git a/tests/file_storage/tests.py b/tests/file_storage/tests.py index 5c7190d6985..d4e59695196 100644 --- a/tests/file_storage/tests.py +++ b/tests/file_storage/tests.py @@ -14,9 +14,14 @@ from urllib.request import urlopen from django.core.cache import cache from django.core.exceptions import SuspiciousFileOperation from django.core.files.base import ContentFile, File -from django.core.files.storage import FileSystemStorage +from django.core.files.storage import FileSystemStorage, InvalidStorageError from django.core.files.storage import Storage as BaseStorage -from django.core.files.storage import default_storage, get_storage_class +from django.core.files.storage import ( + StorageHandler, + default_storage, + get_storage_class, + storages, +) from django.core.files.uploadedfile import ( InMemoryUploadedFile, SimpleUploadedFile, @@ -1157,3 +1162,42 @@ class FileLikeObjectTestCase(LiveServerTestCase): remote_file = urlopen(self.live_server_url + "/") with self.storage.open(stored_filename) as stored_file: self.assertEqual(stored_file.read(), remote_file.read()) + + +class StorageHandlerTests(SimpleTestCase): + @override_settings( + STORAGES={ + "custom_storage": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + }, + } + ) + def test_same_instance(self): + cache1 = storages["custom_storage"] + cache2 = storages["custom_storage"] + self.assertIs(cache1, cache2) + + def test_defaults(self): + storages = StorageHandler() + self.assertEqual(storages.backends, {}) + + def test_nonexistent_alias(self): + msg = "Could not find config for 'nonexistent' in settings.STORAGES." + storages = StorageHandler() + with self.assertRaisesMessage(InvalidStorageError, msg): + storages["nonexistent"] + + def test_nonexistent_backend(self): + test_storages = StorageHandler( + { + "invalid_backend": { + "BACKEND": "django.nonexistent.NonexistentBackend", + }, + } + ) + msg = ( + "Could not find backend 'django.nonexistent.NonexistentBackend': " + "No module named 'django.nonexistent'" + ) + with self.assertRaisesMessage(InvalidStorageError, msg): + test_storages["invalid_backend"]