Fixed #23384 -- Allowed overriding part of a dictionary-type setting

This change is needed for upcoming changes where settings might be
grouped in a parent dictionary.
Thanks Tim Graham for the review.

django/conf/__init__.py

@@ -12,6 +12,7 @@ import time     # Needed for Windows
 
 from django.conf import global_settings
 from django.core.exceptions import ImproperlyConfigured
+from django.utils.datastructures import dict_merge
 from django.utils.functional import LazyObject, empty
 from django.utils import six
 
@@ -77,6 +78,10 @@ class BaseSettings(object):
         elif name == "ALLOWED_INCLUDE_ROOTS" and isinstance(value, six.string_types):
             raise ValueError("The ALLOWED_INCLUDE_ROOTS setting must be set "
                 "to a tuple, not a string.")
+        elif (hasattr(self, name) and name.isupper() and
+                isinstance(getattr(self, name), dict) and isinstance(value, dict)):
+            # This allows defining only a partial dict to update a global setting
+            value = dict_merge(getattr(self, name), value)
         object.__setattr__(self, name, value)
 
 
@@ -144,7 +149,7 @@ class UserSettingsHolder(BaseSettings):
         from the module specified in default_settings (if possible).
         """
         self.__dict__['_deleted'] = set()
-        self.default_settings = default_settings
+        self.__dict__['default_settings'] = default_settings
 
     def __getattr__(self, name):
         if name in self._deleted:

django/db/migrations/loader.py

@@ -49,7 +49,7 @@ class MigrationLoader(object):
 
     @classmethod
     def migrations_module(cls, app_label):
-        if app_label in settings.MIGRATION_MODULES:
+        if settings.MIGRATION_MODULES.get(app_label):
             return settings.MIGRATION_MODULES[app_label]
         else:
             app_package_name = apps.get_app_config(app_label).name

django/utils/datastructures.py

@@ -244,6 +244,26 @@ class SortedDict(dict):
         self.keyOrder = []
 
 
+def dict_merge(a, b):
+    """
+    Utility to recursively merge two dicts, taking care not to overwrite subkeys
+    (which would happen with dict.update), but keeping existing key including
+    those from subdictionaries (optionally opted-out if a `_clear_defaults` key
+    is present).
+    Thanks Ross McFarland (https://www.xormedia.com/recursively-merge-dictionaries-in-python/)
+    """
+    if b.get('_clear_defaults'):
+        return copy.deepcopy(b)
+
+    result = copy.deepcopy(a)
+    for key, value in six.iteritems(b):
+        if key in a and isinstance(result[key], dict):
+            result[key] = dict_merge(result[key], value)
+        else:
+            result[key] = value
+    return result
+
+
 class OrderedSet(object):
     """
     A set which keeps the ordering of the inserted items.

docs/releases/1.8.txt

@@ -530,6 +530,9 @@ Miscellaneous
   widget to allow more customization. The undocumented ``url_markup_template``
   attribute was removed in favor of ``template_with_initial``.
 
+* When a dictionary setting is overridden in user settings, both dictionaries
+  are merged by default. See :ref:`dictionary-settings`.
+
 .. _deprecated-features-1.8:
 
 Features deprecated in 1.8

docs/topics/settings.txt

@@ -110,6 +110,32 @@ between the current settings file and Django's default settings.
 
 For more, see the :djadmin:`diffsettings` documentation.
 
+.. _dictionary-settings:
+
+Overriding dictionary settings
+------------------------------
+
+.. versionchanged:: 1.8
+
+When defining a dictionary-type setting which has a non-empty value (see
+:setting:`CACHES` for example), you do not have to redefine all its keys. You
+can just define the keys differing from the default, and Django will simply
+merge your setting value with the default value. For example, if you define
+:setting:`CACHES` so::
+
+    CACHES = {
+        'special': {
+            'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+            'LOCATION': '127.0.0.1:11211',
+        }
+    }
+
+then ``CACHES['default']`` which is set by default in Django's global settings
+will still be defined, as well as the new ``'special'`` cache backend.
+
+If you want your setting to completely override the default value, you can add
+a ``_clear_defaults`` key with a ``True`` value to the dictionary.
+
 Using settings in Python code
 =============================
 

tests/cache/tests.py

@@ -522,6 +522,7 @@ class BaseCacheTests(object):
     def _perform_cull_test(self, cull_cache, initial_count, final_count):
         # Create initial cache key entries. This will overflow the cache,
         # causing a cull.
+        cull_cache.clear()
         for i in range(1, initial_count):
             cull_cache.set('cull%d' % i, 'value', 1000)
         count = 0
@@ -918,7 +919,10 @@ class DBCacheTests(BaseCacheTests, TransactionTestCase):
             stdout=stdout
         )
         self.assertEqual(stdout.getvalue(),
-            "Cache table 'test cache table' already exists.\n" * len(settings.CACHES))
+            "Cache table 'test cache table' already exists.\n" * len([
+                k for k, v in settings.CACHES.items()
+                if v['BACKEND']=='django.core.cache.backends.db.DatabaseCache'])
+        )
 
     def test_createcachetable_with_table_argument(self):
         """

tests/migrations/test_executor.py

@@ -196,7 +196,7 @@ class ExecutorTests(MigrationTestBase):
     @override_settings(
         MIGRATION_MODULES={
             "migrations": "migrations.test_migrations_custom_user",
-            "django.contrib.auth": "django.contrib.auth.migrations",
+            "auth": "django.contrib.auth.migrations",
         },
         AUTH_USER_MODEL="migrations.Author",
     )

tests/migrations/test_loader.py

@@ -81,7 +81,10 @@ class LoaderTests(TestCase):
         # Ensure we've included unmigrated apps in there too
         self.assertIn("basic", project_state.real_apps)
 
-    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_unmigdep"})
+    @override_settings(MIGRATION_MODULES={
+        "_clear_defaults": True,
+        "migrations": "migrations.test_migrations_unmigdep"
+    })
     def test_load_unmigrated_dependency(self):
         """
         Makes sure the loader can load migrations with a dependency on an unmigrated app.

tests/settings_tests/tests.py

@@ -273,6 +273,63 @@ class SettingsTests(TestCase):
         self.assertRaises(ValueError, setattr, settings,
             'ALLOWED_INCLUDE_ROOTS', '/var/www/ssi/')
 
+    def test_dict_setting(self):
+        """
+        Test that dictionary-type settings can be "complemented", that is existing
+        setting keys/values are not overriden by user settings, but merged into the
+        existing dict.
+        """
+        s = LazySettings()  # Start with fresh settings from global_settings.py
+        # Simply overwriting the key
+        s.configure(CACHES={'default': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}})
+        self.assertEqual(s.CACHES['default']['BACKEND'],
+                         'django.core.cache.backends.dummy.DummyCache')
+
+        s = LazySettings()
+        # More complex overwriting
+        s.configure(CACHES={
+            'default': {'LOCATION': 'unique-snowflake'},
+            'temp': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}
+        })
+        self.assertDictEqual(s.CACHES, {
+            'default': {
+                'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+                'LOCATION': 'unique-snowflake'
+            },
+            'temp': {
+                'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
+            }
+        })
+
+    def test_dict_setting_clear_defaults(self):
+        """
+        Test the ability to deactivate the merge feature of dictionary settings.
+        """
+        s = LazySettings()
+        s.configure(CACHES={
+            '_clear_defaults': True,
+            'temp': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}
+        })
+        self.assertDictEqual(s.CACHES, {
+            '_clear_defaults': True,
+            'temp': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}
+        })
+
+        # Also work on a subkey
+        s = LazySettings()
+        s.configure(CACHES={
+            'default': {
+                '_clear_defaults': True,
+                'LOCATION': 'unique-snowflake',
+            }
+        })
+        self.assertDictEqual(s.CACHES, {
+            'default': {
+                '_clear_defaults': True,
+                'LOCATION': 'unique-snowflake',
+            }
+        })
+
 
 class TestComplexSettingOverride(TestCase):
     def setUp(self):