Fixed #21012 -- New API to access cache backends.

Thanks Curtis Malony and Florian Apolloner.

Squashed commit of the following:

commit 3380495e93
Author: Aymeric Augustin <aymeric.augustin@m4x.org>
Date:   Sat Nov 23 14:18:07 2013 +0100

    Looked up the template_fragments cache at runtime.

commit 905a74f52b
Author: Aymeric Augustin <aymeric.augustin@m4x.org>
Date:   Sat Nov 23 14:19:48 2013 +0100

    Removed all uses of create_cache.

    Refactored the cache tests significantly.

    Made it safe to override the CACHES setting.

commit 35e289fe92
Author: Aymeric Augustin <aymeric.augustin@m4x.org>
Date:   Sat Nov 23 12:23:57 2013 +0100

    Removed create_cache function.

commit 8e274f747a
Author: Aymeric Augustin <aymeric.augustin@m4x.org>
Date:   Sat Nov 23 12:04:52 2013 +0100

    Updated docs to describe a simplified cache backend API.

commit ee7eb0f73e
Author: Curtis Maloney <curtis@tinbrain.net>
Date:   Sat Oct 19 09:49:24 2013 +1100

    Fixed #21012 -- Thread-local caches, like databases.

django/contrib/sessions/backends/cache.py

@@ -1,6 +1,6 @@
 from django.conf import settings
 from django.contrib.sessions.backends.base import SessionBase, CreateError
-from django.core.cache import get_cache
+from django.core.cache import caches
 from django.utils.six.moves import xrange
 
 KEY_PREFIX = "django.contrib.sessions.cache"
@@ -11,7 +11,7 @@ class SessionStore(SessionBase):
     A cache-based session store.
     """
     def __init__(self, session_key=None):
-        self._cache = get_cache(settings.SESSION_CACHE_ALIAS)
+        self._cache = caches[settings.SESSION_CACHE_ALIAS]
         super(SessionStore, self).__init__(session_key)
 
     @property

django/contrib/sessions/backends/cached_db.py

@@ -6,7 +6,7 @@ import logging
 
 from django.conf import settings
 from django.contrib.sessions.backends.db import SessionStore as DBStore
-from django.core.cache import get_cache
+from django.core.cache import caches
 from django.core.exceptions import SuspiciousOperation
 from django.utils import timezone
 from django.utils.encoding import force_text
@@ -20,7 +20,7 @@ class SessionStore(DBStore):
     """
 
     def __init__(self, session_key=None):
-        self._cache = get_cache(settings.SESSION_CACHE_ALIAS)
+        self._cache = caches[settings.SESSION_CACHE_ALIAS]
         super(SessionStore, self).__init__(session_key)
 
     @property

django/contrib/sessions/tests.py

@@ -15,7 +15,7 @@ from django.contrib.sessions.backends.file import SessionStore as FileSession
 from django.contrib.sessions.backends.signed_cookies import SessionStore as CookieSession
 from django.contrib.sessions.models import Session
 from django.contrib.sessions.middleware import SessionMiddleware
-from django.core.cache import get_cache
+from django.core.cache import caches
 from django.core.cache.backends.base import InvalidCacheBackendError
 from django.core import management
 from django.core.exceptions import ImproperlyConfigured
@@ -140,7 +140,7 @@ class SessionTestsMixin(object):
         self.assertTrue(self.session.modified)
 
     def test_save(self):
-        if (hasattr(self.session, '_cache') and'DummyCache' in
+        if (hasattr(self.session, '_cache') and 'DummyCache' in
             settings.CACHES[settings.SESSION_CACHE_ALIAS]['BACKEND']):
             raise unittest.SkipTest("Session saving tests require a real cache backend")
         self.session.save()
@@ -481,7 +481,7 @@ class CacheSessionTests(SessionTestsMixin, unittest.TestCase):
 
     def test_default_cache(self):
         self.session.save()
-        self.assertNotEqual(get_cache('default').get(self.session.cache_key), None)
+        self.assertNotEqual(caches['default'].get(self.session.cache_key), None)
 
     @override_settings(CACHES={
         'default': {
@@ -489,6 +489,7 @@ class CacheSessionTests(SessionTestsMixin, unittest.TestCase):
         },
         'sessions': {
             'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+            'LOCATION': 'session',
         },
     }, SESSION_CACHE_ALIAS='sessions')
     def test_non_default_cache(self):
@@ -496,8 +497,8 @@ class CacheSessionTests(SessionTestsMixin, unittest.TestCase):
         self.session = self.backend()
 
         self.session.save()
-        self.assertEqual(get_cache('default').get(self.session.cache_key), None)
+        self.assertEqual(caches['default'].get(self.session.cache_key), None)
-        self.assertNotEqual(get_cache('sessions').get(self.session.cache_key), None)
+        self.assertNotEqual(caches['sessions'].get(self.session.cache_key), None)
 
 
 class SessionMiddlewareTests(unittest.TestCase):

django/contrib/staticfiles/storage.py

@@ -7,7 +7,7 @@ import posixpath
 import re
 
 from django.conf import settings
-from django.core.cache import (get_cache, InvalidCacheBackendError,
+from django.core.cache import (caches, InvalidCacheBackendError,
                                cache as default_cache)
 from django.core.exceptions import ImproperlyConfigured
 from django.core.files.base import ContentFile
@@ -56,7 +56,7 @@ class CachedFilesMixin(object):
     def __init__(self, *args, **kwargs):
         super(CachedFilesMixin, self).__init__(*args, **kwargs)
         try:
-            self.cache = get_cache('staticfiles')
+            self.cache = caches['staticfiles']
         except InvalidCacheBackendError:
             # Use the default backend
             self.cache = default_cache

django/core/cache/__init__.py

@@ -6,14 +6,15 @@ In a nutshell, a cache is a set of values -- which can be any object that
 may be pickled -- identified by string keys.  For the complete API, see
 the abstract BaseCache class in django.core.cache.backends.base.
 
-Client code should not access a cache backend directly; instead it should
+Client code should use the `cache` variable defined here to access the default
-either use the "cache" variable made available here, or it should use the
+cache backend and look up non-default cache backends in the `caches` dict-like
-get_cache() function made available here. get_cache() takes a CACHES alias or a
+object.
-backend path and config parameters, and returns an instance of a backend cache
-class.
 
 See docs/topics/cache.txt for information on the public API.
 """
+from threading import local
+import warnings
+
 from django.conf import settings
 from django.core import signals
 from django.core.cache.backends.base import (
@@ -35,14 +36,14 @@ if DEFAULT_CACHE_ALIAS not in settings.CACHES:
 
 def get_cache(backend, **kwargs):
     """
-    Function to load a cache backend dynamically. This is flexible by design
+    Function to create a cache backend dynamically. This is flexible by design
     to allow different use cases:
 
     To load a backend that is pre-defined in the settings::
 
         cache = get_cache('default')
 
-    To load a backend with its dotted import path,
+    To create a backend with its dotted import path,
     including arbitrary options::
 
         cache = get_cache('django.core.cache.backends.memcached.MemcachedCache', **{
@@ -50,6 +51,12 @@ def get_cache(backend, **kwargs):
         })
 
     """
+    warnings.warn("'get_cache' is deprecated in favor of 'caches'.",
+                  PendingDeprecationWarning, stacklevel=2)
+    return _create_cache(backend, **kwargs)
+
+
+def _create_cache(backend, **kwargs):
     try:
         # Try to get the CACHES entry for the given backend name first
         try:
@@ -79,4 +86,57 @@ def get_cache(backend, **kwargs):
     signals.request_finished.connect(cache.close)
     return cache
 
-cache = get_cache(DEFAULT_CACHE_ALIAS)
+
+class CacheHandler(object):
+    """
+    A Cache Handler to manage access to Cache instances.
+
+    Ensures only one instance of each alias exists per thread.
+    """
+    def __init__(self):
+        self._caches = local()
+
+    def __getitem__(self, alias):
+        try:
+            return getattr(self._caches, alias)
+        except AttributeError:
+            pass
+
+        if alias not in settings.CACHES:
+            raise InvalidCacheBackendError(
+                "Could not find config for '%s' in settings.CACHES" % alias
+            )
+
+        cache = _create_cache(alias)
+        setattr(self._caches, alias, cache)
+
+        return cache
+
+caches = CacheHandler()
+
+class DefaultCacheProxy(object):
+    """
+    Proxy access to the default Cache object's attributes.
+
+    This allows the legacy `cache` object to be thread-safe using the new
+    ``caches`` API.
+    """
+    def __getattr__(self, name):
+        return getattr(caches[DEFAULT_CACHE_ALIAS], name)
+
+    def __setattr__(self, name, value):
+        return setattr(caches[DEFAULT_CACHE_ALIAS], name, value)
+
+    def __delattr__(self, name):
+        return delattr(caches[DEFAULT_CACHE_ALIAS], name)
+
+    def __contains__(self, key):
+        return key in caches[DEFAULT_CACHE_ALIAS]
+
+    def __eq__(self, other):
+        return caches[DEFAULT_CACHE_ALIAS] == other
+
+    def __ne__(self, other):
+        return caches[DEFAULT_CACHE_ALIAS] != other
+
+cache = DefaultCacheProxy()

django/core/cache/backends/memcached.py

@@ -2,13 +2,13 @@
 
 import time
 import pickle
-from threading import local
 
 from django.core.cache.backends.base import BaseCache, DEFAULT_TIMEOUT
 
 from django.utils import six
 from django.utils.deprecation import RenameMethodsBase
 from django.utils.encoding import force_str
+from django.utils.functional import cached_property
 
 
 class BaseMemcachedCacheMethods(RenameMethodsBase):
@@ -177,24 +177,14 @@ class PyLibMCCache(BaseMemcachedCache):
     "An implementation of a cache binding using pylibmc"
     def __init__(self, server, params):
         import pylibmc
-        self._local = local()
         super(PyLibMCCache, self).__init__(server, params,
                                            library=pylibmc,
                                            value_not_found_exception=pylibmc.NotFound)
 
-    @property
+    @cached_property
     def _cache(self):
-        # PylibMC uses cache options as the 'behaviors' attribute.
-        # It also needs to use threadlocals, because some versions of
-        # PylibMC don't play well with the GIL.
-        client = getattr(self._local, 'client', None)
-        if client:
-            return client
-
         client = self._lib.Client(self._servers)
         if self._options:
             client.behaviors = self._options
 
-        self._local.client = client
-
         return client

django/core/management/commands/createcachetable.py

@@ -1,7 +1,7 @@
 from optparse import make_option
 
 from django.conf import settings
-from django.core.cache import get_cache
+from django.core.cache import caches
 from django.core.cache.backends.db import BaseDatabaseCache
 from django.core.management.base import BaseCommand, CommandError
 from django.db import connections, router, transaction, models, DEFAULT_DB_ALIAS
@@ -30,7 +30,7 @@ class Command(BaseCommand):
                 self.create_table(db, tablename)
         else:
             for cache_alias in settings.CACHES:
-                cache = get_cache(cache_alias)
+                cache = caches[cache_alias]
                 if isinstance(cache, BaseDatabaseCache):
                     self.create_table(db, cache._table)
 

django/middleware/cache.py

@@ -46,7 +46,7 @@ More details about how the caching works:
 import warnings
 
 from django.conf import settings
-from django.core.cache import get_cache, DEFAULT_CACHE_ALIAS
+from django.core.cache import caches, DEFAULT_CACHE_ALIAS
 from django.utils.cache import get_cache_key, learn_cache_key, patch_response_headers, get_max_age
 
 
@@ -64,7 +64,7 @@ class UpdateCacheMiddleware(object):
         self.key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
         self.cache_anonymous_only = getattr(settings, 'CACHE_MIDDLEWARE_ANONYMOUS_ONLY', False)
         self.cache_alias = settings.CACHE_MIDDLEWARE_ALIAS
-        self.cache = get_cache(self.cache_alias)
+        self.cache = caches[self.cache_alias]
 
     def _session_accessed(self, request):
         try:
@@ -122,10 +122,9 @@ class FetchFromCacheMiddleware(object):
     MIDDLEWARE_CLASSES so that it'll get called last during the request phase.
     """
     def __init__(self):
-        self.cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS
         self.key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
         self.cache_alias = settings.CACHE_MIDDLEWARE_ALIAS
-        self.cache = get_cache(self.cache_alias)
+        self.cache = caches[self.cache_alias]
 
     def process_request(self, request):
         """
@@ -169,39 +168,32 @@ class CacheMiddleware(UpdateCacheMiddleware, FetchFromCacheMiddleware):
         # we fall back to system defaults. If it is not provided at all,
         # we need to use middleware defaults.
 
-        cache_kwargs = {}
+        try:
+            key_prefix = kwargs['key_prefix']
+            if key_prefix is None:
+                key_prefix = ''
+        except KeyError:
+            key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
+        self.key_prefix = key_prefix
 
         try:
-            self.key_prefix = kwargs['key_prefix']
+            cache_alias = kwargs['cache_alias']
-            if self.key_prefix is not None:
+            if cache_alias is None:
-                cache_kwargs['KEY_PREFIX'] = self.key_prefix
+                cache_alias = DEFAULT_CACHE_ALIAS
-            else:
-                self.key_prefix = ''
         except KeyError:
-            self.key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
+            cache_alias = settings.CACHE_MIDDLEWARE_ALIAS
-            cache_kwargs['KEY_PREFIX'] = self.key_prefix
+        self.cache_alias = cache_alias
 
-        try:
-            self.cache_alias = kwargs['cache_alias']
-            if self.cache_alias is None:
-                self.cache_alias = DEFAULT_CACHE_ALIAS
-            if cache_timeout is not None:
-                cache_kwargs['TIMEOUT'] = cache_timeout
-        except KeyError:
-            self.cache_alias = settings.CACHE_MIDDLEWARE_ALIAS
         if cache_timeout is None:
-                cache_kwargs['TIMEOUT'] = settings.CACHE_MIDDLEWARE_SECONDS
+            cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS
-            else:
+        self.cache_timeout = cache_timeout
-                cache_kwargs['TIMEOUT'] = cache_timeout
 
         if cache_anonymous_only is None:
-            self.cache_anonymous_only = getattr(settings, 'CACHE_MIDDLEWARE_ANONYMOUS_ONLY', False)
+            cache_anonymous_only = getattr(settings, 'CACHE_MIDDLEWARE_ANONYMOUS_ONLY', False)
-        else:
         self.cache_anonymous_only = cache_anonymous_only
 
         if self.cache_anonymous_only:
             msg = "CACHE_MIDDLEWARE_ANONYMOUS_ONLY has been deprecated and will be removed in Django 1.8."
             warnings.warn(msg, DeprecationWarning, stacklevel=1)
 
-        self.cache = get_cache(self.cache_alias, **cache_kwargs)
+        self.cache = caches[self.cache_alias]
-        self.cache_timeout = self.cache.default_timeout

django/templatetags/cache.py

@@ -1,16 +1,11 @@
 from __future__ import unicode_literals
 
-from django.core.cache import get_cache, InvalidCacheBackendError
+from django.core.cache import cache, caches, InvalidCacheBackendError
 from django.core.cache.utils import make_template_fragment_key
 from django.template import Library, Node, TemplateSyntaxError, VariableDoesNotExist
 
 register = Library()
 
-try:
-    default_cache = get_cache('template_fragments')
-except InvalidCacheBackendError:
-    from django.core.cache import cache as default_cache
-
 
 class CacheNode(Node):
     def __init__(self, nodelist, expire_time_var, fragment_name, vary_on, cache_name):
@@ -35,17 +30,21 @@ class CacheNode(Node):
             except VariableDoesNotExist:
                 raise TemplateSyntaxError('"cache" tag got an unknown variable: %r' % self.cache_name.var)
             try:
-                cache = get_cache(cache_name)
+                fragment_cache = caches[cache_name]
             except InvalidCacheBackendError:
                 raise TemplateSyntaxError('Invalid cache name specified for cache tag: %r' % cache_name)
         else:
-            cache = default_cache
+            try:
+                fragment_cache = caches['template_fragments']
+            except InvalidCacheBackendError:
+                fragment_cache = caches['default']
+
         vary_on = [var.resolve(context) for var in self.vary_on]
         cache_key = make_template_fragment_key(self.fragment_name, vary_on)
-        value = cache.get(cache_key)
+        value = fragment_cache.get(cache_key)
         if value is None:
             value = self.nodelist.render(context)
-            cache.set(cache_key, value, expire_time)
+            fragment_cache.set(cache_key, value, expire_time)
         return value
 
 

django/test/signals.py

@@ -1,5 +1,6 @@
 import os
 import time
+import threading
 import warnings
 
 from django.conf import settings
@@ -19,6 +20,13 @@ setting_changed = Signal(providing_args=["setting", "value", "enter"])
 COMPLEX_OVERRIDE_SETTINGS = set(['DATABASES'])
 
 
+@receiver(setting_changed)
+def clear_cache_handlers(**kwargs):
+    if kwargs['setting'] == 'CACHES':
+        from django.core.cache import caches
+        caches._caches = threading.local()
+
+
 @receiver(setting_changed)
 def update_connections_time_zone(**kwargs):
     if kwargs['setting'] == 'TIME_ZONE':

django/utils/cache.py

@@ -23,7 +23,7 @@ import re
 import time
 
 from django.conf import settings
-from django.core.cache import get_cache
+from django.core.cache import caches
 from django.utils.encoding import iri_to_uri, force_bytes, force_text
 from django.utils.http import http_date
 from django.utils.timezone import get_current_timezone_name
@@ -219,7 +219,7 @@ def get_cache_key(request, key_prefix=None, method='GET', cache=None):
         key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
     cache_key = _generate_cache_header_key(key_prefix, request)
     if cache is None:
-        cache = get_cache(settings.CACHE_MIDDLEWARE_ALIAS)
+        cache = caches[settings.CACHE_MIDDLEWARE_ALIAS]
     headerlist = cache.get(cache_key, None)
     if headerlist is not None:
         return _generate_cache_key(request, method, headerlist, key_prefix)
@@ -246,7 +246,7 @@ def learn_cache_key(request, response, cache_timeout=None, key_prefix=None, cach
         cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS
     cache_key = _generate_cache_header_key(key_prefix, request)
     if cache is None:
-        cache = get_cache(settings.CACHE_MIDDLEWARE_ALIAS)
+        cache = caches[settings.CACHE_MIDDLEWARE_ALIAS]
     if response.has_header('Vary'):
         is_accept_language_redundant = settings.USE_I18N or settings.USE_L10N
         # If i18n or l10n are used, the generated cache key will be suffixed

docs/internals/deprecation.txt

@@ -114,7 +114,7 @@ these changes.
   no longer appears to be actively maintained & does not work on Python 3.
   You are advised to install `Pillow`_, which should be used instead.
 
-.. _`Pillow`: https://pypi.python.org/pypi/Pillow
+  .. _`Pillow`: https://pypi.python.org/pypi/Pillow
 
 * The following private APIs will be removed:
 
@@ -215,6 +215,9 @@ these changes.
 
 * The internal ``django.utils.functional.memoize`` will be removed.
 
+* ``django.core.cache.get_cache`` will be removed. Add suitable entries
+  to :setting:`CACHES` and use :data:`django.core.cache.caches` instead.
+
 2.0
 ---
 

docs/releases/1.7.txt

@@ -269,6 +269,18 @@ Minor features
   allowing the ``published`` element to be included in the feed (which
   relies on ``pubdate``).
 
+Cache
+^^^^^
+
+* Access to caches configured in :setting:`CACHES` is now available via
+  :data:`django.core.cache.caches`. This dict-like object provides a different
+  instance per thread. It supersedes :func:`django.core.cache.get_cache` which
+  is now deprecated.
+
+* If you instanciate cache backends directly, be aware that they aren't
+  thread-safe any more, as :data:`django.core.cache.caches` now yields
+  differend instances per thread.
+
 Email
 ^^^^^
 
@@ -643,6 +655,12 @@ Miscellaneous
 Features deprecated in 1.7
 ==========================
 
+``django.core.cache.get_cache``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:func:`django.core.cache.get_cache` has been supplanted by
+:data:`django.core.cache.caches`.
+
 ``django.utils.dictconfig``/``django.utils.importlib``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/topics/cache.txt

@@ -703,22 +703,50 @@ pickling.)
 Accessing the cache
 -------------------
 
-.. function:: django.core.cache.get_cache(backend, **kwargs)
+.. data:: django.core.cache.caches
 
-The cache module, ``django.core.cache``, has a ``cache`` object that's
+    .. versionadded:: 1.7
-automatically created from the ``'default'`` entry in the :setting:`CACHES`
+
-setting::
+    You can access the caches configured in the :setting:`CACHES` setting
+    through a dict-like object: ``django.core.cache.caches``. Repeated
+    requests for the same alias in the same thread will return the same
+    object.
+
+        >>> from django.core.cache import caches
+        >>> cache1 = caches['myalias']
+        >>> cache2 = caches['myalias']
+        >>> cache1 is cache2
+        True
+
+    If the named key does not exist, ``InvalidCacheBackendError`` will be
+    raised.
+
+    To provide thread-safety, a different instance of the cache backend will
+    be returned for each thread.
+
+.. data:: django.core.cache.cache
+
+    As a shortcut, the default cache is available as
+    ``django.core.cache.cache``::
 
         >>> from django.core.cache import cache
 
-If you have multiple caches defined in :setting:`CACHES`, then you can use
+    This object is equivalent to ``caches['default']``.
-:func:`django.core.cache.get_cache` to retrieve a cache object for any key::
+
+.. function:: django.core.cache.get_cache(backend, **kwargs)
+
+    .. deprecated:: 1.7
+        This function has been deprecated in favour of
+        :data:`~django.core.cache.caches`.
+
+    Before Django 1.7 this function was the canonical way to obtain a cache
+    instance. It could also be used to create a new cache instance with a
+    different configuration.
 
         >>> from django.core.cache import get_cache
-    >>> cache = get_cache('alternate')
+        >>> get_cache('default')
-
+        >>> get_cache('django.core.cache.backends.memcached.MemcachedCache', LOCATION='127.0.0.2')
-If the named key does not exist, ``InvalidCacheBackendError`` will be raised.
+        >>> get_cache('default', TIMEOUT=300)
-
 
 Basic usage
 -----------

tests/template_tests/tests.py

@@ -514,7 +514,7 @@ class TemplateRegressionTests(TestCase):
         o2 = t2.render(ctx)
 
         self.assertEqual(o1, 'foo')
-        self.assertNotEqual(o1, o2)
+        self.assertEqual(o2, 'bar')
 
     def test_cache_missing_backend(self):
         """