From b4d46df5cad6c936d83dd4f8038d0dc1121bc21e Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Fri, 18 Jan 2019 23:26:50 +0000 Subject: [PATCH] Fixed #29887 -- Added a cache backend for pymemcache. --- django/core/cache/backends/memcached.py | 14 +++++++++++ docs/ref/settings.txt | 5 ++++ docs/releases/3.2.txt | 10 ++++++++ docs/topics/cache.txt | 32 +++++++++++++++++++++---- tests/cache/tests.py | 31 ++++++++++++++++++++++++ tests/requirements/py3.txt | 1 + 6 files changed, 89 insertions(+), 4 deletions(-) diff --git a/django/core/cache/backends/memcached.py b/django/core/cache/backends/memcached.py index 6a7ced6073d..cc5648bb1ca 100644 --- a/django/core/cache/backends/memcached.py +++ b/django/core/cache/backends/memcached.py @@ -214,3 +214,17 @@ class PyLibMCCache(BaseMemcachedCache): # libmemcached manages its own connections. Don't call disconnect_all() # as it resets the failover state and creates unnecessary reconnects. pass + + +class PyMemcacheCache(BaseMemcachedCache): + """An implementation of a cache binding using pymemcache.""" + def __init__(self, server, params): + import pymemcache.serde + super().__init__(server, params, library=pymemcache, value_not_found_exception=KeyError) + self._class = self._lib.HashClient + self._options = { + 'allow_unicode_keys': True, + 'default_noreply': False, + 'serde': pymemcache.serde.pickle_serde, + **self._options, + } diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 9a2f1457ac9..506267a0288 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -158,11 +158,16 @@ The cache backend to use. The built-in cache backends are: * ``'django.core.cache.backends.locmem.LocMemCache'`` * ``'django.core.cache.backends.memcached.MemcachedCache'`` * ``'django.core.cache.backends.memcached.PyLibMCCache'`` +* ``'django.core.cache.backends.memcached.PyMemcacheCache'`` You can use a cache backend that doesn't ship with Django by setting :setting:`BACKEND ` to a fully-qualified path of a cache backend class (i.e. ``mypackage.backends.whatever.WhateverCache``). +.. versionchanged:: 3.2 + + The ``PyMemcacheCache`` backend was added. + .. setting:: CACHES-KEY_FUNCTION ``KEY_FUNCTION`` diff --git a/docs/releases/3.2.txt b/docs/releases/3.2.txt index e77785f8e2a..e66e5777d58 100644 --- a/docs/releases/3.2.txt +++ b/docs/releases/3.2.txt @@ -53,6 +53,16 @@ needed. As a consequence, it's deprecated. See :ref:`configuring-applications-ref` for full details. +``pymemcache`` support +---------------------- + +The new ``django.core.cache.backends.memcached.PyMemcacheCache`` cache backend +allows using the pymemcache_ library for memcached. ``pymemcache`` 3.4.0 or +higher is required. For more details, see the :doc:`documentation on caching in +Django `. + +.. _pymemcache: https://pypi.org/project/pymemcache/ + Minor features -------------- diff --git a/docs/topics/cache.txt b/docs/topics/cache.txt index 4e1b2546adc..dcf47e65cab 100644 --- a/docs/topics/cache.txt +++ b/docs/topics/cache.txt @@ -77,17 +77,19 @@ database or filesystem usage. After installing Memcached itself, you'll need to install a Memcached binding. There are several Python Memcached bindings available; the -two most common are `python-memcached`_ and `pylibmc`_. +three most common are `python-memcached`_, `pylibmc`_, and `pymemcache`_. .. _`python-memcached`: https://pypi.org/project/python-memcached/ .. _`pylibmc`: https://pypi.org/project/pylibmc/ +.. _`pymemcache`: https://pypi.org/project/pymemcache/ To use Memcached with Django: * Set :setting:`BACKEND ` to - ``django.core.cache.backends.memcached.MemcachedCache`` or - ``django.core.cache.backends.memcached.PyLibMCCache`` (depending - on your chosen memcached binding) + ``django.core.cache.backends.memcached.MemcachedCache``, + ``django.core.cache.backends.memcached.PyLibMCCache``, or + ``django.core.cache.backends.memcached.PyMemcacheCache`` (depending on your + chosen memcached binding) * Set :setting:`LOCATION ` to ``ip:port`` values, where ``ip`` is the IP address of the Memcached daemon and ``port`` is the @@ -159,6 +161,10 @@ permanent storage -- they're all intended to be solutions for caching, not storage -- but we point this out here because memory-based caching is particularly temporary. +.. versionchanged:: 3.2 + + The ``PyMemcacheCache`` backend was added. + .. _database-caching: Database caching @@ -466,6 +472,24 @@ the binary protocol, SASL authentication, and the ``ketama`` behavior mode:: } } +Here's an example configuration for a ``pymemcache`` based backend that enables +client pooling (which may improve performance by keeping clients connected), +treats memcache/network errors as cache misses, and sets the ``TCP_NODELAY`` +flag on the connection's socket:: + + CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', + 'LOCATION': '127.0.0.1:11211', + 'OPTIONS': { + 'no_delay': True, + 'ignore_exc': True, + 'max_pool_size': 4, + 'use_pooling': True, + } + } + } + .. _the-per-site-cache: The per-site cache diff --git a/tests/cache/tests.py b/tests/cache/tests.py index 9a800e2b6dd..1ef2cc1bc1a 100644 --- a/tests/cache/tests.py +++ b/tests/cache/tests.py @@ -1277,6 +1277,7 @@ for _cache_params in settings.CACHES.values(): MemcachedCache_params = configured_caches.get('django.core.cache.backends.memcached.MemcachedCache') PyLibMCCache_params = configured_caches.get('django.core.cache.backends.memcached.PyLibMCCache') +PyMemcacheCache_params = configured_caches.get('django.core.cache.backends.memcached.PyMemcacheCache') # The memcached backends don't support cull-related options like `MAX_ENTRIES`. memcached_excluded_caches = {'cull', 'zero_cull'} @@ -1459,6 +1460,36 @@ class PyLibMCCacheTests(BaseMemcachedTests, TestCase): self.assertEqual(cache.client_servers, [expected]) +@unittest.skipUnless(PyMemcacheCache_params, 'PyMemcacheCache backend not configured') +@override_settings(CACHES=caches_setting_for_tests( + base=PyMemcacheCache_params, + exclude=memcached_excluded_caches, +)) +class PyMemcacheCacheTests(BaseMemcachedTests, TestCase): + base_params = PyMemcacheCache_params + + def test_pymemcache_highest_pickle_version(self): + self.assertEqual( + cache._cache.default_kwargs['serde']._serialize_func.keywords['pickle_version'], + pickle.HIGHEST_PROTOCOL, + ) + for cache_key in settings.CACHES: + for client_key, client in caches[cache_key]._cache.clients.items(): + with self.subTest(cache_key=cache_key, server=client_key): + self.assertEqual( + client.serde._serialize_func.keywords['pickle_version'], + pickle.HIGHEST_PROTOCOL, + ) + + @override_settings(CACHES=caches_setting_for_tests( + base=PyMemcacheCache_params, + exclude=memcached_excluded_caches, + OPTIONS={'no_delay': True}, + )) + def test_pymemcache_options(self): + self.assertIs(cache._cache.default_kwargs['no_delay'], True) + + @override_settings(CACHES=caches_setting_for_tests( BACKEND='django.core.cache.backends.filebased.FileBasedCache', )) diff --git a/tests/requirements/py3.txt b/tests/requirements/py3.txt index 694b803b36c..d5c59b4e30a 100644 --- a/tests/requirements/py3.txt +++ b/tests/requirements/py3.txt @@ -8,6 +8,7 @@ numpy Pillow >= 6.2.0 # pylibmc/libmemcached can't be built on Windows. pylibmc; sys.platform != 'win32' +pymemcache >= 3.4.0 python-memcached >= 1.59 pytz pywatchman; sys.platform != 'win32'