Added ManifestStaticFilesStorage to staticfiles contrib app.

It uses a static manifest file that is created when running
collectstatic in the JSON format.

django/contrib/staticfiles/storage.py

@@ -5,6 +5,7 @@ from importlib import import_module
 import os
 import posixpath
 import re
+import json
 
 from django.conf import settings
 from django.core.cache import (caches, InvalidCacheBackendError,
@@ -49,7 +50,7 @@ class StaticFilesStorage(FileSystemStorage):
         return super(StaticFilesStorage, self).path(name)
 
 
-class CachedFilesMixin(object):
+class HashedFilesMixin(object):
     default_template = """url("%s")"""
     patterns = (
         ("*.css", (
@@ -59,13 +60,9 @@ class CachedFilesMixin(object):
     )
 
     def __init__(self, *args, **kwargs):
-        super(CachedFilesMixin, self).__init__(*args, **kwargs)
+        super(HashedFilesMixin, self).__init__(*args, **kwargs)
-        try:
-            self.cache = caches['staticfiles']
-        except InvalidCacheBackendError:
-            # Use the default backend
-            self.cache = default_cache
         self._patterns = OrderedDict()
+        self.hashed_files = {}
         for extension, patterns in self.patterns:
             for pattern in patterns:
                 if isinstance(pattern, (tuple, list)):
@@ -119,9 +116,6 @@ class CachedFilesMixin(object):
             unparsed_name[2] += '?'
         return urlunsplit(unparsed_name)
 
-    def cache_key(self, name):
-        return 'staticfiles:%s' % hashlib.md5(force_bytes(name)).hexdigest()
-
     def url(self, name, force=False):
         """
         Returns the real URL in DEBUG mode.
@@ -133,15 +127,9 @@ class CachedFilesMixin(object):
             if urlsplit(clean_name).path.endswith('/'):  # don't hash paths
                 hashed_name = name
             else:
-                cache_key = self.cache_key(name)
+                hashed_name = self.stored_name(clean_name)
-                hashed_name = self.cache.get(cache_key)
-                if hashed_name is None:
-                    hashed_name = self.hashed_name(clean_name).replace('\\', '/')
-                    # set the cache if there was a miss
-                    # (e.g. if cache server goes down)
-                    self.cache.set(cache_key, hashed_name)
 
-        final_url = super(CachedFilesMixin, self).url(hashed_name)
+        final_url = super(HashedFilesMixin, self).url(hashed_name)
 
         # Special casing for a @font-face hack, like url(myfont.eot?#iefix")
         # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
@@ -220,7 +208,7 @@ class CachedFilesMixin(object):
             return
 
         # where to store the new paths
-        hashed_paths = {}
+        hashed_files = OrderedDict()
 
         # build a list of adjustable files
         matches = lambda path: matches_patterns(path, self._patterns.keys())
@@ -261,7 +249,7 @@ class CachedFilesMixin(object):
                     # then save the processed result
                     content_file = ContentFile(force_bytes(content))
                     saved_name = self._save(hashed_name, content_file)
-                    hashed_name = force_text(saved_name.replace('\\', '/'))
+                    hashed_name = force_text(self.clean_name(saved_name))
                     processed = True
                 else:
                     # or handle the case in which neither processing nor
@@ -269,14 +257,114 @@ class CachedFilesMixin(object):
                     if not hashed_file_exists:
                         processed = True
                         saved_name = self._save(hashed_name, original_file)
-                        hashed_name = force_text(saved_name.replace('\\', '/'))
+                        hashed_name = force_text(self.clean_name(saved_name))
 
                 # and then set the cache accordingly
-                hashed_paths[self.cache_key(name.replace('\\', '/'))] = hashed_name
+                hashed_files[self.hash_key(name)] = hashed_name
                 yield name, hashed_name, processed
 
-        # Finally set the cache
+        # Finally store the processed paths
-        self.cache.set_many(hashed_paths)
+        self.hashed_files.update(hashed_files)
+
+    def clean_name(self, name):
+        return name.replace('\\', '/')
+
+    def hash_key(self, name):
+        return name
+
+    def stored_name(self, name):
+        hash_key = self.hash_key(name)
+        cache_name = self.hashed_files.get(hash_key)
+        if cache_name is None:
+            cache_name = self.clean_name(self.hashed_name(name))
+            # store the hashed name if there was a miss, e.g.
+            # when the files are still processed
+            self.hashed_files[hash_key] = cache_name
+        return cache_name
+
+
+class ManifestFilesMixin(HashedFilesMixin):
+    manifest_version = '1.0'  # the manifest format standard
+    manifest_name = 'staticfiles.json'
+
+    def __init__(self, *args, **kwargs):
+        super(ManifestFilesMixin, self).__init__(*args, **kwargs)
+        self.hashed_files = self.load_manifest()
+
+    def read_manifest(self):
+        try:
+            with self.open(self.manifest_name) as manifest:
+                return manifest.read()
+        except IOError:
+            return None
+
+    def load_manifest(self):
+        content = self.read_manifest()
+        if content is None:
+            return OrderedDict()
+        try:
+            stored = json.loads(content, object_pairs_hook=OrderedDict)
+        except ValueError:
+            pass
+        else:
+            version = stored.get('version', None)
+            if version == '1.0':
+                return stored.get('paths', OrderedDict())
+        raise ValueError("Couldn't load manifest '%s' (version %s)" %
+                         (self.manifest_name, self.manifest_version))
+
+    def post_process(self, *args, **kwargs):
+        all_post_processed = super(ManifestFilesMixin,
+                                   self).post_process(*args, **kwargs)
+        for post_processed in all_post_processed:
+            yield post_processed
+        payload = {'paths': self.hashed_files, 'version': self.manifest_version}
+        if self.exists(self.manifest_name):
+            self.delete(self.manifest_name)
+        self._save(self.manifest_name, ContentFile(json.dumps(payload)))
+
+
+class _MappingCache(object):
+    """
+    A small dict-like wrapper for a given cache backend instance.
+    """
+    def __init__(self, cache):
+        self.cache = cache
+
+    def __setitem__(self, key, value):
+        self.cache.set(key, value)
+
+    def __getitem__(self, key):
+        value = self.cache.get(key, None)
+        if value is None:
+            raise KeyError("Couldn't find a file name '%s'" % key)
+        return value
+
+    def clear(self):
+        self.cache.clear()
+
+    def update(self, data):
+        self.cache.set_many(data)
+
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+
+class CachedFilesMixin(HashedFilesMixin):
+    def __init__(self, *args, **kwargs):
+        super(CachedFilesMixin, self).__init__(*args, **kwargs)
+        try:
+            self.hashed_files = _MappingCache(caches['staticfiles'])
+        except InvalidCacheBackendError:
+            # Use the default backend
+            self.hashed_files = _MappingCache(default_cache)
+
+    def hash_key(self, name):
+        key = hashlib.md5(force_bytes(self.clean_name(name))).hexdigest()
+        return 'staticfiles:%s' % key
 
 
 class CachedStaticFilesStorage(CachedFilesMixin, StaticFilesStorage):
@@ -287,6 +375,14 @@ class CachedStaticFilesStorage(CachedFilesMixin, StaticFilesStorage):
     pass
 
 
+class ManifestStaticFilesStorage(ManifestFilesMixin, StaticFilesStorage):
+    """
+    A static file system storage backend which also saves
+    hashed copies of the files it saves.
+    """
+    pass
+
+
 class AppStaticStorage(FileSystemStorage):
     """
     A file system storage backend that takes an app module and works

docs/ref/contrib/staticfiles.txt

@@ -210,90 +210,106 @@ StaticFilesStorage
 
 .. class:: storage.StaticFilesStorage
 
-    A subclass of the :class:`~django.core.files.storage.FileSystemStorage`
+A subclass of the :class:`~django.core.files.storage.FileSystemStorage`
-    storage backend that uses the :setting:`STATIC_ROOT` setting as the base
+storage backend that uses the :setting:`STATIC_ROOT` setting as the base
-    file system location and the :setting:`STATIC_URL` setting respectively
+file system location and the :setting:`STATIC_URL` setting respectively
-    as the base URL.
+as the base URL.
 
-    .. method:: post_process(paths, **options)
+.. method:: post_process(paths, **options)
 
-    This method is called by the :djadmin:`collectstatic` management command
+This method is called by the :djadmin:`collectstatic` management command
-    after each run and gets passed the local storages and paths of found
+after each run and gets passed the local storages and paths of found
-    files as a dictionary, as well as the command line options.
+files as a dictionary, as well as the command line options.
 
-    The :class:`~django.contrib.staticfiles.storage.CachedStaticFilesStorage`
+The :class:`~django.contrib.staticfiles.storage.CachedStaticFilesStorage`
-    uses this behind the scenes to replace the paths with their hashed
+uses this behind the scenes to replace the paths with their hashed
-    counterparts and update the cache appropriately.
+counterparts and update the cache appropriately.
+
+ManifestStaticFilesStorage
+--------------------------
+
+.. versionadded:: 1.7
+
+.. class:: storage.ManifestStaticFilesStorage
+
+A subclass of the :class:`~django.contrib.staticfiles.storage.StaticFilesStorage`
+storage backend which stores the file names it handles by appending the MD5
+hash of the file's content to the filename. For example, the file
+``css/styles.css`` would also be saved as ``css/styles.55e7cbb9ba48.css``.
+
+The purpose of this storage is to keep serving the old files in case some
+pages still refer to those files, e.g. because they are cached by you or
+a 3rd party proxy server. Additionally, it's very helpful if you want to
+apply `far future Expires headers`_ to the deployed files to speed up the
+load time for subsequent page visits.
+
+The storage backend automatically replaces the paths found in the saved
+files matching other saved files with the path of the cached copy (using
+the :meth:`~django.contrib.staticfiles.storage.StaticFilesStorage.post_process`
+method). The regular expressions used to find those paths
+(``django.contrib.staticfiles.storage.HashedFilesMixin.patterns``)
+by default covers the `@import`_ rule and `url()`_ statement of `Cascading
+Style Sheets`_. For example, the ``'css/styles.css'`` file with the
+content
+
+.. code-block:: css+django
+
+    @import url("../admin/css/base.css");
+
+would be replaced by calling the :meth:`~django.core.files.storage.Storage.url`
+method of the ``ManifestStaticFilesStorage`` storage backend, ultimately
+saving a ``'css/styles.55e7cbb9ba48.css'`` file with the following
+content:
+
+.. code-block:: css+django
+
+    @import url("../admin/css/base.27e20196a850.css");
+
+To enable the ``ManifestStaticFilesStorage`` you have to make sure the
+following requirements are met:
+
+* the :setting:`STATICFILES_STORAGE` setting is set to
+  ``'django.contrib.staticfiles.storage.ManifestStaticFilesStorage'``
+* the :setting:`DEBUG` setting is set to ``False``
+* you use the ``staticfiles`` :ttag:`static<staticfiles-static>` template
+  tag to refer to your static files in your templates
+* you've collected all your static files by using the
+  :djadmin:`collectstatic` management command
+
+Since creating the MD5 hash can be a performance burden to your website
+during runtime, ``staticfiles`` will automatically store the mapping with
+hashed names for all processed files in a file called ``staticfiles.json``.
+This happens once when you run the :djadmin:`collectstatic` management
+command.
+
+.. method:: file_hash(name, content=None)
+
+The method that is used when creating the hashed name of a file.
+Needs to return a hash for the given file name and content.
+By default it calculates a MD5 hash from the content's chunks as
+mentioned above. Feel free to override this method to use your own
+hashing algorithm.
+
+.. _`far future Expires headers`: http://developer.yahoo.com/performance/rules.html#expires
+.. _`@import`: http://www.w3.org/TR/CSS2/cascade.html#at-import
+.. _`url()`: http://www.w3.org/TR/CSS2/syndata.html#uri
+.. _`Cascading Style Sheets`: http://www.w3.org/Style/CSS/
 
 CachedStaticFilesStorage
 ------------------------
 
 .. class:: storage.CachedStaticFilesStorage
 
-    A subclass of the :class:`~django.contrib.staticfiles.storage.StaticFilesStorage`
+``CachedStaticFilesStorage`` is a similar class like the
-    storage backend which caches the files it saves by appending the MD5 hash
+:class:`~django.contrib.staticfiles.storage.ManifestStaticFilesStorage` class
-    of the file's content to the filename. For example, the file
+but uses Django's :doc:`caching framework</topics/cache>` for storing the
-    ``css/styles.css`` would also be saved as ``css/styles.55e7cbb9ba48.css``.
+hashed names of processed files instead of a static manifest file called
+``staticfiles.json``. This is mostly useful for situations in which you don't
+have accesss to the file system.
 
-    The purpose of this storage is to keep serving the old files in case some
+If you want to override certain options of the cache backend the storage uses,
-    pages still refer to those files, e.g. because they are cached by you or
+simply specify a custom entry in the :setting:`CACHES` setting named
-    a 3rd party proxy server. Additionally, it's very helpful if you want to
+``'staticfiles'``. It falls back to using the ``'default'`` cache backend.
-    apply `far future Expires headers`_ to the deployed files to speed up the
-    load time for subsequent page visits.
-
-    The storage backend automatically replaces the paths found in the saved
-    files matching other saved files with the path of the cached copy (using
-    the :meth:`~django.contrib.staticfiles.storage.StaticFilesStorage.post_process`
-    method). The regular expressions used to find those paths
-    (``django.contrib.staticfiles.storage.CachedStaticFilesStorage.cached_patterns``)
-    by default cover the `@import`_ rule and `url()`_ statement of `Cascading
-    Style Sheets`_. For example, the ``'css/styles.css'`` file with the
-    content
-
-    .. code-block:: css+django
-
-        @import url("../admin/css/base.css");
-
-    would be replaced by calling the
-    :meth:`~django.core.files.storage.Storage.url`
-    method of the ``CachedStaticFilesStorage`` storage backend, ultimately
-    saving a ``'css/styles.55e7cbb9ba48.css'`` file with the following
-    content:
-
-    .. code-block:: css+django
-
-        @import url("../admin/css/base.27e20196a850.css");
-
-    To enable the ``CachedStaticFilesStorage`` you have to make sure the
-    following requirements are met:
-
-    * the :setting:`STATICFILES_STORAGE` setting is set to
-      ``'django.contrib.staticfiles.storage.CachedStaticFilesStorage'``
-    * the :setting:`DEBUG` setting is set to ``False``
-    * you use the ``staticfiles`` :ttag:`static<staticfiles-static>` template
-      tag to refer to your static files in your templates
-    * you've collected all your static files by using the
-      :djadmin:`collectstatic` management command
-
-    Since creating the MD5 hash can be a performance burden to your website
-    during runtime, ``staticfiles`` will automatically try to cache the
-    hashed name for each file path using Django's :doc:`caching
-    framework</topics/cache>`. If you want to override certain options of the
-    cache backend the storage uses, simply specify a custom entry in the
-    :setting:`CACHES` setting named ``'staticfiles'``. It falls back to using
-    the ``'default'`` cache backend.
-
-    .. method:: file_hash(name, content=None)
-
-    The method that is used when creating the hashed name of a file.
-    Needs to return a hash for the given file name and content.
-    By default it calculates a MD5 hash from the content's chunks as
-    mentioned above.
-
-.. _`far future Expires headers`: http://developer.yahoo.com/performance/rules.html#expires
-.. _`@import`: http://www.w3.org/TR/CSS2/cascade.html#at-import
-.. _`url()`: http://www.w3.org/TR/CSS2/syndata.html#uri
-.. _`Cascading Style Sheets`: http://www.w3.org/Style/CSS/
 
 .. currentmodule:: django.contrib.staticfiles.templatetags.staticfiles
 

docs/releases/1.7.txt

@@ -340,6 +340,19 @@ Minor features
   and :attr:`~django.core.files.storage.FileSystemStorage.directory_permissions_mode`
   parameters. See :djadmin:`collectstatic` for example usage.
 
+* The :class:`~django.contrib.staticfiles.storage.CachedStaticFilesStorage`
+  backend gets a sibling class called
+  :class:`~django.contrib.staticfiles.storage.ManifestStaticFilesStorage`
+  that doesn't use the cache system at all but instead a JSON file called
+  ``staticfiles.json`` for storing the mapping between the original file name
+  (e.g. ``css/styles.css``) and the hashed file name (e.g.
+  ``css/styles.55e7cbb9ba48.css``. The ``staticfiles.json`` file is created
+  when running the :djadmin:`collectstatic` management command and should
+  be a less expensive alternative for remote storages such as Amazon S3.
+
+  See the :class:`~django.contrib.staticfiles.storage.ManifestStaticFilesStorage`
+  docs for more information.
+
 :mod:`django.contrib.syndication`
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

tests/staticfiles_tests/tests.py

@@ -371,20 +371,13 @@ class TestCollectionNonLocalStorage(CollectionTestCase, TestNoFilesCreated):
     pass
 
 
-# we set DEBUG to False here since the template tag wouldn't work otherwise
+def hashed_file_path(test, path):
-@override_settings(**dict(
+    fullpath = test.render_template(test.static_template_snippet(path))
-    TEST_SETTINGS,
+    return fullpath.replace(settings.STATIC_URL, '')
-    STATICFILES_STORAGE='django.contrib.staticfiles.storage.CachedStaticFilesStorage',
+
-    DEBUG=False,
+
-))
+class TestHashedFiles(object):
-class TestCollectionCachedStorage(BaseCollectionTestCase,
+    hashed_file_path = hashed_file_path
-        BaseStaticFilesTestCase, TestCase):
-    """
-    Tests for the Cache busting storage
-    """
-    def cached_file_path(self, path):
-        fullpath = self.render_template(self.static_template_snippet(path))
-        return fullpath.replace(settings.STATIC_URL, '')
 
     def test_template_tag_return(self):
         """
@@ -405,7 +398,7 @@ class TestCollectionCachedStorage(BaseCollectionTestCase,
                                  "/static/path/?query")
 
     def test_template_tag_simple_content(self):
-        relpath = self.cached_file_path("cached/styles.css")
+        relpath = self.hashed_file_path("cached/styles.css")
         self.assertEqual(relpath, "cached/styles.93b1147e8552.css")
         with storage.staticfiles_storage.open(relpath) as relfile:
             content = relfile.read()
@@ -413,7 +406,7 @@ class TestCollectionCachedStorage(BaseCollectionTestCase,
             self.assertIn(b"other.d41d8cd98f00.css", content)
 
     def test_path_ignored_completely(self):
-        relpath = self.cached_file_path("cached/css/ignored.css")
+        relpath = self.hashed_file_path("cached/css/ignored.css")
         self.assertEqual(relpath, "cached/css/ignored.6c77f2643390.css")
         with storage.staticfiles_storage.open(relpath) as relfile:
             content = relfile.read()
@@ -424,7 +417,7 @@ class TestCollectionCachedStorage(BaseCollectionTestCase,
             self.assertIn(b'//foobar', content)
 
     def test_path_with_querystring(self):
-        relpath = self.cached_file_path("cached/styles.css?spam=eggs")
+        relpath = self.hashed_file_path("cached/styles.css?spam=eggs")
         self.assertEqual(relpath,
                          "cached/styles.93b1147e8552.css?spam=eggs")
         with storage.staticfiles_storage.open(
@@ -434,7 +427,7 @@ class TestCollectionCachedStorage(BaseCollectionTestCase,
             self.assertIn(b"other.d41d8cd98f00.css", content)
 
     def test_path_with_fragment(self):
-        relpath = self.cached_file_path("cached/styles.css#eggs")
+        relpath = self.hashed_file_path("cached/styles.css#eggs")
         self.assertEqual(relpath, "cached/styles.93b1147e8552.css#eggs")
         with storage.staticfiles_storage.open(
                 "cached/styles.93b1147e8552.css") as relfile:
@@ -443,7 +436,7 @@ class TestCollectionCachedStorage(BaseCollectionTestCase,
             self.assertIn(b"other.d41d8cd98f00.css", content)
 
     def test_path_with_querystring_and_fragment(self):
-        relpath = self.cached_file_path("cached/css/fragments.css")
+        relpath = self.hashed_file_path("cached/css/fragments.css")
         self.assertEqual(relpath, "cached/css/fragments.75433540b096.css")
         with storage.staticfiles_storage.open(relpath) as relfile:
             content = relfile.read()
@@ -453,7 +446,7 @@ class TestCollectionCachedStorage(BaseCollectionTestCase,
             self.assertIn(b'#default#VML', content)
 
     def test_template_tag_absolute(self):
-        relpath = self.cached_file_path("cached/absolute.css")
+        relpath = self.hashed_file_path("cached/absolute.css")
         self.assertEqual(relpath, "cached/absolute.23f087ad823a.css")
         with storage.staticfiles_storage.open(relpath) as relfile:
             content = relfile.read()
@@ -462,7 +455,7 @@ class TestCollectionCachedStorage(BaseCollectionTestCase,
             self.assertIn(b'/static/cached/img/relative.acae32e4532b.png', content)
 
     def test_template_tag_denorm(self):
-        relpath = self.cached_file_path("cached/denorm.css")
+        relpath = self.hashed_file_path("cached/denorm.css")
         self.assertEqual(relpath, "cached/denorm.c5bd139ad821.css")
         with storage.staticfiles_storage.open(relpath) as relfile:
             content = relfile.read()
@@ -472,7 +465,7 @@ class TestCollectionCachedStorage(BaseCollectionTestCase,
             self.assertIn(b'url("img/relative.acae32e4532b.png', content)
 
     def test_template_tag_relative(self):
-        relpath = self.cached_file_path("cached/relative.css")
+        relpath = self.hashed_file_path("cached/relative.css")
         self.assertEqual(relpath, "cached/relative.2217ea7273c2.css")
         with storage.staticfiles_storage.open(relpath) as relfile:
             content = relfile.read()
@@ -484,13 +477,13 @@ class TestCollectionCachedStorage(BaseCollectionTestCase,
 
     def test_import_replacement(self):
         "See #18050"
-        relpath = self.cached_file_path("cached/import.css")
+        relpath = self.hashed_file_path("cached/import.css")
         self.assertEqual(relpath, "cached/import.2b1d40b0bbd4.css")
         with storage.staticfiles_storage.open(relpath) as relfile:
             self.assertIn(b"""import url("styles.93b1147e8552.css")""", relfile.read())
 
     def test_template_tag_deep_relative(self):
-        relpath = self.cached_file_path("cached/css/window.css")
+        relpath = self.hashed_file_path("cached/css/window.css")
         self.assertEqual(relpath, "cached/css/window.9db38d5169f3.css")
         with storage.staticfiles_storage.open(relpath) as relfile:
             content = relfile.read()
@@ -498,26 +491,11 @@ class TestCollectionCachedStorage(BaseCollectionTestCase,
             self.assertIn(b'url("img/window.acae32e4532b.png")', content)
 
     def test_template_tag_url(self):
-        relpath = self.cached_file_path("cached/url.css")
+        relpath = self.hashed_file_path("cached/url.css")
         self.assertEqual(relpath, "cached/url.615e21601e4b.css")
         with storage.staticfiles_storage.open(relpath) as relfile:
             self.assertIn(b"https://", relfile.read())
 
-    def test_cache_invalidation(self):
-        name = "cached/styles.css"
-        hashed_name = "cached/styles.93b1147e8552.css"
-        # check if the cache is filled correctly as expected
-        cache_key = storage.staticfiles_storage.cache_key(name)
-        cached_name = storage.staticfiles_storage.cache.get(cache_key)
-        self.assertEqual(self.cached_file_path(name), cached_name)
-        # clearing the cache to make sure we re-set it correctly in the url method
-        storage.staticfiles_storage.cache.clear()
-        cached_name = storage.staticfiles_storage.cache.get(cache_key)
-        self.assertEqual(cached_name, None)
-        self.assertEqual(self.cached_file_path(name), hashed_name)
-        cached_name = storage.staticfiles_storage.cache.get(cache_key)
-        self.assertEqual(cached_name, hashed_name)
-
     def test_post_processing(self):
         """Test that post_processing behaves correctly.
 
@@ -545,18 +523,8 @@ class TestCollectionCachedStorage(BaseCollectionTestCase,
         self.assertIn(os.path.join('cached', 'css', 'img', 'window.png'), stats['unmodified'])
         self.assertIn(os.path.join('test', 'nonascii.css'), stats['post_processed'])
 
-    def test_cache_key_memcache_validation(self):
-        """
-        Handle cache key creation correctly, see #17861.
-        """
-        name = "/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/" + "\x16" + "\xb4"
-        cache_key = storage.staticfiles_storage.cache_key(name)
-        cache_validator = BaseCache({})
-        cache_validator.validate_key(cache_key)
-        self.assertEqual(cache_key, 'staticfiles:821ea71ef36f95b3922a77f7364670e7')
-
     def test_css_import_case_insensitive(self):
-        relpath = self.cached_file_path("cached/styles_insensitive.css")
+        relpath = self.hashed_file_path("cached/styles_insensitive.css")
         self.assertEqual(relpath, "cached/styles_insensitive.2f0151cca872.css")
         with storage.staticfiles_storage.open(relpath) as relfile:
             content = relfile.read()
@@ -579,6 +547,67 @@ class TestCollectionCachedStorage(BaseCollectionTestCase,
         self.assertEqual("Post-processing 'faulty.css' failed!\n\n", err.getvalue())
 
 
+# we set DEBUG to False here since the template tag wouldn't work otherwise
+@override_settings(**dict(
+    TEST_SETTINGS,
+    STATICFILES_STORAGE='django.contrib.staticfiles.storage.CachedStaticFilesStorage',
+    DEBUG=False,
+))
+class TestCollectionCachedStorage(TestHashedFiles, BaseCollectionTestCase,
+        BaseStaticFilesTestCase, TestCase):
+    """
+    Tests for the Cache busting storage
+    """
+    def test_cache_invalidation(self):
+        name = "cached/styles.css"
+        hashed_name = "cached/styles.93b1147e8552.css"
+        # check if the cache is filled correctly as expected
+        cache_key = storage.staticfiles_storage.hash_key(name)
+        cached_name = storage.staticfiles_storage.hashed_files.get(cache_key)
+        self.assertEqual(self.hashed_file_path(name), cached_name)
+        # clearing the cache to make sure we re-set it correctly in the url method
+        storage.staticfiles_storage.hashed_files.clear()
+        cached_name = storage.staticfiles_storage.hashed_files.get(cache_key)
+        self.assertEqual(cached_name, None)
+        self.assertEqual(self.hashed_file_path(name), hashed_name)
+        cached_name = storage.staticfiles_storage.hashed_files.get(cache_key)
+        self.assertEqual(cached_name, hashed_name)
+
+    def test_cache_key_memcache_validation(self):
+        """
+        Handle cache key creation correctly, see #17861.
+        """
+        name = "/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/" + "\x16" + "\xb4"
+        cache_key = storage.staticfiles_storage.hash_key(name)
+        cache_validator = BaseCache({})
+        cache_validator.validate_key(cache_key)
+        self.assertEqual(cache_key, 'staticfiles:821ea71ef36f95b3922a77f7364670e7')
+
+
+# we set DEBUG to False here since the template tag wouldn't work otherwise
+@override_settings(**dict(
+    TEST_SETTINGS,
+    STATICFILES_STORAGE='django.contrib.staticfiles.storage.ManifestStaticFilesStorage',
+    DEBUG=False,
+))
+class TestCollectionManifestStorage(TestHashedFiles, BaseCollectionTestCase,
+        BaseStaticFilesTestCase, TestCase):
+    """
+    Tests for the Cache busting storage
+    """
+    def test_manifest_exists(self):
+        filename = storage.staticfiles_storage.manifest_name
+        path = storage.staticfiles_storage.path(filename)
+        self.assertTrue(os.path.exists(path))
+
+    def test_loaded_cache(self):
+        self.assertNotEqual(storage.staticfiles_storage.hashed_files, {})
+        manifest_content = storage.staticfiles_storage.read_manifest()
+        self.assertIn('"version": "%s"' %
+                      storage.staticfiles_storage.manifest_version,
+                      force_text(manifest_content))
+
+
 # we set DEBUG to False here since the template tag wouldn't work otherwise
 @override_settings(**dict(
     TEST_SETTINGS,
@@ -590,9 +619,7 @@ class TestCollectionSimpleCachedStorage(BaseCollectionTestCase,
     """
     Tests for the Cache busting storage
     """
-    def cached_file_path(self, path):
+    hashed_file_path = hashed_file_path
-        fullpath = self.render_template(self.static_template_snippet(path))
-        return fullpath.replace(settings.STATIC_URL, '')
 
     def test_template_tag_return(self):
         """
@@ -611,7 +638,7 @@ class TestCollectionSimpleCachedStorage(BaseCollectionTestCase,
                                  "/static/path/?query")
 
     def test_template_tag_simple_content(self):
-        relpath = self.cached_file_path("cached/styles.css")
+        relpath = self.hashed_file_path("cached/styles.css")
         self.assertEqual(relpath, "cached/styles.deploy12345.css")
         with storage.staticfiles_storage.open(relpath) as relfile:
             content = relfile.read()