Made django.test.testcases not depend on staticfiles contrib app.

Do this by introducing a django.contrib.staticfiles.testing.StaticLiveServerCase
unittest TestCase subclass.

Fixes #20739.

django/contrib/staticfiles/testing.py

@@ -0,0 +1,14 @@
+from django.test import LiveServerTestCase
+
+from django.contrib.staticfiles.handlers import StaticFilesHandler
+
+
+class StaticLiveServerCase(LiveServerTestCase):
+    """
+    Extends django.test.LiveServerTestCase to transparently overlay at test
+    execution-time the assets provided by the staticfiles app finders. This
+    means you don't need to run collectstatic before or as a part of your tests
+    setup.
+    """
+
+    static_handler = StaticFilesHandler

django/contrib/staticfiles/views.py

@@ -27,7 +27,7 @@ def serve(request, path, insecure=False, **kwargs):
 
     in your URLconf.
 
-    It uses the django.views.static view to serve the found files.
+    It uses the django.views.static.serve() view to serve the found files.
     """
     if not settings.DEBUG and not insecure:
         raise Http404

django/test/testcases.py

@@ -6,23 +6,25 @@ import errno
 from functools import wraps
 import json
 import os
+import posixpath
 import re
 import sys
-import socket
 import threading
 import unittest
 from unittest import skipIf         # Imported here for backward compatibility
 from unittest.util import safe_repr
 try:
-    from urllib.parse import urlsplit, urlunsplit
+    from urllib.parse import urlsplit, urlunsplit, urlparse, unquote
+    from urllib.request import url2pathname
 except ImportError:     # Python 2
-    from urlparse import urlsplit, urlunsplit
+    from urlparse import urlsplit, urlunsplit, urlparse
+    from urllib import url2pathname, unquote
 
 from django.conf import settings
-from django.contrib.staticfiles.handlers import StaticFilesHandler
 from django.core import mail
 from django.core.exceptions import ValidationError, ImproperlyConfigured
 from django.core.handlers.wsgi import WSGIHandler
+from django.core.handlers.base import get_path_info
 from django.core.management import call_command
 from django.core.management.color import no_style
 from django.core.management.commands import flush
@@ -933,10 +935,70 @@ class QuietWSGIRequestHandler(WSGIRequestHandler):
         pass
 
 
-class _MediaFilesHandler(StaticFilesHandler):
+class FSFilesHandler(WSGIHandler):
     """
-    Handler for serving the media files. This is a private class that is
+    WSGI middleware that intercepts calls to a directory, as defined by one of
-    meant to be used solely as a convenience by LiveServerThread.
+    the *_ROOT settings, and serves those files, publishing them under *_URL.
+    """
+    def __init__(self, application):
+        self.application = application
+        self.base_url = urlparse(self.get_base_url())
+        super(FSFilesHandler, self).__init__()
+
+    def _should_handle(self, path):
+        """
+        Checks if the path should be handled. Ignores the path if:
+
+        * the host is provided as part of the base_url
+        * the request's path isn't under the media path (or equal)
+        """
+        return path.startswith(self.base_url[2]) and not self.base_url[1]
+
+    def file_path(self, url):
+        """
+        Returns the relative path to the file on disk for the given URL.
+        """
+        relative_url = url[len(self.base_url[2]):]
+        return url2pathname(relative_url)
+
+    def get_response(self, request):
+        from django.http import Http404
+
+        if self._should_handle(request.path):
+            try:
+                return self.serve(request)
+            except Http404:
+                pass
+        return super(FSFilesHandler, self).get_response(request)
+
+    def serve(self, request):
+        os_rel_path = self.file_path(request.path)
+        final_rel_path = posixpath.normpath(unquote(os_rel_path)).lstrip('/')
+        return serve(request, final_rel_path, document_root=self.get_base_dir())
+
+    def __call__(self, environ, start_response):
+        if not self._should_handle(get_path_info(environ)):
+            return self.application(environ, start_response)
+        return super(FSFilesHandler, self).__call__(environ, start_response)
+
+
+class _StaticFilesHandler(FSFilesHandler):
+    """
+    Handler for serving static files. A private class that is meant to be used
+    solely as a convenience by LiveServerThread.
+    """
+
+    def get_base_dir(self):
+        return settings.STATIC_ROOT
+
+    def get_base_url(self):
+        return settings.STATIC_URL
+
+
+class _MediaFilesHandler(FSFilesHandler):
+    """
+    Handler for serving the media files. A private class that is meant to be
+    used solely as a convenience by LiveServerThread.
     """
 
     def get_base_dir(self):
@@ -945,22 +1007,19 @@ class _MediaFilesHandler(StaticFilesHandler):
     def get_base_url(self):
         return settings.MEDIA_URL
 
-    def serve(self, request):
-        relative_url = request.path[len(self.base_url[2]):]
-        return serve(request, relative_url, document_root=self.get_base_dir())
-
 
 class LiveServerThread(threading.Thread):
     """
     Thread for running a live http server while the tests are running.
     """
 
-    def __init__(self, host, possible_ports, connections_override=None):
+    def __init__(self, host, possible_ports, static_handler, connections_override=None):
         self.host = host
         self.port = None
         self.possible_ports = possible_ports
         self.is_ready = threading.Event()
         self.error = None
+        self.static_handler = static_handler
         self.connections_override = connections_override
         super(LiveServerThread, self).__init__()
 
@@ -976,7 +1035,7 @@ class LiveServerThread(threading.Thread):
                 connections[alias] = conn
         try:
             # Create the handler for serving static and media files
-            handler = StaticFilesHandler(_MediaFilesHandler(WSGIHandler()))
+            handler = self.static_handler(_MediaFilesHandler(WSGIHandler()))
 
             # Go through the list of possible ports, hoping that we can find
             # one that is free to use for the WSGI server.
@@ -1028,6 +1087,8 @@ class LiveServerTestCase(TransactionTestCase):
     other thread can see the changes.
     """
 
+    static_handler = _StaticFilesHandler
+
     @property
     def live_server_url(self):
         return 'http://%s:%s' % (
@@ -1069,8 +1130,9 @@ class LiveServerTestCase(TransactionTestCase):
         except Exception:
             msg = 'Invalid address ("%s") for live server.' % specified_address
             six.reraise(ImproperlyConfigured, ImproperlyConfigured(msg), sys.exc_info()[2])
-        cls.server_thread = LiveServerThread(
+        cls.server_thread = LiveServerThread(host, possible_ports,
-            host, possible_ports, connections_override)
+                                             cls.static_handler,
+                                             connections_override=connections_override)
         cls.server_thread.daemon = True
         cls.server_thread.start()
 

docs/howto/static-files/index.txt

@@ -100,6 +100,33 @@ this by adding the following snippet to your urls.py::
     the given prefix is local (e.g. ``/static/``) and not a URL (e.g.
     ``http://static.example.com/``).
 
+.. _staticfiles-testing-support:
+
+Testing
+=======
+
+When running tests that use actual HTTP requests instead of the built-in
+testing client (i.e. when using the built-in :class:`LiveServerTestCase
+<django.test.LiveServerTestCase>`) the static assets need to be served along
+the rest of the content so the test environment reproduces the real one as
+faithfully as possible, but ``LiveServerTestCase`` has only very basic static
+file-serving functionality: It doesn't know about the finders feature of the
+``staticfiles`` application and assumes the static content has already been
+collected under :setting:`STATIC_ROOT`.
+
+Because of this, ``staticfiles`` ships its own
+:class:`django.contrib.staticfiles.testing.StaticLiveServerCase`, a subclass
+of the built-in one that has the ability to transparently serve all the assets
+during execution of these tests in a way very similar to what we get at
+development time with ``DEBUG = True``, i.e. without having to collect them
+using :djadmin:`collectstatic` first.
+
+.. versionadded:: 1.7
+
+    :class:`django.contrib.staticfiles.testing.StaticLiveServerCase` is new in
+    Django 1.7. Previously its functionality was provided by
+    :class:`django.test.LiveServerTestCase`.
+
 Deployment
 ==========
 

docs/ref/contrib/staticfiles.txt

@@ -406,3 +406,26 @@ files in app directories.
     That's because this view is **grossly inefficient** and probably
     **insecure**. This is only intended for local development, and should
     **never be used in production**.
+
+Specialized test case to support 'live testing'
+-----------------------------------------------
+
+.. class:: testing.StaticLiveServerCase
+
+This unittest TestCase subclass extends :class:`django.test.LiveServerTestCase`.
+
+Just like its parent, you can use it to write tests that involve running the
+code under test and consuming it with testing tools through HTTP (e.g. Selenium,
+PhantomJS, etc.), because of which it's needed that the static assets are also
+published.
+
+But given the fact that it makes use of the
+:func:`django.contrib.staticfiles.views.serve` view described above, it can
+transparently overlay at test execution-time the assets provided by the
+``staticfiles`` finders. This means you don't need to run
+:djadmin:`collectstatic` before or as a part of your tests setup.
+
+.. versionadded:: 1.7
+
+    ``StaticLiveServerCase`` is new in Django 1.7. Previously its functionality
+    was provided by :class:`django.test.LiveServerTestCase`.

docs/releases/1.7.txt

@@ -332,6 +332,20 @@ Miscellaneous
   Define a ``get_absolute_url()`` method on your own custom user object or use
   :setting:`ABSOLUTE_URL_OVERRIDES` if you want a URL for your user.
 
+* The static asset-serving functionality of the
+  :class:`django.test.LiveServerTestCase` class has been simplified: Now it's
+  only able to serve content already present in :setting:`STATIC_ROOT` when
+  tests are run. The ability to transparently serve all the static assets
+  (similarly to what one gets with :setting:`DEBUG = True <DEBUG>` at
+  development-time) has been moved to a new class that lives in the
+  ``staticfiles`` application (the one actually in charge of such feature):
+  :class:`django.contrib.staticfiles.testing.StaticLiveServerCase`. In other
+  words, ``LiveServerTestCase`` itself is less powerful but at the same time
+  has less magic.
+
+  Rationale behind this is removal of dependency of non-contrib code on
+  contrib applications.
+
 Features deprecated in 1.7
 ==========================
 

docs/topics/testing/overview.txt

@@ -1041,11 +1041,25 @@ out the `full reference`_ for more details.
 .. _full reference: http://selenium-python.readthedocs.org/en/latest/api.html
 .. _Firefox: http://www.mozilla.com/firefox/
 
-.. note::
+.. versionchanged:: 1.7
 
-    ``LiveServerTestCase`` makes use of the :doc:`staticfiles contrib app
+    Before Django 1.7 ``LiveServerTestCase`` used to rely on the
-    </howto/static-files/index>` so you'll need to have your project configured
+    :doc:`staticfiles contrib app </howto/static-files/index>` to get the
-    accordingly (in particular by setting :setting:`STATIC_URL`).
+    static assets of the application(s) under test transparently served at their
+    expected locations during the execution of these tests.
+
+    In Django 1.7 this dependency of core functionality on a ``contrib``
+    appplication has been removed, because of which ``LiveServerTestCase``
+    ability in this respect has been retrofitted to simply publish the contents
+    of the file system under :setting:`STATIC_ROOT` at the :setting:`STATIC_URL`
+    URL.
+
+    If you use the ``staticfiles`` app in your project and need to perform live
+    testing then you might want to consider using the
+    :class:`~django.contrib.staticfiles.testing.StaticLiveServerCase` subclass
+    shipped with it instead because it's the one that implements the original
+    behavior now. See :ref:`the relevant documentation
+    <staticfiles-testing-support>` for more details.
 
 .. note::
 

tests/servers/tests.py

@@ -82,13 +82,6 @@ class LiveServerAddress(LiveServerBase):
         cls.raises_exception('localhost:8081-blah', ImproperlyConfigured)
         cls.raises_exception('localhost:8081-8082-8083', ImproperlyConfigured)
 
-        # If contrib.staticfiles isn't configured properly, the exception
-        # should bubble up to the main thread.
-        old_STATIC_URL = TEST_SETTINGS['STATIC_URL']
-        TEST_SETTINGS['STATIC_URL'] = None
-        cls.raises_exception('localhost:8081', ImproperlyConfigured)
-        TEST_SETTINGS['STATIC_URL'] = old_STATIC_URL
-
         # Restore original environment variable
         if address_predefined:
             os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS'] = old_address
@@ -145,13 +138,18 @@ class LiveServerViews(LiveServerBase):
         f = self.urlopen('/static/example_static_file.txt')
         self.assertEqual(f.read().rstrip(b'\r\n'), b'example static file')
 
-    def test_collectstatic_emulation(self):
+    def test_no_collectstatic_emulation(self):
         """
-        Test LiveServerTestCase use of staticfiles' serve() allows it to
+        Test that LiveServerTestCase reports a 404 status code when HTTP client
-        discover app's static assets without having to collectstatic first.
+        tries to access a static file that isn't explictly put under
+        STATIC_ROOT.
         """
-        f = self.urlopen('/static/another_app/another_app_static_file.txt')
+        try:
-        self.assertEqual(f.read().rstrip(b'\r\n'), b'static file from another_app')
+            self.urlopen('/static/another_app/another_app_static_file.txt')
+        except HTTPError as err:
+            self.assertEqual(err.code, 404, 'Expected 404 response')
+        else:
+            self.fail('Expected 404 response (got %d)' % err.code)
 
     def test_media_files(self):
         """

tests/staticfiles_tests/test_liveserver.py

@@ -0,0 +1,101 @@
+"""
+A subset of the tests in tests/servers/tests exercicing
+django.contrib.staticfiles.testing.StaticLiveServerCase instead of
+django.test.LiveServerTestCase.
+"""
+
+import os
+try:
+    from urllib.request import urlopen
+except ImportError:     # Python 2
+    from urllib2 import urlopen
+
+from django.core.exceptions import ImproperlyConfigured
+from django.test.utils import override_settings
+from django.utils._os import upath
+
+from django.contrib.staticfiles.testing import StaticLiveServerCase
+
+
+TEST_ROOT = os.path.dirname(upath(__file__))
+TEST_SETTINGS = {
+    'MEDIA_URL': '/media/',
+    'STATIC_URL': '/static/',
+    'MEDIA_ROOT': os.path.join(TEST_ROOT, 'project', 'site_media', 'media'),
+    'STATIC_ROOT': os.path.join(TEST_ROOT, 'project', 'site_media', 'static'),
+}
+
+
+class LiveServerBase(StaticLiveServerCase):
+
+    available_apps = []
+
+    @classmethod
+    def setUpClass(cls):
+        # Override settings
+        cls.settings_override = override_settings(**TEST_SETTINGS)
+        cls.settings_override.enable()
+        super(LiveServerBase, cls).setUpClass()
+
+    @classmethod
+    def tearDownClass(cls):
+        # Restore original settings
+        cls.settings_override.disable()
+        super(LiveServerBase, cls).tearDownClass()
+
+
+class StaticLiveServerChecks(LiveServerBase):
+
+    @classmethod
+    def setUpClass(cls):
+        # Backup original environment variable
+        address_predefined = 'DJANGO_LIVE_TEST_SERVER_ADDRESS' in os.environ
+        old_address = os.environ.get('DJANGO_LIVE_TEST_SERVER_ADDRESS')
+
+        # If contrib.staticfiles isn't configured properly, the exception
+        # should bubble up to the main thread.
+        old_STATIC_URL = TEST_SETTINGS['STATIC_URL']
+        TEST_SETTINGS['STATIC_URL'] = None
+        cls.raises_exception('localhost:8081', ImproperlyConfigured)
+        TEST_SETTINGS['STATIC_URL'] = old_STATIC_URL
+
+        # Restore original environment variable
+        if address_predefined:
+            os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS'] = old_address
+        else:
+            del os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS']
+
+    @classmethod
+    def tearDownClass(cls):
+        # skip it, as setUpClass doesn't call its parent either
+        pass
+
+    @classmethod
+    def raises_exception(cls, address, exception):
+        os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS'] = address
+        try:
+            super(StaticLiveServerChecks, cls).setUpClass()
+            raise Exception("The line above should have raised an exception")
+        except exception:
+            pass
+        finally:
+            super(StaticLiveServerChecks, cls).tearDownClass()
+
+    def test_test_test(self):
+        # Intentionally empty method so that the test is picked up by the
+        # test runner and the overridden setUpClass() method is executed.
+        pass
+
+
+class StaticLiveServerView(LiveServerBase):
+
+    def urlopen(self, url):
+        return urlopen(self.live_server_url + url)
+
+    def test_collectstatic_emulation(self):
+        """
+        Test that StaticLiveServerCase use of staticfiles' serve() allows it to
+        discover app's static assets without having to collectstatic first.
+        """
+        f = self.urlopen('/static/test/file.txt')
+        self.assertEqual(f.read().rstrip(b'\r\n'), b'In app media directory.')