Added some better error reporting and path handling when creating template paths.

We now raise UnicodeDecodeError for non-UTF-8 bytestrings (thanks to Daniel
Pope for diagnosing this was being swallowed by ValueError) and allow UTF-8
bytestrings as template directories.

Refs #8965.


git-svn-id: http://code.djangoproject.com/svn/django/trunk@9161 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/template/loaders/app_directories.py

@@ -33,11 +33,19 @@ for app in settings.INSTALLED_APPS:
 app_template_dirs = tuple(app_template_dirs)
 
 def get_template_sources(template_name, template_dirs=None):
+    """
+    Returns the absolute paths to "template_name", when appended to each
+    directory in "template_dirs". Any paths that don't lie inside one of the
+    template dirs are excluded from the result set, for security reasons.
+    """
     if not template_dirs:
         template_dirs = app_template_dirs
     for template_dir in template_dirs:
         try:
             yield safe_join(template_dir, template_name)
+        except UnicodeDecodeError:
+            # The template dir name was a bytestring that wasn't valid UTF-8.
+            raise
         except ValueError:
             # The joined path was located outside of template_dir.
             pass

django/template/loaders/filesystem.py

@@ -7,13 +7,23 @@ from django.template import TemplateDoesNotExist
 from django.utils._os import safe_join
 
 def get_template_sources(template_name, template_dirs=None):
+    """
+    Returns the absolute paths to "template_name", when appended to each
+    directory in "template_dirs". Any paths that don't lie inside one of the
+    template dirs are excluded from the result set, for security reasons.
+    """
     if not template_dirs:
         template_dirs = settings.TEMPLATE_DIRS
     for template_dir in template_dirs:
         try:
             yield safe_join(template_dir, template_name)
+        except UnicodeDecodeError:
+            # The template dir name was a bytestring that wasn't valid UTF-8.
+            raise
         except ValueError:
-            # The joined path was located outside of template_dir.
+            # The joined path was located outside of this particular
+            # template_dir (it might be inside another one, so this isn't
+            # fatal).
             pass
 
 def load_template_source(template_name, template_dirs=None):

django/utils/_os.py

@@ -1,4 +1,5 @@
 from os.path import join, normcase, abspath, sep
+from django.utils.encoding import force_unicode
 
 def safe_join(base, *paths):
     """
@@ -10,6 +11,8 @@ def safe_join(base, *paths):
     """
     # We need to use normcase to ensure we don't false-negative on case
     # insensitive operating systems (like Windows).
+    base = force_unicode(base)
+    paths = [force_unicode(p) for p in paths]
     final_path = normcase(abspath(join(base, *paths)))
     base_path = normcase(abspath(base))
     base_path_len = len(base_path)

tests/regressiontests/templates/tests.py

@@ -92,34 +92,45 @@ class UTF8Class:
 class Templates(unittest.TestCase):
     def test_loaders_security(self):
         def test_template_sources(path, template_dirs, expected_sources):
-            # Fix expected sources so they are normcased and abspathed
+            if isinstance(expected_sources, list):
-            expected_sources = [os.path.normcase(os.path.abspath(s)) for s in expected_sources]
+                # Fix expected sources so they are normcased and abspathed
-            # Test app_directories loader
+                expected_sources = [os.path.normcase(os.path.abspath(s)) for s in expected_sources]
-            sources = app_directories.get_template_sources(path, template_dirs)
+            # Test the two loaders (app_directores and filesystem).
-            self.assertEqual(list(sources), expected_sources)
+            func1 = lambda p, t: list(app_directories.get_template_sources(p, t))
-            # Test filesystem loader
+            func2 = lambda p, t: list(filesystem.get_template_sources(p, t))
-            sources = filesystem.get_template_sources(path, template_dirs)
+            for func in (func1, func2):
-            self.assertEqual(list(sources), expected_sources)
+                if isinstance(expected_sources, list):
+                    self.assertEqual(func(path, template_dirs), expected_sources)
+                else:
+                    self.assertRaises(expected_sources, func, path, template_dirs)
 
         template_dirs = ['/dir1', '/dir2']
         test_template_sources('index.html', template_dirs,
                               ['/dir1/index.html', '/dir2/index.html'])
-        test_template_sources('/etc/passwd', template_dirs,
+        test_template_sources('/etc/passwd', template_dirs, [])
-                              [])
         test_template_sources('etc/passwd', template_dirs,
                               ['/dir1/etc/passwd', '/dir2/etc/passwd'])
-        test_template_sources('../etc/passwd', template_dirs,
+        test_template_sources('../etc/passwd', template_dirs, [])
-                              [])
+        test_template_sources('../../../etc/passwd', template_dirs, [])
-        test_template_sources('../../../etc/passwd', template_dirs,
-                              [])
         test_template_sources('/dir1/index.html', template_dirs,
                               ['/dir1/index.html'])
         test_template_sources('../dir2/index.html', template_dirs,
                               ['/dir2/index.html'])
-        test_template_sources('/dir1blah', template_dirs,
+        test_template_sources('/dir1blah', template_dirs, [])
-                              [])
+        test_template_sources('../dir1blah', template_dirs, [])
-        test_template_sources('../dir1blah', template_dirs,
+
-                              [])
+        # UTF-8 bytestrings are permitted.
+        test_template_sources('\xc3\x85ngstr\xc3\xb6m', template_dirs,
+                              [u'/dir1/Ångström', u'/dir2/Ångström'])
+        # Unicode strings are permitted.
+        test_template_sources(u'Ångström', template_dirs,
+                              [u'/dir1/Ångström', u'/dir2/Ångström'])
+        test_template_sources(u'Ångström', ['/Straße'], [u'/Straße/Ångström'])
+        test_template_sources('\xc3\x85ngstr\xc3\xb6m', ['/Straße'],
+                              [u'/Straße/Ångström'])
+        # Invalid UTF-8 encoding in bytestrings is not. Should raise a
+        # semi-useful error message.
+        test_template_sources('\xc3\xc3', template_dirs, UnicodeDecodeError)
 
         # Case insensitive tests (for win32). Not run unless we're on
         # a case insensitive operating system.
@@ -372,7 +383,7 @@ class Templates(unittest.TestCase):
 
             # Numbers as filter arguments should work
             'filter-syntax19': ('{{ var|truncatewords:1 }}', {"var": "hello world"}, "hello ..."),
-            
+
             #filters should accept empty string constants
             'filter-syntax20': ('{{ ""|default_if_none:"was none" }}', {}, ""),