From 8fb1459b5294fb9327b241fffec8576c5aa3fc7e Mon Sep 17 00:00:00 2001 From: Malcolm Tredinnick Date: Mon, 6 Oct 2008 06:34:54 +0000 Subject: [PATCH] Added some better error reporting and path handling when creating template paths. We now raise UnicodeDecodeError for non-UTF-8 bytestrings (thanks to Daniel Pope for diagnosing this was being swallowed by ValueError) and allow UTF-8 bytestrings as template directories. Refs #8965. git-svn-id: http://code.djangoproject.com/svn/django/trunk@9161 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- django/template/loaders/app_directories.py | 8 ++++ django/template/loaders/filesystem.py | 12 +++++- django/utils/_os.py | 3 ++ tests/regressiontests/templates/tests.py | 49 +++++++++++++--------- 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/django/template/loaders/app_directories.py b/django/template/loaders/app_directories.py index f0f4b1a70e..24a1beec16 100644 --- a/django/template/loaders/app_directories.py +++ b/django/template/loaders/app_directories.py @@ -33,11 +33,19 @@ for app in settings.INSTALLED_APPS: app_template_dirs = tuple(app_template_dirs) def get_template_sources(template_name, template_dirs=None): + """ + Returns the absolute paths to "template_name", when appended to each + directory in "template_dirs". Any paths that don't lie inside one of the + template dirs are excluded from the result set, for security reasons. + """ if not template_dirs: template_dirs = app_template_dirs for template_dir in template_dirs: try: yield safe_join(template_dir, template_name) + except UnicodeDecodeError: + # The template dir name was a bytestring that wasn't valid UTF-8. + raise except ValueError: # The joined path was located outside of template_dir. pass diff --git a/django/template/loaders/filesystem.py b/django/template/loaders/filesystem.py index 9997eb9a54..afee3c0519 100644 --- a/django/template/loaders/filesystem.py +++ b/django/template/loaders/filesystem.py @@ -7,13 +7,23 @@ from django.template import TemplateDoesNotExist from django.utils._os import safe_join def get_template_sources(template_name, template_dirs=None): + """ + Returns the absolute paths to "template_name", when appended to each + directory in "template_dirs". Any paths that don't lie inside one of the + template dirs are excluded from the result set, for security reasons. + """ if not template_dirs: template_dirs = settings.TEMPLATE_DIRS for template_dir in template_dirs: try: yield safe_join(template_dir, template_name) + except UnicodeDecodeError: + # The template dir name was a bytestring that wasn't valid UTF-8. + raise except ValueError: - # The joined path was located outside of template_dir. + # The joined path was located outside of this particular + # template_dir (it might be inside another one, so this isn't + # fatal). pass def load_template_source(template_name, template_dirs=None): diff --git a/django/utils/_os.py b/django/utils/_os.py index 30d7cbd444..39ba9f2112 100644 --- a/django/utils/_os.py +++ b/django/utils/_os.py @@ -1,4 +1,5 @@ from os.path import join, normcase, abspath, sep +from django.utils.encoding import force_unicode def safe_join(base, *paths): """ @@ -10,6 +11,8 @@ def safe_join(base, *paths): """ # We need to use normcase to ensure we don't false-negative on case # insensitive operating systems (like Windows). + base = force_unicode(base) + paths = [force_unicode(p) for p in paths] final_path = normcase(abspath(join(base, *paths))) base_path = normcase(abspath(base)) base_path_len = len(base_path) diff --git a/tests/regressiontests/templates/tests.py b/tests/regressiontests/templates/tests.py index c69e7cdd1b..cdf56ab56c 100644 --- a/tests/regressiontests/templates/tests.py +++ b/tests/regressiontests/templates/tests.py @@ -92,34 +92,45 @@ class UTF8Class: class Templates(unittest.TestCase): def test_loaders_security(self): def test_template_sources(path, template_dirs, expected_sources): - # Fix expected sources so they are normcased and abspathed - expected_sources = [os.path.normcase(os.path.abspath(s)) for s in expected_sources] - # Test app_directories loader - sources = app_directories.get_template_sources(path, template_dirs) - self.assertEqual(list(sources), expected_sources) - # Test filesystem loader - sources = filesystem.get_template_sources(path, template_dirs) - self.assertEqual(list(sources), expected_sources) + if isinstance(expected_sources, list): + # Fix expected sources so they are normcased and abspathed + expected_sources = [os.path.normcase(os.path.abspath(s)) for s in expected_sources] + # Test the two loaders (app_directores and filesystem). + func1 = lambda p, t: list(app_directories.get_template_sources(p, t)) + func2 = lambda p, t: list(filesystem.get_template_sources(p, t)) + for func in (func1, func2): + if isinstance(expected_sources, list): + self.assertEqual(func(path, template_dirs), expected_sources) + else: + self.assertRaises(expected_sources, func, path, template_dirs) template_dirs = ['/dir1', '/dir2'] test_template_sources('index.html', template_dirs, ['/dir1/index.html', '/dir2/index.html']) - test_template_sources('/etc/passwd', template_dirs, - []) + test_template_sources('/etc/passwd', template_dirs, []) test_template_sources('etc/passwd', template_dirs, ['/dir1/etc/passwd', '/dir2/etc/passwd']) - test_template_sources('../etc/passwd', template_dirs, - []) - test_template_sources('../../../etc/passwd', template_dirs, - []) + test_template_sources('../etc/passwd', template_dirs, []) + test_template_sources('../../../etc/passwd', template_dirs, []) test_template_sources('/dir1/index.html', template_dirs, ['/dir1/index.html']) test_template_sources('../dir2/index.html', template_dirs, ['/dir2/index.html']) - test_template_sources('/dir1blah', template_dirs, - []) - test_template_sources('../dir1blah', template_dirs, - []) + test_template_sources('/dir1blah', template_dirs, []) + test_template_sources('../dir1blah', template_dirs, []) + + # UTF-8 bytestrings are permitted. + test_template_sources('\xc3\x85ngstr\xc3\xb6m', template_dirs, + [u'/dir1/Ångström', u'/dir2/Ångström']) + # Unicode strings are permitted. + test_template_sources(u'Ångström', template_dirs, + [u'/dir1/Ångström', u'/dir2/Ångström']) + test_template_sources(u'Ångström', ['/Straße'], [u'/Straße/Ångström']) + test_template_sources('\xc3\x85ngstr\xc3\xb6m', ['/Straße'], + [u'/Straße/Ångström']) + # Invalid UTF-8 encoding in bytestrings is not. Should raise a + # semi-useful error message. + test_template_sources('\xc3\xc3', template_dirs, UnicodeDecodeError) # Case insensitive tests (for win32). Not run unless we're on # a case insensitive operating system. @@ -372,7 +383,7 @@ class Templates(unittest.TestCase): # Numbers as filter arguments should work 'filter-syntax19': ('{{ var|truncatewords:1 }}', {"var": "hello world"}, "hello ..."), - + #filters should accept empty string constants 'filter-syntax20': ('{{ ""|default_if_none:"was none" }}', {}, ""),