From a6f429e37e7fe9cf8801c8fbce4c40af7e0cec0c Mon Sep 17 00:00:00 2001 From: Malcolm Tredinnick Date: Mon, 23 Mar 2009 09:40:25 +0000 Subject: [PATCH] Added consistent support for double- and single-quote delimiters in templates. Some template filters and tags understood single-quoted arguments, others didn't. This makes everything consistent. Based on a patch from akaihola. Fixed #7295. git-svn-id: http://code.djangoproject.com/svn/django/trunk@10118 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- django/template/__init__.py | 113 ++++++++++++---------- django/utils/text.py | 40 +++++--- tests/regressiontests/templates/parser.py | 59 +++++++++++ tests/regressiontests/templates/tests.py | 4 +- tests/regressiontests/text/tests.py | 2 +- 5 files changed, 152 insertions(+), 66 deletions(-) create mode 100644 tests/regressiontests/templates/parser.py diff --git a/django/template/__init__.py b/django/template/__init__.py index d008b7f874..c5b8ba0b8f 100644 --- a/django/template/__init__.py +++ b/django/template/__init__.py @@ -50,12 +50,13 @@ u'' """ import re from inspect import getargspec + from django.conf import settings from django.template.context import Context, RequestContext, ContextPopException from django.utils.importlib import import_module from django.utils.itercompat import is_iterable from django.utils.functional import curry, Promise -from django.utils.text import smart_split +from django.utils.text import smart_split, unescape_string_literal from django.utils.encoding import smart_unicode, force_unicode, smart_str from django.utils.translation import ugettext as _ from django.utils.safestring import SafeData, EscapeData, mark_safe, mark_for_escaping @@ -444,33 +445,44 @@ class TokenParser(object): self.pointer = i return s +constant_string = r""" +(?:%(i18n_open)s%(strdq)s%(i18n_close)s| +%(i18n_open)s%(strsq)s%(i18n_close)s| +%(strdq)s| +%(strsq)s)| +%(num)s +""" % { + 'strdq': r'"[^"\\]*(?:\\.[^"\\]*)*"', # double-quoted string + 'strsq': r"'[^'\\]*(?:\\.[^'\\]*)*'", # single-quoted string + 'num': r'[-+\.]?\d[\d\.e]*', # numeric constant + 'i18n_open' : re.escape("_("), + 'i18n_close' : re.escape(")"), + } +constant_string = constant_string.replace("\n", "") + filter_raw_string = r""" -^%(i18n_open)s"(?P%(str)s)"%(i18n_close)s| -^"(?P%(str)s)"| +^(?P%(constant)s)| ^(?P[%(var_chars)s]+)| (?:%(filter_sep)s (?P\w+) (?:%(arg_sep)s (?: - %(i18n_open)s"(?P%(str)s)"%(i18n_close)s| - "(?P%(str)s)"| + (?P%(constant)s)| (?P[%(var_chars)s]+) ) )? )""" % { - 'str': r"""[^"\\]*(?:\\.[^"\\]*)*""", + 'constant': constant_string, 'var_chars': "\w\." , 'filter_sep': re.escape(FILTER_SEPARATOR), 'arg_sep': re.escape(FILTER_ARGUMENT_SEPARATOR), - 'i18n_open' : re.escape("_("), - 'i18n_close' : re.escape(")"), } filter_raw_string = filter_raw_string.replace("\n", "").replace(" ", "") filter_re = re.compile(filter_raw_string, re.UNICODE) class FilterExpression(object): - """ + r""" Parses a variable token and its optional filters (all as a single string), and return a list of tuples of the filter name and arguments. Sample: @@ -488,65 +500,64 @@ class FilterExpression(object): def __init__(self, token, parser): self.token = token matches = filter_re.finditer(token) - var = None + var_obj = None filters = [] upto = 0 for match in matches: start = match.start() if upto != start: raise TemplateSyntaxError("Could not parse some characters: %s|%s|%s" % \ - (token[:upto], token[upto:start], token[start:])) - if var == None: - var, constant, i18n_constant = match.group("var", "constant", "i18n_constant") - if i18n_constant is not None: - # Don't pass the empty string to gettext, because the empty - # string translates to meta information. - if i18n_constant == "": - var = '""' - else: - var = '"%s"' % _(i18n_constant.replace(r'\"', '"')) - elif constant is not None: - var = '"%s"' % constant.replace(r'\"', '"') - upto = match.end() - if var == None: - raise TemplateSyntaxError("Could not find variable at start of %s" % token) + (token[:upto], token[upto:start], token[start:])) + if var_obj is None: + var, constant = match.group("var", "constant") + if constant: + try: + var_obj = Variable(constant).resolve({}) + except VariableDoesNotExist: + var_obj = None + elif var is None: + raise TemplateSyntaxError("Could not find variable at start of %s." % token) elif var.find(VARIABLE_ATTRIBUTE_SEPARATOR + '_') > -1 or var[0] == '_': raise TemplateSyntaxError("Variables and attributes may not begin with underscores: '%s'" % var) + else: + var_obj = Variable(var) else: filter_name = match.group("filter_name") args = [] - constant_arg, i18n_arg, var_arg = match.group("constant_arg", "i18n_arg", "var_arg") - if i18n_arg: - args.append((False, _(i18n_arg.replace(r'\"', '"')))) - elif constant_arg is not None: - args.append((False, constant_arg.replace(r'\"', '"'))) + constant_arg, var_arg = match.group("constant_arg", "var_arg") + if constant_arg: + args.append((False, Variable(constant_arg).resolve({}))) elif var_arg: args.append((True, Variable(var_arg))) filter_func = parser.find_filter(filter_name) self.args_check(filter_name,filter_func, args) filters.append( (filter_func,args)) - upto = match.end() + upto = match.end() if upto != len(token): raise TemplateSyntaxError("Could not parse the remainder: '%s' from '%s'" % (token[upto:], token)) + self.filters = filters - self.var = Variable(var) + self.var = var_obj def resolve(self, context, ignore_failures=False): - try: - obj = self.var.resolve(context) - except VariableDoesNotExist: - if ignore_failures: - obj = None - else: - if settings.TEMPLATE_STRING_IF_INVALID: - global invalid_var_format_string - if invalid_var_format_string is None: - invalid_var_format_string = '%s' in settings.TEMPLATE_STRING_IF_INVALID - if invalid_var_format_string: - return settings.TEMPLATE_STRING_IF_INVALID % self.var - return settings.TEMPLATE_STRING_IF_INVALID + if isinstance(self.var, Variable): + try: + obj = self.var.resolve(context) + except VariableDoesNotExist: + if ignore_failures: + obj = None else: - obj = settings.TEMPLATE_STRING_IF_INVALID + if settings.TEMPLATE_STRING_IF_INVALID: + global invalid_var_format_string + if invalid_var_format_string is None: + invalid_var_format_string = '%s' in settings.TEMPLATE_STRING_IF_INVALID + if invalid_var_format_string: + return settings.TEMPLATE_STRING_IF_INVALID % self.var + return settings.TEMPLATE_STRING_IF_INVALID + else: + obj = settings.TEMPLATE_STRING_IF_INVALID + else: + obj = self.var for func, args in self.filters: arg_vals = [] for lookup, arg in args: @@ -611,7 +622,7 @@ def resolve_variable(path, context): return Variable(path).resolve(context) class Variable(object): - """ + r""" A template variable, resolvable against a given context. The variable may be a hard-coded string (if it begins and ends with single or double quote marks):: @@ -625,8 +636,6 @@ class Variable(object): >>> c = AClass() >>> c.article = AClass() >>> c.article.section = u'News' - >>> Variable('article.section').resolve(c) - u'News' (The example assumes VARIABLE_ATTRIBUTE_SEPARATOR is '.') """ @@ -663,9 +672,9 @@ class Variable(object): var = var[2:-1] # If it's wrapped with quotes (single or double), then # we're also dealing with a literal. - if var[0] in "\"'" and var[0] == var[-1]: - self.literal = mark_safe(var[1:-1]) - else: + try: + self.literal = mark_safe(unescape_string_literal(var)) + except ValueError: # Otherwise we'll set self.lookups so that resolve() knows we're # dealing with a bonafide variable self.lookups = tuple(var.split(VARIABLE_ATTRIBUTE_SEPARATOR)) diff --git a/django/utils/text.py b/django/utils/text.py index 1548cfa77e..c90065265a 100644 --- a/django/utils/text.py +++ b/django/utils/text.py @@ -203,24 +203,19 @@ def smart_split(text): Generator that splits a string by spaces, leaving quoted phrases together. Supports both single and double quotes, and supports escaping quotes with backslashes. In the output, strings will keep their initial and trailing - quote marks. + quote marks and escaped quotes will remain escaped (the results can then + be further processed with unescape_string_literal()). >>> list(smart_split(r'This is "a person\'s" test.')) [u'This', u'is', u'"a person\\\'s"', u'test.'] - >>> list(smart_split(r"Another 'person\'s' test.")) - [u'Another', u"'person's'", u'test.'] - >>> list(smart_split(r'A "\"funky\" style" test.')) - [u'A', u'""funky" style"', u'test.'] + >>> list(smart_split(r"Another 'person\'s' test.")) + [u'Another', u"'person\\'s'", u'test.'] + >>> list(smart_split(r'A "\"funky\" style" test.')) + [u'A', u'"\\"funky\\" style"', u'test.'] """ text = force_unicode(text) for bit in smart_split_re.finditer(text): - bit = bit.group(0) - if bit[0] == '"' and bit[-1] == '"': - yield '"' + bit[1:-1].replace('\\"', '"').replace('\\\\', '\\') + '"' - elif bit[0] == "'" and bit[-1] == "'": - yield "'" + bit[1:-1].replace("\\'", "'").replace("\\\\", "\\") + "'" - else: - yield bit + yield bit.group(0) smart_split = allow_lazy(smart_split, unicode) def _replace_entity(match): @@ -246,3 +241,24 @@ _entity_re = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));") def unescape_entities(text): return _entity_re.sub(_replace_entity, text) unescape_entities = allow_lazy(unescape_entities, unicode) + +def unescape_string_literal(s): + r""" + Convert quoted string literals to unquoted strings with escaped quotes and + backslashes unquoted:: + + >>> unescape_string_literal('"abc"') + 'abc' + >>> unescape_string_literal("'abc'") + 'abc' + >>> unescape_string_literal('"a \"bc\""') + 'a "bc"' + >>> unescape_string_literal("'\'ab\' c'") + "'ab' c" + """ + if s[0] not in "\"'" or s[-1] != s[0]: + raise ValueError("Not a string literal: %r" % s) + quote = s[0] + return s[1:-1].replace(r'\%s' % quote, quote).replace(r'\\', '\\') +unescape_string_literal = allow_lazy(unescape_string_literal) + diff --git a/tests/regressiontests/templates/parser.py b/tests/regressiontests/templates/parser.py new file mode 100644 index 0000000000..6ad301d9c3 --- /dev/null +++ b/tests/regressiontests/templates/parser.py @@ -0,0 +1,59 @@ +""" +Testing some internals of the template processing. These are *not* examples to be copied in user code. +""" + +filter_parsing = r""" +>>> from django.template import FilterExpression, Parser + +>>> c = {'article': {'section': u'News'}} +>>> p = Parser("") +>>> def fe_test(s): return FilterExpression(s, p).resolve(c) + +>>> fe_test('article.section') +u'News' +>>> fe_test('article.section|upper') +u'NEWS' +>>> fe_test(u'"News"') +u'News' +>>> fe_test(u"'News'") +u'News' +>>> fe_test(ur'"Some \"Good\" News"') +u'Some "Good" News' +>>> fe_test(ur"'Some \'Bad\' News'") +u"Some 'Bad' News" + +>>> fe = FilterExpression(ur'"Some \"Good\" News"', p) +>>> fe.filters +[] +>>> fe.var +u'Some "Good" News' +""" + +variable_parsing = r""" +>>> from django.template import Variable + +>>> c = {'article': {'section': u'News'}} +>>> Variable('article.section').resolve(c) +u'News' +>>> Variable(u'"News"').resolve(c) +u'News' +>>> Variable(u"'News'").resolve(c) +u'News' + +Translated strings are handled correctly. + +>>> Variable('_(article.section)').resolve(c) +u'News' +>>> Variable('_("Good News")').resolve(c) +u'Good News' +>>> Variable("_('Better News')").resolve(c) +u'Better News' + +Escaped quotes work correctly as well. + +>>> Variable(ur'"Some \"Good\" News"').resolve(c) +u'Some "Good" News' +>>> Variable(ur"'Some \'Better\' News'").resolve(c) +u"Some 'Better' News" + +""" diff --git a/tests/regressiontests/templates/tests.py b/tests/regressiontests/templates/tests.py index a68f1e8f67..7bb37b9fd0 100644 --- a/tests/regressiontests/templates/tests.py +++ b/tests/regressiontests/templates/tests.py @@ -20,6 +20,7 @@ from django.utils.tzinfo import LocalTimezone from unicode import unicode_tests from context import context_tests +from parser import filter_parsing, variable_parsing try: from loaders import * @@ -31,7 +32,8 @@ import filters # Some other tests we would like to run __test__ = { 'unicode': unicode_tests, - 'context': context_tests + 'context': context_tests, + 'filter_parsing': filter_parsing, } ################################# diff --git a/tests/regressiontests/text/tests.py b/tests/regressiontests/text/tests.py index 7cfe44517a..75c360f344 100644 --- a/tests/regressiontests/text/tests.py +++ b/tests/regressiontests/text/tests.py @@ -10,7 +10,7 @@ r""" >>> print list(smart_split(r'''This is "a person's" test.'''))[2] "a person's" >>> print list(smart_split(r'''This is "a person\"s" test.'''))[2] -"a person"s" +"a person\"s" >>> list(smart_split('''"a 'one''')) [u'"a', u"'one"] >>> print list(smart_split(r'''all friends' tests'''))[1]