From 358850781f8fd3d79aba5b1a9a0b8d28f544bf8a Mon Sep 17 00:00:00 2001 From: Preston Timmons Date: Thu, 19 Feb 2015 14:04:25 -0600 Subject: [PATCH] Fixed #24372 - Replaced TokenParser usage with traditional parsing. --- django/template/base.py | 116 ------------------ django/templatetags/i18n.py | 83 +++++++------ tests/i18n/tests.py | 4 - .../template_tests/syntax_tests/test_i18n.py | 43 +++++++ tests/template_tests/test_parser.py | 27 +--- 5 files changed, 90 insertions(+), 183 deletions(-) diff --git a/django/template/base.py b/django/template/base.py index d89021d7c0..25a42531fd 100644 --- a/django/template/base.py +++ b/django/template/base.py @@ -431,122 +431,6 @@ class Parser(object): raise TemplateSyntaxError("Invalid filter: '%s'" % filter_name) -class TokenParser(object): - """ - Subclass this and implement the top() method to parse a template line. - When instantiating the parser, pass in the line from the Django template - parser. - - The parser's "tagname" instance-variable stores the name of the tag that - the filter was called with. - """ - def __init__(self, subject): - self.subject = subject - self.pointer = 0 - self.backout = [] - self.tagname = self.tag() - - def top(self): - """ - Overload this method to do the actual parsing and return the result. - """ - raise NotImplementedError('subclasses of Tokenparser must provide a top() method') - - def more(self): - """ - Returns True if there is more stuff in the tag. - """ - return self.pointer < len(self.subject) - - def back(self): - """ - Undoes the last microparser. Use this for lookahead and backtracking. - """ - if not len(self.backout): - raise TemplateSyntaxError("back called without some previous " - "parsing") - self.pointer = self.backout.pop() - - def tag(self): - """ - A microparser that just returns the next tag from the line. - """ - subject = self.subject - i = self.pointer - if i >= len(subject): - raise TemplateSyntaxError("expected another tag, found " - "end of string: %s" % subject) - p = i - while i < len(subject) and subject[i] not in (' ', '\t'): - i += 1 - s = subject[p:i] - while i < len(subject) and subject[i] in (' ', '\t'): - i += 1 - self.backout.append(self.pointer) - self.pointer = i - return s - - def value(self): - """ - A microparser that parses for a value: some string constant or - variable name. - """ - subject = self.subject - i = self.pointer - - def next_space_index(subject, i): - """ - Increment pointer until a real space (i.e. a space not within - quotes) is encountered - """ - while i < len(subject) and subject[i] not in (' ', '\t'): - if subject[i] in ('"', "'"): - c = subject[i] - i += 1 - while i < len(subject) and subject[i] != c: - i += 1 - if i >= len(subject): - raise TemplateSyntaxError("Searching for value. " - "Unexpected end of string in column %d: %s" % - (i, subject)) - i += 1 - return i - - if i >= len(subject): - raise TemplateSyntaxError("Searching for value. Expected another " - "value but found end of string: %s" % - subject) - if subject[i] in ('"', "'"): - p = i - i += 1 - while i < len(subject) and subject[i] != subject[p]: - i += 1 - if i >= len(subject): - raise TemplateSyntaxError("Searching for value. Unexpected " - "end of string in column %d: %s" % - (i, subject)) - i += 1 - - # Continue parsing until next "real" space, - # so that filters are also included - i = next_space_index(subject, i) - - res = subject[p:i] - while i < len(subject) and subject[i] in (' ', '\t'): - i += 1 - self.backout.append(self.pointer) - self.pointer = i - return res - else: - p = i - i = next_space_index(subject, i) - s = subject[p:i] - while i < len(subject) and subject[i] in (' ', '\t'): - i += 1 - self.backout.append(self.pointer) - self.pointer = i - return s - # This only matches constant *strings* (things in quotes or marked for # translation). Numbers are treated as variables for implementation reasons # (so that they retain their type when passed to filters). diff --git a/django/templatetags/i18n.py b/django/templatetags/i18n.py index bb81c56fb9..28d93f5ea6 100644 --- a/django/templatetags/i18n.py +++ b/django/templatetags/i18n.py @@ -1,13 +1,10 @@ from __future__ import unicode_literals -import re import sys from django.conf import settings from django.template import Library, Node, TemplateSyntaxError, Variable -from django.template.base import ( - TOKEN_TEXT, TOKEN_VAR, TokenParser, render_value_in_context, -) +from django.template.base import TOKEN_TEXT, TOKEN_VAR, render_value_in_context from django.template.defaulttags import token_kwargs from django.utils import six, translation @@ -348,42 +345,54 @@ def do_translate(parser, token): This is equivalent to calling pgettext instead of (u)gettext. """ - class TranslateParser(TokenParser): - def top(self): - value = self.value() + bits = token.split_contents() + if len(bits) < 2: + raise TemplateSyntaxError("'%s' takes at least one argument" % bits[0]) + message_string = parser.compile_filter(bits[1]) + remaining = bits[2:] - # Backwards Compatibility fix: - # FilterExpression does not support single-quoted strings, - # so we make a cheap localized fix in order to maintain - # backwards compatibility with existing uses of ``trans`` - # where single quote use is supported. - if value[0] == "'": - m = re.match("^'([^']+)'(\|.*$)", value) - if m: - value = '"%s"%s' % (m.group(1).replace('"', '\\"'), m.group(2)) - elif value[-1] == "'": - value = '"%s"' % value[1:-1].replace('"', '\\"') + noop = False + asvar = None + message_context = None + seen = set() + invalid_context = {'as', 'noop'} - noop = False - asvar = None - message_context = None + while remaining: + option = remaining.pop(0) + if option in seen: + raise TemplateSyntaxError( + "The '%s' option was specified more than once." % option, + ) + elif option == 'noop': + noop = True + elif option == 'context': + try: + value = remaining.pop(0) + except IndexError: + msg = "No argument provided to the '%s' tag for the context option." % bits[0] + six.reraise(TemplateSyntaxError, TemplateSyntaxError(msg), sys.exc_info()[2]) + if value in invalid_context: + raise TemplateSyntaxError( + "Invalid argument '%s' provided to the '%s' tag for the context option" % (value, bits[0]), + ) + message_context = parser.compile_filter(value) + elif option == 'as': + try: + value = remaining.pop(0) + except IndexError: + msg = "No argument provided to the '%s' tag for the as option." % bits[0] + six.reraise(TemplateSyntaxError, TemplateSyntaxError(msg), sys.exc_info()[2]) + asvar = value + else: + raise TemplateSyntaxError( + "Unknown argument for '%s' tag: '%s'. The only options " + "available are 'noop', 'context' \"xxx\", and 'as VAR'." % ( + bits[0], option, + ) + ) + seen.add(option) - while self.more(): - tag = self.tag() - if tag == 'noop': - noop = True - elif tag == 'context': - message_context = parser.compile_filter(self.value()) - elif tag == 'as': - asvar = self.tag() - else: - raise TemplateSyntaxError( - "Only options for 'trans' are 'noop', " - "'context \"xxx\"', and 'as VAR'.") - return value, noop, asvar, message_context - value, noop, asvar, message_context = TranslateParser(token.contents).top() - return TranslateNode(parser.compile_filter(value), noop, asvar, - message_context) + return TranslateNode(message_string, noop, asvar, message_context) @register.tag("blocktrans") diff --git a/tests/i18n/tests.py b/tests/i18n/tests.py index 9c5b9fddab..ad23861aea 100644 --- a/tests/i18n/tests.py +++ b/tests/i18n/tests.py @@ -261,10 +261,6 @@ class TranslationTests(TestCase): rendered = t.render(Context()) self.assertEqual(rendered, 'Value: Kann') - # Mis-uses - self.assertRaises(TemplateSyntaxError, Template, '{% load i18n %}{% trans "May" context as var %}{{ var }}') - self.assertRaises(TemplateSyntaxError, Template, '{% load i18n %}{% trans "May" as var context %}{{ var }}') - # {% blocktrans %} ------------------------------ # Inexisting context... diff --git a/tests/template_tests/syntax_tests/test_i18n.py b/tests/template_tests/syntax_tests/test_i18n.py index 11ca002440..da53a8c279 100644 --- a/tests/template_tests/syntax_tests/test_i18n.py +++ b/tests/template_tests/syntax_tests/test_i18n.py @@ -1,6 +1,7 @@ # coding: utf-8 from __future__ import unicode_literals +from django.template import TemplateSyntaxError from django.test import SimpleTestCase from django.utils import translation from django.utils.safestring import mark_safe @@ -412,3 +413,45 @@ class I18nTagTests(SimpleTestCase): def test_i18n38_2(self): output = self.engine.render_to_string('i18n38_2', {'langcodes': ['it', 'no']}) self.assertEqual(output, 'it: Italian/italiano bidi=False; no: Norwegian/norsk bidi=False; ') + + @setup({'template': '{% load i18n %}{% trans %}A}'}) + def test_syntax_error_no_arguments(self): + msg = "'trans' takes at least one argument" + with self.assertRaisesMessage(TemplateSyntaxError, msg): + self.engine.render_to_string('template') + + @setup({'template': '{% load i18n %}{% trans "Yes" badoption %}'}) + def test_syntax_error_bad_option(self): + msg = "Unknown argument for 'trans' tag: 'badoption'" + with self.assertRaisesMessage(TemplateSyntaxError, msg): + self.engine.render_to_string('template') + + @setup({'template': '{% load i18n %}{% trans "Yes" as %}'}) + def test_syntax_error_missing_assignment(self): + msg = "No argument provided to the 'trans' tag for the as option." + with self.assertRaisesMessage(TemplateSyntaxError, msg): + self.engine.render_to_string('template') + + @setup({'template': '{% load i18n %}{% trans "Yes" as var context %}'}) + def test_syntax_error_missing_context(self): + msg = "No argument provided to the 'trans' tag for the context option." + with self.assertRaisesMessage(TemplateSyntaxError, msg): + self.engine.render_to_string('template') + + @setup({'template': '{% load i18n %}{% trans "Yes" context as var %}'}) + def test_syntax_error_context_as(self): + msg = "Invalid argument 'as' provided to the 'trans' tag for the context option" + with self.assertRaisesMessage(TemplateSyntaxError, msg): + self.engine.render_to_string('template') + + @setup({'template': '{% load i18n %}{% trans "Yes" context noop %}'}) + def test_syntax_error_context_noop(self): + msg = "Invalid argument 'noop' provided to the 'trans' tag for the context option" + with self.assertRaisesMessage(TemplateSyntaxError, msg): + self.engine.render_to_string('template') + + @setup({'template': '{% load i18n %}{% trans "Yes" noop noop %}'}) + def test_syntax_error_duplicate_option(self): + msg = "The 'noop' option was specified more than once." + with self.assertRaisesMessage(TemplateSyntaxError, msg): + self.engine.render_to_string('template') diff --git a/tests/template_tests/test_parser.py b/tests/template_tests/test_parser.py index 0c5198fb64..a88dec285e 100644 --- a/tests/template_tests/test_parser.py +++ b/tests/template_tests/test_parser.py @@ -7,7 +7,7 @@ from unittest import TestCase from django.template import Library, Template, TemplateSyntaxError from django.template.base import ( - TOKEN_BLOCK, FilterExpression, Parser, Token, TokenParser, Variable, + TOKEN_BLOCK, FilterExpression, Parser, Token, Variable, ) from django.test import override_settings from django.utils import six @@ -23,31 +23,6 @@ class ParserTests(TestCase): split = token.split_contents() self.assertEqual(split, ["sometag", '_("Page not found")', 'value|yesno:_("yes,no")']) - def test_token_parsing(self): - # Tests for TokenParser behavior in the face of quoted strings with - # spaces. - - p = TokenParser("tag thevar|filter sometag") - self.assertEqual(p.tagname, "tag") - self.assertEqual(p.value(), "thevar|filter") - self.assertTrue(p.more()) - self.assertEqual(p.tag(), "sometag") - self.assertFalse(p.more()) - - p = TokenParser('tag "a value"|filter sometag') - self.assertEqual(p.tagname, "tag") - self.assertEqual(p.value(), '"a value"|filter') - self.assertTrue(p.more()) - self.assertEqual(p.tag(), "sometag") - self.assertFalse(p.more()) - - p = TokenParser("tag 'a value'|filter sometag") - self.assertEqual(p.tagname, "tag") - self.assertEqual(p.value(), "'a value'|filter") - self.assertTrue(p.more()) - self.assertEqual(p.tag(), "sometag") - self.assertFalse(p.more()) - def test_filter_parsing(self): c = {"article": {"section": "News"}} p = Parser("")