Fixed #24372 - Replaced TokenParser usage with traditional parsing.

django/template/base.py

@@ -431,122 +431,6 @@ class Parser(object):
             raise TemplateSyntaxError("Invalid filter: '%s'" % filter_name)
 
 
-class TokenParser(object):
-    """
-    Subclass this and implement the top() method to parse a template line.
-    When instantiating the parser, pass in the line from the Django template
-    parser.
-
-    The parser's "tagname" instance-variable stores the name of the tag that
-    the filter was called with.
-    """
-    def __init__(self, subject):
-        self.subject = subject
-        self.pointer = 0
-        self.backout = []
-        self.tagname = self.tag()
-
-    def top(self):
-        """
-        Overload this method to do the actual parsing and return the result.
-        """
-        raise NotImplementedError('subclasses of Tokenparser must provide a top() method')
-
-    def more(self):
-        """
-        Returns True if there is more stuff in the tag.
-        """
-        return self.pointer < len(self.subject)
-
-    def back(self):
-        """
-        Undoes the last microparser. Use this for lookahead and backtracking.
-        """
-        if not len(self.backout):
-            raise TemplateSyntaxError("back called without some previous "
-                                      "parsing")
-        self.pointer = self.backout.pop()
-
-    def tag(self):
-        """
-        A microparser that just returns the next tag from the line.
-        """
-        subject = self.subject
-        i = self.pointer
-        if i >= len(subject):
-            raise TemplateSyntaxError("expected another tag, found "
-                                      "end of string: %s" % subject)
-        p = i
-        while i < len(subject) and subject[i] not in (' ', '\t'):
-            i += 1
-        s = subject[p:i]
-        while i < len(subject) and subject[i] in (' ', '\t'):
-            i += 1
-        self.backout.append(self.pointer)
-        self.pointer = i
-        return s
-
-    def value(self):
-        """
-        A microparser that parses for a value: some string constant or
-        variable name.
-        """
-        subject = self.subject
-        i = self.pointer
-
-        def next_space_index(subject, i):
-            """
-            Increment pointer until a real space (i.e. a space not within
-            quotes) is encountered
-            """
-            while i < len(subject) and subject[i] not in (' ', '\t'):
-                if subject[i] in ('"', "'"):
-                    c = subject[i]
-                    i += 1
-                    while i < len(subject) and subject[i] != c:
-                        i += 1
-                    if i >= len(subject):
-                        raise TemplateSyntaxError("Searching for value. "
-                            "Unexpected end of string in column %d: %s" %
-                            (i, subject))
-                i += 1
-            return i
-
-        if i >= len(subject):
-            raise TemplateSyntaxError("Searching for value. Expected another "
-                                      "value but found end of string: %s" %
-                                      subject)
-        if subject[i] in ('"', "'"):
-            p = i
-            i += 1
-            while i < len(subject) and subject[i] != subject[p]:
-                i += 1
-            if i >= len(subject):
-                raise TemplateSyntaxError("Searching for value. Unexpected "
-                                          "end of string in column %d: %s" %
-                                          (i, subject))
-            i += 1
-
-            # Continue parsing until next "real" space,
-            # so that filters are also included
-            i = next_space_index(subject, i)
-
-            res = subject[p:i]
-            while i < len(subject) and subject[i] in (' ', '\t'):
-                i += 1
-            self.backout.append(self.pointer)
-            self.pointer = i
-            return res
-        else:
-            p = i
-            i = next_space_index(subject, i)
-            s = subject[p:i]
-            while i < len(subject) and subject[i] in (' ', '\t'):
-                i += 1
-            self.backout.append(self.pointer)
-            self.pointer = i
-            return s
-
 # This only matches constant *strings* (things in quotes or marked for
 # translation). Numbers are treated as variables for implementation reasons
 # (so that they retain their type when passed to filters).

django/templatetags/i18n.py

@@ -1,13 +1,10 @@
 from __future__ import unicode_literals
 
-import re
 import sys
 
 from django.conf import settings
 from django.template import Library, Node, TemplateSyntaxError, Variable
-from django.template.base import (
+from django.template.base import TOKEN_TEXT, TOKEN_VAR, render_value_in_context
-    TOKEN_TEXT, TOKEN_VAR, TokenParser, render_value_in_context,
-)
 from django.template.defaulttags import token_kwargs
 from django.utils import six, translation
 
@@ -348,42 +345,54 @@ def do_translate(parser, token):
 
     This is equivalent to calling pgettext instead of (u)gettext.
     """
-    class TranslateParser(TokenParser):
+    bits = token.split_contents()
-        def top(self):
+    if len(bits) < 2:
-            value = self.value()
+        raise TemplateSyntaxError("'%s' takes at least one argument" % bits[0])
-
+    message_string = parser.compile_filter(bits[1])
-            # Backwards Compatibility fix:
+    remaining = bits[2:]
-            # FilterExpression does not support single-quoted strings,
-            # so we make a cheap localized fix in order to maintain
-            # backwards compatibility with existing uses of ``trans``
-            # where single quote use is supported.
-            if value[0] == "'":
-                m = re.match("^'([^']+)'(\|.*$)", value)
-                if m:
-                    value = '"%s"%s' % (m.group(1).replace('"', '\\"'), m.group(2))
-                elif value[-1] == "'":
-                    value = '"%s"' % value[1:-1].replace('"', '\\"')
 
     noop = False
     asvar = None
     message_context = None
+    seen = set()
+    invalid_context = {'as', 'noop'}
 
-            while self.more():
+    while remaining:
-                tag = self.tag()
+        option = remaining.pop(0)
-                if tag == 'noop':
+        if option in seen:
+            raise TemplateSyntaxError(
+                "The '%s' option was specified more than once." % option,
+            )
+        elif option == 'noop':
             noop = True
-                elif tag == 'context':
+        elif option == 'context':
-                    message_context = parser.compile_filter(self.value())
+            try:
-                elif tag == 'as':
+                value = remaining.pop(0)
-                    asvar = self.tag()
+            except IndexError:
+                msg = "No argument provided to the '%s' tag for the context option." % bits[0]
+                six.reraise(TemplateSyntaxError, TemplateSyntaxError(msg), sys.exc_info()[2])
+            if value in invalid_context:
+                raise TemplateSyntaxError(
+                    "Invalid argument '%s' provided to the '%s' tag for the context option" % (value, bits[0]),
+                )
+            message_context = parser.compile_filter(value)
+        elif option == 'as':
+            try:
+                value = remaining.pop(0)
+            except IndexError:
+                msg = "No argument provided to the '%s' tag for the as option." % bits[0]
+                six.reraise(TemplateSyntaxError, TemplateSyntaxError(msg), sys.exc_info()[2])
+            asvar = value
         else:
             raise TemplateSyntaxError(
-                        "Only options for 'trans' are 'noop', "
+                "Unknown argument for '%s' tag: '%s'. The only options "
-                        "'context \"xxx\"', and 'as VAR'.")
+                "available are 'noop', 'context' \"xxx\", and 'as VAR'." % (
-            return value, noop, asvar, message_context
+                    bits[0], option,
-    value, noop, asvar, message_context = TranslateParser(token.contents).top()
+                )
-    return TranslateNode(parser.compile_filter(value), noop, asvar,
+            )
-                         message_context)
+        seen.add(option)
+
+    return TranslateNode(message_string, noop, asvar, message_context)
 
 
 @register.tag("blocktrans")

tests/i18n/tests.py

@@ -261,10 +261,6 @@ class TranslationTests(TestCase):
             rendered = t.render(Context())
             self.assertEqual(rendered, 'Value: Kann')
 
-            # Mis-uses
-            self.assertRaises(TemplateSyntaxError, Template, '{% load i18n %}{% trans "May" context as var %}{{ var }}')
-            self.assertRaises(TemplateSyntaxError, Template, '{% load i18n %}{% trans "May" as var context %}{{ var }}')
-
             # {% blocktrans %} ------------------------------
 
             # Inexisting context...

tests/template_tests/syntax_tests/test_i18n.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+from django.template import TemplateSyntaxError
 from django.test import SimpleTestCase
 from django.utils import translation
 from django.utils.safestring import mark_safe
@@ -412,3 +413,45 @@ class I18nTagTests(SimpleTestCase):
     def test_i18n38_2(self):
         output = self.engine.render_to_string('i18n38_2', {'langcodes': ['it', 'no']})
         self.assertEqual(output, 'it: Italian/italiano bidi=False; no: Norwegian/norsk bidi=False; ')
+
+    @setup({'template': '{% load i18n %}{% trans %}A}'})
+    def test_syntax_error_no_arguments(self):
+        msg = "'trans' takes at least one argument"
+        with self.assertRaisesMessage(TemplateSyntaxError, msg):
+            self.engine.render_to_string('template')
+
+    @setup({'template': '{% load i18n %}{% trans "Yes" badoption %}'})
+    def test_syntax_error_bad_option(self):
+        msg = "Unknown argument for 'trans' tag: 'badoption'"
+        with self.assertRaisesMessage(TemplateSyntaxError, msg):
+            self.engine.render_to_string('template')
+
+    @setup({'template': '{% load i18n %}{% trans "Yes" as %}'})
+    def test_syntax_error_missing_assignment(self):
+        msg = "No argument provided to the 'trans' tag for the as option."
+        with self.assertRaisesMessage(TemplateSyntaxError, msg):
+            self.engine.render_to_string('template')
+
+    @setup({'template': '{% load i18n %}{% trans "Yes" as var context %}'})
+    def test_syntax_error_missing_context(self):
+        msg = "No argument provided to the 'trans' tag for the context option."
+        with self.assertRaisesMessage(TemplateSyntaxError, msg):
+            self.engine.render_to_string('template')
+
+    @setup({'template': '{% load i18n %}{% trans "Yes" context as var %}'})
+    def test_syntax_error_context_as(self):
+        msg = "Invalid argument 'as' provided to the 'trans' tag for the context option"
+        with self.assertRaisesMessage(TemplateSyntaxError, msg):
+            self.engine.render_to_string('template')
+
+    @setup({'template': '{% load i18n %}{% trans "Yes" context noop %}'})
+    def test_syntax_error_context_noop(self):
+        msg = "Invalid argument 'noop' provided to the 'trans' tag for the context option"
+        with self.assertRaisesMessage(TemplateSyntaxError, msg):
+            self.engine.render_to_string('template')
+
+    @setup({'template': '{% load i18n %}{% trans "Yes" noop noop %}'})
+    def test_syntax_error_duplicate_option(self):
+        msg = "The 'noop' option was specified more than once."
+        with self.assertRaisesMessage(TemplateSyntaxError, msg):
+            self.engine.render_to_string('template')

tests/template_tests/test_parser.py

@@ -7,7 +7,7 @@ from unittest import TestCase
 
 from django.template import Library, Template, TemplateSyntaxError
 from django.template.base import (
-    TOKEN_BLOCK, FilterExpression, Parser, Token, TokenParser, Variable,
+    TOKEN_BLOCK, FilterExpression, Parser, Token, Variable,
 )
 from django.test import override_settings
 from django.utils import six
@@ -23,31 +23,6 @@ class ParserTests(TestCase):
         split = token.split_contents()
         self.assertEqual(split, ["sometag", '_("Page not found")', 'value|yesno:_("yes,no")'])
 
-    def test_token_parsing(self):
-        # Tests for TokenParser behavior in the face of quoted strings with
-        # spaces.
-
-        p = TokenParser("tag thevar|filter sometag")
-        self.assertEqual(p.tagname, "tag")
-        self.assertEqual(p.value(), "thevar|filter")
-        self.assertTrue(p.more())
-        self.assertEqual(p.tag(), "sometag")
-        self.assertFalse(p.more())
-
-        p = TokenParser('tag "a value"|filter sometag')
-        self.assertEqual(p.tagname, "tag")
-        self.assertEqual(p.value(), '"a value"|filter')
-        self.assertTrue(p.more())
-        self.assertEqual(p.tag(), "sometag")
-        self.assertFalse(p.more())
-
-        p = TokenParser("tag 'a value'|filter sometag")
-        self.assertEqual(p.tagname, "tag")
-        self.assertEqual(p.value(), "'a value'|filter")
-        self.assertTrue(p.more())
-        self.assertEqual(p.tag(), "sometag")
-        self.assertFalse(p.more())
-
     def test_filter_parsing(self):
         c = {"article": {"section": "News"}}
         p = Parser("")