p31829507
/
django
mirror of https://github.com/django/django.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Replaced TOKEN_* constants by TokenType enums. 

Thanks Tim Graham for the review.
This commit is contained in:
Claude Paroz 2018-05-10 17:51:51 +02:00 committed by GitHub
parent 1e20fedb35
commit 9c4ea63e87
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 39 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
django/template/base.py
View File

@ -3,9 +3,10 @@ This is the Django template system.
How it works:
The Lexer.tokenize() function converts a template string (i.e., a string containing
markup with custom template tags) to tokens, which can be either plain text
(TOKEN_TEXT), variables (TOKEN_VAR) or block statements (TOKEN_BLOCK).
The Lexer.tokenize() method converts a template string (i.e., a string
containing markup with custom template tags) to tokens, which can be either
plain text (TokenType.TEXT), variables (TokenType.VAR), or block statements
(TokenType.BLOCK).
The Parser() class takes a list of tokens in its constructor, and its parse()
method returns a compiled template -- which is, under the hood, a list of
@ -51,6 +52,7 @@ times with multiple contexts)
import logging
import re
from enum import Enum
from inspect import getcallargs, getfullargspec
from django.template.context import ( # NOQA: imported for backwards compatibility
@ -67,17 +69,6 @@ from django.utils.translation import gettext_lazy, pgettext_lazy
from .exceptions import TemplateSyntaxError
TOKEN_TEXT = 0
TOKEN_VAR = 1
TOKEN_BLOCK = 2
TOKEN_COMMENT = 3
TOKEN_MAPPING = {
TOKEN_TEXT: 'Text',
TOKEN_VAR: 'Var',
TOKEN_BLOCK: 'Block',
TOKEN_COMMENT: 'Comment',
}
# template syntax constants
FILTER_SEPARATOR = '|'
FILTER_ARGUMENT_SEPARATOR = ':'
@ -106,6 +97,13 @@ tag_re = (re.compile('(%s.*?%s|%s.*?%s|%s.*?%s)' %
logger = logging.getLogger('django.template')
class TokenType(Enum):
TEXT = 0
VAR = 1
BLOCK = 2
COMMENT = 3
class VariableDoesNotExist(Exception):
def __init__(self, msg, params=()):
@ -293,7 +291,7 @@ class Token:
A token representing a string from the template.
token_type
One of TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK, or TOKEN_COMMENT.
A TokenType, either .TEXT, .VAR, .BLOCK, or .COMMENT.
contents
The token source string.
@ -312,7 +310,7 @@ class Token:
self.position = position
def __str__(self):
token_name = TOKEN_MAPPING[self.token_type]
token_name = self.token_type.name.capitalize()
return ('<%s token: "%s...">' %
(token_name, self.contents[:20].replace('\n', '')))
@ -367,18 +365,18 @@ class Lexer:
self.verbatim = False
if in_tag and not self.verbatim:
if token_string.startswith(VARIABLE_TAG_START):
return Token(TOKEN_VAR, token_string[2:-2].strip(), position, lineno)
return Token(TokenType.VAR, token_string[2:-2].strip(), position, lineno)
elif token_string.startswith(BLOCK_TAG_START):
if block_content[:9] in ('verbatim', 'verbatim '):
self.verbatim = 'end%s' % block_content
return Token(TOKEN_BLOCK, block_content, position, lineno)
return Token(TokenType.BLOCK, block_content, position, lineno)
elif token_string.startswith(COMMENT_TAG_START):
content = ''
if token_string.find(TRANSLATOR_COMMENT_MARK):
content = token_string[2:-2].strip()
return Token(TOKEN_COMMENT, content, position, lineno)
return Token(TokenType.COMMENT, content, position, lineno)
else:
return Token(TOKEN_TEXT, token_string, position, lineno)
return Token(TokenType.TEXT, token_string, position, lineno)
class DebugLexer(Lexer):
@ -439,10 +437,10 @@ class Parser:
nodelist = NodeList()
while self.tokens:
token = self.next_token()
# Use the raw values here for TOKEN_* for a tiny performance boost.
if token.token_type == 0: # TOKEN_TEXT
# Use the raw values here for TokenType.* for a tiny performance boost.
if token.token_type.value == 0: # TokenType.TEXT
self.extend_nodelist(nodelist, TextNode(token.contents), token)
elif token.token_type == 1: # TOKEN_VAR
elif token.token_type.value == 1: # TokenType.VAR
if not token.contents:
raise self.error(token, 'Empty variable tag on line %d' % token.lineno)
try:
@ -451,7 +449,7 @@ class Parser:
raise self.error(token, e)
var_node = VariableNode(filter_expression)
self.extend_nodelist(nodelist, var_node, token)
elif token.token_type == 2: # TOKEN_BLOCK
elif token.token_type.value == 2: # TokenType.BLOCK
try:
command = token.contents.split()[0]
except IndexError:
@ -488,7 +486,7 @@ class Parser:
def skip_past(self, endtag):
while self.tokens:
token = self.next_token()
if token.token_type == TOKEN_BLOCK and token.contents == endtag:
if token.token_type == TokenType.BLOCK and token.contents == endtag:
return
self.unclosed_block_tag([endtag])

10
django/templatetags/i18n.py
View File

@ -1,6 +1,6 @@
from django.conf import settings
from django.template import Library, Node, TemplateSyntaxError, Variable
from django.template.base import TOKEN_TEXT, TOKEN_VAR, render_value_in_context
from django.template.base import TokenType, render_value_in_context
from django.template.defaulttags import token_kwargs
from django.utils import translation
from django.utils.safestring import SafeData, mark_safe
@ -112,9 +112,9 @@ class BlockTranslateNode(Node):
result = []
vars = []
for token in tokens:
if token.token_type == TOKEN_TEXT:
if token.token_type == TokenType.TEXT:
result.append(token.contents.replace('%', '%%'))
elif token.token_type == TOKEN_VAR:
elif token.token_type == TokenType.VAR:
result.append('%%(%s)s' % token.contents)
vars.append(token.contents)
msg = ''.join(result)
@ -510,7 +510,7 @@ def do_block_translate(parser, token):
plural = []
while parser.tokens:
token = parser.next_token()
if token.token_type in (TOKEN_VAR, TOKEN_TEXT):
if token.token_type in (TokenType.VAR, TokenType.TEXT):
singular.append(token)
else:
break
@ -519,7 +519,7 @@ def do_block_translate(parser, token):
raise TemplateSyntaxError("'blocktrans' doesn't allow other block tags inside it")
while parser.tokens:
token = parser.next_token()
if token.token_type in (TOKEN_VAR, TOKEN_TEXT):
if token.token_type in (TokenType.VAR, TokenType.TEXT):
plural.append(token)
else:
break

21
django/utils/translation/template.py
View File

@ -2,10 +2,7 @@ import re
import warnings
from io import StringIO
from django.template.base import (
TOKEN_BLOCK, TOKEN_COMMENT, TOKEN_TEXT, TOKEN_VAR, TRANSLATOR_COMMENT_MARK,
Lexer,
)
from django.template.base import TRANSLATOR_COMMENT_MARK, Lexer, TokenType
from . import TranslatorCommentWarning, trim_whitespace
@ -63,7 +60,7 @@ def templatize(src, origin=None):
for t in Lexer(src).tokenize():
if incomment:
if t.token_type == TOKEN_BLOCK and t.contents == 'endcomment':
if t.token_type == TokenType.BLOCK and t.contents == 'endcomment':
content = ''.join(comment)
translators_comment_start = None
for lineno, line in enumerate(content.splitlines(True)):
@ -79,7 +76,7 @@ def templatize(src, origin=None):
else:
comment.append(t.contents)
elif intrans:
if t.token_type == TOKEN_BLOCK:
if t.token_type == TokenType.BLOCK:
endbmatch = endblock_re.match(t.contents)
pluralmatch = plural_re.match(t.contents)
if endbmatch:
@ -130,12 +127,12 @@ def templatize(src, origin=None):
"Translation blocks must not include other block tags: "
"%s (%sline %d)" % (t.contents, filemsg, t.lineno)
)
elif t.token_type == TOKEN_VAR:
elif t.token_type == TokenType.VAR:
if inplural:
plural.append('%%(%s)s' % t.contents)
else:
singular.append('%%(%s)s' % t.contents)
elif t.token_type == TOKEN_TEXT:
elif t.token_type == TokenType.TEXT:
contents = t.contents.replace('%', '%%')
if inplural:
plural.append(contents)
@ -147,7 +144,7 @@ def templatize(src, origin=None):
if comment_lineno_cache is not None:
cur_lineno = t.lineno + t.contents.count('\n')
if comment_lineno_cache == cur_lineno:
if t.token_type != TOKEN_COMMENT:
if t.token_type != TokenType.COMMENT:
for c in lineno_comment_map[comment_lineno_cache]:
filemsg = ''
if origin:
@ -163,7 +160,7 @@ def templatize(src, origin=None):
out.write('# %s' % ' | '.join(lineno_comment_map[comment_lineno_cache]))
comment_lineno_cache = None
if t.token_type == TOKEN_BLOCK:
if t.token_type == TokenType.BLOCK:
imatch = inline_re.match(t.contents)
bmatch = block_re.match(t.contents)
cmatches = constant_re.findall(t.contents)
@ -211,7 +208,7 @@ def templatize(src, origin=None):
incomment = True
else:
out.write(blankout(t.contents, 'B'))
elif t.token_type == TOKEN_VAR:
elif t.token_type == TokenType.VAR:
parts = t.contents.split('|')
cmatch = constant_re.match(parts[0])
if cmatch:
@ -221,7 +218,7 @@ def templatize(src, origin=None):
out.write(' %s ' % p.split(':', 1)[1])
else:
out.write(blankout(p, 'F'))
elif t.token_type == TOKEN_COMMENT:
elif t.token_type == TokenType.COMMENT:
if t.contents.lstrip().startswith(TRANSLATOR_COMMENT_MARK):
lineno_comment_map.setdefault(t.lineno, []).append(t.contents)
comment_lineno_cache = t.lineno

4
tests/template_tests/test_parser.py
View File

@ -3,7 +3,7 @@ Testing some internals of the template processing. These are *not* examples to b
"""
from django.template import Library, TemplateSyntaxError
from django.template.base import (
TOKEN_BLOCK, FilterExpression, Parser, Token, Variable,
FilterExpression, Parser, Token, TokenType, Variable,
)
from django.template.defaultfilters import register as filter_library
from django.test import SimpleTestCase
@ -15,7 +15,7 @@ class ParserTests(SimpleTestCase):
"""
#7027 -- _() syntax should work with spaces
"""
token = Token(TOKEN_BLOCK, 'sometag _("Page not found") value|yesno:_("yes,no")')
token = Token(TokenType.BLOCK, 'sometag _("Page not found") value|yesno:_("yes,no")')
split = token.split_contents()
self.assertEqual(split, ["sometag", '_("Page not found")', 'value|yesno:_("yes,no")'])