Refs #33002 -- Added DebugLexer/Lexer.tokenize() test for trailing text token.

This commit is contained in:
Mariusz Felisiak 2021-08-09 13:33:16 +02:00
parent b2be7e12cc
commit 65ed96fa39
1 changed files with 2 additions and 0 deletions

View File

@ -9,6 +9,7 @@ class LexerTestMixin:
'text\n' 'text\n'
'{% if test %}{{ varvalue }}{% endif %}' '{% if test %}{{ varvalue }}{% endif %}'
'{#comment {{not a var}} %{not a block}% #}' '{#comment {{not a var}} %{not a block}% #}'
'end text'
) )
expected_token_tuples = [ expected_token_tuples = [
# (token_type, contents, lineno, position) # (token_type, contents, lineno, position)
@ -17,6 +18,7 @@ class LexerTestMixin:
(TokenType.VAR, 'varvalue', 2, (18, 32)), (TokenType.VAR, 'varvalue', 2, (18, 32)),
(TokenType.BLOCK, 'endif', 2, (32, 43)), (TokenType.BLOCK, 'endif', 2, (32, 43)),
(TokenType.COMMENT, 'comment {{not a var}} %{not a block}%', 2, (43, 85)), (TokenType.COMMENT, 'comment {{not a var}} %{not a block}%', 2, (43, 85)),
(TokenType.TEXT, 'end text', 2, (85, 93)),
] ]
def test_tokenize(self): def test_tokenize(self):