Added unicode_literals to the jslexer.

This ensure that ''.join(c) in jslex.py always returns text.
This commit is contained in:
Florian Apolloner 2014-10-14 21:55:54 +02:00
parent 4443c6f6d8
commit 2ccbaba1f2
2 changed files with 3 additions and 1 deletions

View File

@ -1,5 +1,6 @@
"""JsLex: a lexer for Javascript"""
# Originally from https://bitbucket.org/ned/jslex
from __future__ import unicode_literals
import re

View File

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
"""Tests for jslex."""
# originally from https://bitbucket.org/ned/jslex
from __future__ import unicode_literals
from django.test import TestCase
from django.utils.jslex import JsLexer, prepare_js_for_gettext
@ -10,7 +11,7 @@ class JsTokensTest(TestCase):
LEX_CASES = [
# ids
("a ABC $ _ a123", ["id a", "id ABC", "id $", "id _", "id a123"]),
(r"\u1234 abc\u0020 \u0065_\u0067", [r"id \u1234", r"id abc\u0020", r"id \u0065_\u0067"]),
("\\u1234 abc\\u0020 \\u0065_\\u0067", ["id \\u1234", "id abc\\u0020", "id \\u0065_\\u0067"]),
# numbers
("123 1.234 0.123e-3 0 1E+40 1e1 .123", ["dnum 123", "dnum 1.234", "dnum 0.123e-3", "dnum 0", "dnum 1E+40", "dnum 1e1", "dnum .123"]),
("0x1 0xabCD 0XABcd", ["hnum 0x1", "hnum 0xabCD", "hnum 0XABcd"]),