Merge pull request #2038 from ericvergnaud/fix-unicode-2031

fix #2031

.travis.yml

@@ -154,4 +154,3 @@ before_install:
 
 script:
   - cd runtime-testsuite; ../.travis/run-tests-$TARGET.sh
-

runtime/Python2/src/antlr4/tree/RuleTagToken.py

@@ -36,14 +36,13 @@ class RuleTagToken(Token):
         self.tokenIndex = -1 # from 0..n-1 of the token object in the input stream
         self.line = 0 # line=1..n of the 1st character
         self.column = -1 # beginning of the line at which it occurs, 0..n-1
-        self.label = label
+        self.label = unicode(label)
         self._text = self.getText() # text of the token.
-
+        self.ruleName = unicode(ruleName)
-        self.ruleName = ruleName
 
 
     def getText(self):
         if self.label is None:
-            return "<" + self.ruleName + ">"
+            return u"<" + self.ruleName + u">"
         else:
-            return "<" + self.label + ":" + self.ruleName + ">"
+            return u"<" + self.label + ":" + self.ruleName + u">"

runtime/Python2/src/antlr4/tree/TokenTagToken.py

@@ -24,8 +24,8 @@ class TokenTagToken(CommonToken):
     #
     def __init__(self, tokenName, type, label=None):
         super(TokenTagToken, self).__init__(type=type)
-        self.tokenName = tokenName
+        self.tokenName = unicode(tokenName)
-        self.label = label
+        self.label = unicode(label)
         self._text = self.getText()
 
     #
@@ -36,9 +36,9 @@ class TokenTagToken(CommonToken):
     #
     def getText(self):
         if self.label is None:
-            return "<" + self.tokenName + ">"
+            return u"<" + self.tokenName + u">"
         else:
-            return "<" + self.label + ":" + self.tokenName + ">"
+            return u"<" + self.label + u":" + self.tokenName + u">"
 
     # <p>The implementation for {@link TokenTagToken} returns a string of the form
     # {@code tokenName:type}.</p>

runtime/Python2/src/antlr4/tree/Tree.py

@@ -108,13 +108,13 @@ class TerminalNodeImpl(TerminalNode):
         return visitor.visitTerminal(self)
 
     def getText(self):
-        return self.symbol.text
+        return unicode(self.symbol.text)
 
     def __unicode__(self):
         if self.symbol.type == Token.EOF:
-            return "<EOF>"
+            return u"<EOF>"
         else:
-            return self.symbol.text
+            return unicode(self.symbol.text)
 
 # Represents a token that was consumed during resynchronization
 #  rather than during a valid match operation. For example,