forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Merge pull request #2458 from ericvergnaud/fix-python-getText-api 

Fix python get text api
This commit is contained in:
Terence Parr 2019-01-01 11:55:30 -07:00 committed by GitHub
parent 6808f9c4b4 e06b7a8b5d
commit 9f422d87b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 58 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
runtime/Python2/src/antlr4/BufferedTokenStream.py
View File

@ -269,36 +269,32 @@ class BufferedTokenStream(TokenStream):
def getSourceName(self):
return self.tokenSource.getSourceName()
def getText(self, interval=None):
def getText(self, start=None, stop=None):
"""
Get the text of all tokens in this buffer.
:param interval:
:type interval: antlr4.IntervalSet.Interval
:return: string
"""
self.lazyInit()
self.fill()
if interval is None:
interval = (0, len(self.tokens)-1)
start = interval[0]
if isinstance(start, Token):
start = start.tokenIndex
stop = interval[1]
elif start is None:
start = 0
if isinstance(stop, Token):
stop = stop.tokenIndex
if start is None or stop is None or start<0 or stop<0:
return ""
if stop >= len(self.tokens):
stop = len(self.tokens)-1
elif stop is None or stop >= len(self.tokens):
stop = len(self.tokens) - 1
if start < 0 or stop < 0 or stop<start:
return u""
with StringIO() as buf:
for i in range(start, stop+1):
for i in xrange(start, stop+1):
t = self.tokens[i]
if t.type==Token.EOF:
break
buf.write(t.text)
return buf.getvalue()
def fill(self):
"""
Get all tokens from lexer until EOF

6
runtime/Python2/src/antlr4/IntervalSet.py
View File

@ -24,12 +24,6 @@ class Interval(object):
def __iter__(self):
return iter(self.range)
def __getitem__(self, idx):
if idx == 0:
return self.start
elif idx == 1:
return self.stop
raise IndexError('Interval index out or range [{}]'.format(idx))
class IntervalSet(object):

14
runtime/Python2/src/antlr4/TokenStreamRewriter.py
View File

@ -100,9 +100,9 @@ class TokenStreamRewriter(object):
return self.programs.setdefault(program_name, [])
def getDefaultText(self):
return self.getText(self.DEFAULT_PROGRAM_NAME, Interval(0, len(self.tokens.tokens)))
return self.getText(self.DEFAULT_PROGRAM_NAME, 0, len(self.tokens.tokens))
def getText(self, program_name, interval):
def getText(self, program_name, start, stop):
"""
:type interval: Interval.Interval
:param program_name:
@ -110,15 +110,15 @@ class TokenStreamRewriter(object):
:return:
"""
rewrites = self.programs.get(program_name)
start = interval.start
stop = interval.stop
# ensure start/end are in range
if stop > len(self.tokens.tokens) - 1: stop = len(self.tokens.tokens) - 1
if start < 0: start = 0
if stop > len(self.tokens.tokens) - 1:
stop = len(self.tokens.tokens) - 1
if start < 0:
start = 0
# if no instructions to execute
if not rewrites: return self.tokens.getText(interval)
if not rewrites: return self.tokens.getText(start, stop)
buf = StringIO()
indexToOp = self._reduceToSingleOperationPerIndex(rewrites)
i = start

11
runtime/Python2/src/antlr4/atn/ParserATNSimulator.py
View File

@ -1603,17 +1603,15 @@ class ParserATNSimulator(ATNSimulator):
def reportAttemptingFullContext(self, dfa, conflictingAlts, configs, startIndex, stopIndex):
if ParserATNSimulator.debug or ParserATNSimulator.retry_debug:
interval = range(startIndex, stopIndex + 1)
print("reportAttemptingFullContext decision=" + str(dfa.decision) + ":" + str(configs) +
", input=" + self.parser.getTokenStream().getText(interval))
", input=" + self.parser.getTokenStream().getText(startIndex, stopIndex + 1))
if self.parser is not None:
self.parser.getErrorListenerDispatch().reportAttemptingFullContext(self.parser, dfa, startIndex, stopIndex, conflictingAlts, configs)
def reportContextSensitivity(self, dfa, prediction, configs, startIndex, stopIndex):
if ParserATNSimulator.debug or ParserATNSimulator.retry_debug:
interval = range(startIndex, stopIndex + 1)
print("reportContextSensitivity decision=" + str(dfa.decision) + ":" + str(configs) +
", input=" + self.parser.getTokenStream().getText(interval))
", input=" + self.parser.getTokenStream().getText(startIndex, stopIndex + 1))
if self.parser is not None:
self.parser.getErrorListenerDispatch().reportContextSensitivity(self.parser, dfa, startIndex, stopIndex, prediction, configs)
@ -1637,9 +1635,8 @@ class ParserATNSimulator(ATNSimulator):
# }
# i++;
# }
interval = range(startIndex, stopIndex + 1)
print("reportAmbiguity " + str(ambigAlts) + ":" + str(configs) +
", input=" + self.parser.getTokenStream().getText(interval))
print("reportAmbiguity " + str(ambigAlts) + ":" + str(configs) +
", input=" + self.parser.getTokenStream().getText(startIndex, stopIndex + 1))
if self.parser is not None:
self.parser.getErrorListenerDispatch().reportAmbiguity(self.parser, dfa, startIndex, stopIndex, exact, ambigAlts, configs)

6
runtime/Python2/src/antlr4/error/DiagnosticErrorListener.py
View File

@ -45,7 +45,7 @@ class DiagnosticErrorListener(ErrorListener):
buf.write(u": ambigAlts=")
buf.write(str_set(self.getConflictingAlts(ambigAlts, configs)))
buf.write(u", input='")
buf.write(recognizer.getTokenStream().getText((startIndex, stopIndex)))
buf.write(recognizer.getTokenStream().getText(startIndex, stopIndex))
buf.write(u"'")
recognizer.notifyErrorListeners(buf.getvalue())
@ -56,7 +56,7 @@ class DiagnosticErrorListener(ErrorListener):
buf.write(u"reportAttemptingFullContext d=")
buf.write(self.getDecisionDescription(recognizer, dfa))
buf.write(u", input='")
buf.write(recognizer.getTokenStream().getText((startIndex, stopIndex)))
buf.write(recognizer.getTokenStream().getText(startIndex, stopIndex))
buf.write(u"'")
recognizer.notifyErrorListeners(buf.getvalue())
@ -66,7 +66,7 @@ class DiagnosticErrorListener(ErrorListener):
buf.write(u"reportContextSensitivity d=")
buf.write(self.getDecisionDescription(recognizer, dfa))
buf.write(u", input='")
buf.write(recognizer.getTokenStream().getText((startIndex, stopIndex)))
buf.write(recognizer.getTokenStream().getText(startIndex, stopIndex))
buf.write(u"'")
recognizer.notifyErrorListeners(buf.getvalue())

2
runtime/Python2/src/antlr4/error/ErrorStrategy.py
View File

@ -238,7 +238,7 @@ class DefaultErrorStrategy(ErrorStrategy):
if e.startToken.type==Token.EOF:
input = "<EOF>"
else:
input = tokens.getText((e.startToken, e.offendingToken))
input = tokens.getText(e.startToken, e.offendingToken)
else:
input = "<unknown input>"
msg = "no viable alternative at input " + self.escapeWSAndQuote(input)

2
runtime/Python2/src/antlr4/error/Errors.py
View File

@ -83,7 +83,7 @@ class LexerNoViableAltException(RecognitionException):
def __unicode__(self):
symbol = ""
if self.startIndex >= 0 and self.startIndex < self.input.size():
symbol = self.input.getText((self.startIndex,self.startIndex))
symbol = self.input.getText(self.startIndex,self.startIndex)
# TODO symbol = Utils.escapeWhitespace(symbol, false);
return u"LexerNoViableAltException" + symbol

18
runtime/Python2/tests/TestTokenStreamRewriter.py
View File

@ -4,8 +4,6 @@
import unittest
from antlr4.IntervalSet import Interval
from mocks.TestLexer import TestLexer, TestLexer2
from antlr4.TokenStreamRewriter import TokenStreamRewriter
from antlr4.InputStream import InputStream
@ -88,8 +86,8 @@ class TestTokenStreamRewriter(unittest.TestCase):
rewriter.replaceRange(4, 8, '0')
self.assertEquals(rewriter.getDefaultText(), 'x = 0;')
self.assertEquals(rewriter.getText('default', Interval(0, 9)), 'x = 0;')
self.assertEquals(rewriter.getText('default', Interval(4, 8)), '0')
self.assertEquals(rewriter.getText('default', 0, 9), 'x = 0;')
self.assertEquals(rewriter.getText('default', 4, 8), '0')
def testToStringStartStop2(self):
input = InputStream('x = 3 * 0 + 2 * 0;')
@ -103,15 +101,15 @@ class TestTokenStreamRewriter(unittest.TestCase):
# replace 3 * 0 with 0
rewriter.replaceRange(4, 8, '0')
self.assertEquals('x = 0 + 2 * 0;', rewriter.getDefaultText())
self.assertEquals('x = 0 + 2 * 0;', rewriter.getText('default', Interval(0, 17)))
self.assertEquals('0', rewriter.getText('default', Interval(4, 8)))
self.assertEquals('x = 0', rewriter.getText('default', Interval(0, 8)))
self.assertEquals('2 * 0', rewriter.getText('default', Interval(12, 16)))
self.assertEquals('x = 0 + 2 * 0;', rewriter.getText('default', 0, 17))
self.assertEquals('0', rewriter.getText('default', 4, 8))
self.assertEquals('x = 0', rewriter.getText('default', 0, 8))
self.assertEquals('2 * 0', rewriter.getText('default', 12, 16))
rewriter.insertAfter(17, "// comment")
self.assertEquals('2 * 0;// comment', rewriter.getText('default', Interval(12, 18)))
self.assertEquals('2 * 0;// comment', rewriter.getText('default', 12, 18))
self.assertEquals('x = 0', rewriter.getText('default', Interval(0, 8)))
self.assertEquals('x = 0', rewriter.getText('default', 0, 8))
def test2ReplaceMiddleIndex(self):
input = InputStream('abc')

14
runtime/Python3/src/antlr4/BufferedTokenStream.py
View File

@ -272,21 +272,19 @@ class BufferedTokenStream(TokenStream):
return self.tokenSource.getSourceName()
# Get the text of all tokens in this buffer.#/
def getText(self, interval:tuple=None):
def getText(self, start:int=None, stop:int=None):
self.lazyInit()
self.fill()
if interval is None:
interval = (0, len(self.tokens)-1)
start = interval[0]
if isinstance(start, Token):
start = start.tokenIndex
stop = interval[1]
elif start is None:
start = 0
if isinstance(stop, Token):
stop = stop.tokenIndex
if start is None or stop is None or start<0 or stop<0:
elif stop is None or stop >= len(self.tokens):
stop = len(self.tokens) - 1
if start < 0 or stop < 0 or stop < start:
return ""
if stop >= len(self.tokens):
stop = len(self.tokens)-1
with StringIO() as buf:
for i in range(start, stop+1):
t = self.tokens[i]

19
runtime/Python3/src/antlr4/TokenStreamRewriter.py
View File

@ -96,23 +96,20 @@ class TokenStreamRewriter(object):
def getProgram(self, program_name):
return self.programs.setdefault(program_name, [])
def getText(self, program_name, interval):
def getText(self, program_name, start:int, stop:int):
"""
:type interval: Interval.Interval
:param program_name:
:param interval:
:return:
"""
rewrites = self.programs.get(program_name)
start = interval.start
stop = interval.stop
# ensure start/end are in range
if stop > len(self.tokens.tokens) - 1: stop = len(self.tokens.tokens) - 1
if start < 0: start = 0
if stop > len(self.tokens.tokens) - 1:
stop = len(self.tokens.tokens) - 1
if start < 0:
start = 0
# if no instructions to execute
if not rewrites: return self.tokens.getText(interval)
if not rewrites: return self.tokens.getText(start, stop)
buf = StringIO()
indexToOp = self._reduceToSingleOperationPerIndex(rewrites)
i = start
@ -149,12 +146,12 @@ class TokenStreamRewriter(object):
prevReplaces = [op for op in rewrites[:i] if isinstance(op, TokenStreamRewriter.ReplaceOp)]
for prevRop in prevReplaces:
if all((prevRop.index >= rop.index, prevRop.last_index <= rop.last_index)):
rewrites[prevRop.instructioIndex] = None
rewrites[prevRop.instructionIndex] = None
continue
isDisjoint = any((prevRop.last_index<rop.index, prevRop.index>rop))
isSame = all((prevRop.index == rop.index, prevRop.last_index == rop.last_index))
if all((prevRop.text is None, rop.text is None, not isDisjoint)):
rewrites[prevRop.instructioIndex] = None
rewrites[prevRop.instructionIndex] = None
rop.index = min(prevRop.index, rop.index)
rop.last_index = min(prevRop.last_index, rop.last_index)
print('New rop {}'.format(rop))

9
runtime/Python3/src/antlr4/atn/ParserATNSimulator.py
View File

@ -1607,17 +1607,15 @@ class ParserATNSimulator(ATNSimulator):
def reportAttemptingFullContext(self, dfa:DFA, conflictingAlts:set, configs:ATNConfigSet, startIndex:int, stopIndex:int):
if ParserATNSimulator.debug or ParserATNSimulator.retry_debug:
interval = (startIndex, stopIndex + 1)
print("reportAttemptingFullContext decision=" + str(dfa.decision) + ":" + str(configs) +
", input=" + self.parser.getTokenStream().getText(interval))
", input=" + self.parser.getTokenStream().getText(startIndex, stopIndex))
if self.parser is not None:
self.parser.getErrorListenerDispatch().reportAttemptingFullContext(self.parser, dfa, startIndex, stopIndex, conflictingAlts, configs)
def reportContextSensitivity(self, dfa:DFA, prediction:int, configs:ATNConfigSet, startIndex:int, stopIndex:int):
if ParserATNSimulator.debug or ParserATNSimulator.retry_debug:
interval = (startIndex, stopIndex + 1)
print("reportContextSensitivity decision=" + str(dfa.decision) + ":" + str(configs) +
", input=" + self.parser.getTokenStream().getText(interval))
", input=" + self.parser.getTokenStream().getText(startIndex, stopIndex))
if self.parser is not None:
self.parser.getErrorListenerDispatch().reportContextSensitivity(self.parser, dfa, startIndex, stopIndex, prediction, configs)
@ -1641,9 +1639,8 @@ class ParserATNSimulator(ATNSimulator):
# }
# i++;
# }
interval = (startIndex, stopIndex + 1)
print("reportAmbiguity " + str(ambigAlts) + ":" + str(configs) +
", input=" + self.parser.getTokenStream().getText(interval))
", input=" + self.parser.getTokenStream().getText(startIndex, stopIndex))
if self.parser is not None:
self.parser.getErrorListenerDispatch().reportAmbiguity(self.parser, dfa, startIndex, stopIndex, exact, ambigAlts, configs)

6
runtime/Python3/src/antlr4/error/DiagnosticErrorListener.py
View File

@ -46,7 +46,7 @@ class DiagnosticErrorListener(ErrorListener):
buf.write(": ambigAlts=")
buf.write(str(self.getConflictingAlts(ambigAlts, configs)))
buf.write(", input='")
buf.write(recognizer.getTokenStream().getText((startIndex, stopIndex)))
buf.write(recognizer.getTokenStream().getText(startIndex, stopIndex))
buf.write("'")
recognizer.notifyErrorListeners(buf.getvalue())
@ -57,7 +57,7 @@ class DiagnosticErrorListener(ErrorListener):
buf.write("reportAttemptingFullContext d=")
buf.write(self.getDecisionDescription(recognizer, dfa))
buf.write(", input='")
buf.write(recognizer.getTokenStream().getText((startIndex, stopIndex)))
buf.write(recognizer.getTokenStream().getText(startIndex, stopIndex))
buf.write("'")
recognizer.notifyErrorListeners(buf.getvalue())
@ -67,7 +67,7 @@ class DiagnosticErrorListener(ErrorListener):
buf.write("reportContextSensitivity d=")
buf.write(self.getDecisionDescription(recognizer, dfa))
buf.write(", input='")
buf.write(recognizer.getTokenStream().getText((startIndex, stopIndex)))
buf.write(recognizer.getTokenStream().getText(startIndex, stopIndex))
buf.write("'")
recognizer.notifyErrorListeners(buf.getvalue())

2
runtime/Python3/src/antlr4/error/ErrorStrategy.py
View File

@ -243,7 +243,7 @@ class DefaultErrorStrategy(ErrorStrategy):
if e.startToken.type==Token.EOF:
input = "<EOF>"
else:
input = tokens.getText((e.startToken, e.offendingToken))
input = tokens.getText(e.startToken, e.offendingToken)
else:
input = "<unknown input>"
msg = "no viable alternative at input " + self.escapeWSAndQuote(input)

4
tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg
View File

@ -558,13 +558,13 @@ TokenPropertyRef_int(t) ::= "(0 if <ctx(t)>.<t.label> is None else int(<ctx(t)>.
RulePropertyRef_start(r) ::= "(None if <ctx(r)>.<r.label> is None else <ctx(r)>.<r.label>.start)"
RulePropertyRef_stop(r) ::= "(None if <ctx(r)>.<r.label> is None else <ctx(r)>.<r.label>.stop)"
RulePropertyRef_text(r) ::= "(None if <ctx(r)>.<r.label> is None else self._input.getText((<ctx(r)>.<r.label>.start,<ctx(r)>.<r.label>.stop)))"
RulePropertyRef_text(r) ::= "(None if <ctx(r)>.<r.label> is None else self._input.getText(<ctx(r)>.<r.label>.start,<ctx(r)>.<r.label>.stop))"
RulePropertyRef_ctx(r) ::= "<ctx(r)>.<r.label>"
RulePropertyRef_parser(r) ::= "self"
ThisRulePropertyRef_start(r) ::= "localctx.start"
ThisRulePropertyRef_stop(r) ::= "localctx.stop"
ThisRulePropertyRef_text(r) ::= "self._input.getText((localctx.start, self._input.LT(-1)))"
ThisRulePropertyRef_text(r) ::= "self._input.getText(localctx.start, self._input.LT(-1))"
ThisRulePropertyRef_ctx(r) ::= "localctx"
ThisRulePropertyRef_parser(r) ::= "self"

4
tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg
View File

@ -570,13 +570,13 @@ TokenPropertyRef_int(t) ::= "(0 if <ctx(t)>.<t.label> is None else int(<ctx(t)>.
RulePropertyRef_start(r) ::= "(None if <ctx(r)>.<r.label> is None else <ctx(r)>.<r.label>.start)"
RulePropertyRef_stop(r) ::= "(None if <ctx(r)>.<r.label> is None else <ctx(r)>.<r.label>.stop)"
RulePropertyRef_text(r) ::= "(None if <ctx(r)>.<r.label> is None else self._input.getText((<ctx(r)>.<r.label>.start,<ctx(r)>.<r.label>.stop)))"
RulePropertyRef_text(r) ::= "(None if <ctx(r)>.<r.label> is None else self._input.getText(<ctx(r)>.<r.label>.start,<ctx(r)>.<r.label>.stop))"
RulePropertyRef_ctx(r) ::= "<ctx(r)>.<r.label>"
RulePropertyRef_parser(r) ::= "self"
ThisRulePropertyRef_start(r) ::= "localctx.start"
ThisRulePropertyRef_stop(r) ::= "localctx.stop"
ThisRulePropertyRef_text(r) ::= "self._input.getText((localctx.start, self._input.LT(-1)))"
ThisRulePropertyRef_text(r) ::= "self._input.getText(localctx.start, self._input.LT(-1))"
ThisRulePropertyRef_ctx(r) ::= "localctx"
ThisRulePropertyRef_parser(r) ::= "self"