jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Merge pull request #3046 from ericvergnaud/update-LL1Analyzer 

Update LL1 analyzer in C#, Python, JavaScript, Cpp and Swift runtimes, aligning on the Java runtime
Fixes #3042
This commit is contained in:
ericvergnaud 2021-01-24 11:12:18 +08:00 committed by GitHub
parent 954649d5aa 58da376bef
commit b050ac43de
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 264 additions and 326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -100,3 +100,4 @@ javac-services.0.log.lck
# Don't ignore python tests # Don't ignore python tests
!runtime/Python3/test/ !runtime/Python3/test/
Antlr4.sln

401
runtime/CSharp/Atn/LL1Analyzer.cs
View File

@ -3,7 +3,6 @@
* can be found in the LICENSE.txt file in the project root. * can be found in the LICENSE.txt file in the project root.
*/ */
using System.Collections.Generic; using System.Collections.Generic;
using Antlr4.Runtime.Atn;
using Antlr4.Runtime.Misc; using Antlr4.Runtime.Misc;
using Antlr4.Runtime.Sharpen; using Antlr4.Runtime.Sharpen;
@ -11,12 +10,9 @@ namespace Antlr4.Runtime.Atn
{ {
public class LL1Analyzer public class LL1Analyzer
{ {
/// <summary> /** Special value added to the lookahead sets to indicate that we hit
/// Special value added to the lookahead sets to indicate that we hit * a predicate during analysis if {@code seeThruPreds==false}.
/// a predicate during analysis if */
/// <c>seeThruPreds==false</c>
/// .
/// </summary>
public const int HitPred = TokenConstants.InvalidType; public const int HitPred = TokenConstants.InvalidType;
[NotNull] [NotNull]
@ -27,25 +23,16 @@ namespace Antlr4.Runtime.Atn
this.atn = atn; this.atn = atn;
} }
/// <summary> /**
/// Calculates the SLL(1) expected lookahead set for each outgoing transition * Calculates the SLL(1) expected lookahead set for each outgoing transition
/// of an * of an {@link ATNState}. The returned array has one element for each
/// <see cref="ATNState"/> * outgoing transition in {@code s}. If the closure from transition
/// . The returned array has one element for each * <em>i</em> leads to a semantic predicate before matching a symbol, the
/// outgoing transition in * element at index <em>i</em> of the result will be {@code null}.
/// <paramref name="s"/> *
/// . If the closure from transition * @param s the ATN state
/// <em>i</em> leads to a semantic predicate before matching a symbol, the * @return the expected symbols for each outgoing transition of {@code s}.
/// element at index <em>i</em> of the result will be */
/// <see langword="null"/>
/// .
/// </summary>
/// <param name="s">the ATN state</param>
/// <returns>
/// the expected symbols for each outgoing transition of
/// <paramref name="s"/>
/// .
/// </returns>
[return: Nullable] [return: Nullable]
public virtual IntervalSet[] GetDecisionLookahead(ATNState s) public virtual IntervalSet[] GetDecisionLookahead(ATNState s)
{ {
@ -61,7 +48,7 @@ namespace Antlr4.Runtime.Atn
HashSet<ATNConfig> lookBusy = new HashSet<ATNConfig>(); HashSet<ATNConfig> lookBusy = new HashSet<ATNConfig>();
bool seeThruPreds = false; bool seeThruPreds = false;
// fail to get lookahead upon pred // fail to get lookahead upon pred
Look(s.Transition(alt).target, null, PredictionContext.EMPTY, look[alt], lookBusy, new BitSet(), seeThruPreds, false); Look_(s.Transition(alt).target, null, PredictionContext.EMPTY, look[alt], lookBusy, new BitSet(), seeThruPreds, false);
// Wipe out lookahead for this alternative if we found nothing // Wipe out lookahead for this alternative if we found nothing
// or we had a predicate when we !seeThruPreds // or we had a predicate when we !seeThruPreds
if (look[alt].Count == 0 || look[alt].Contains(HitPred)) if (look[alt].Count == 0 || look[alt].Contains(HitPred))
@ -72,190 +59,88 @@ namespace Antlr4.Runtime.Atn
return look; return look;
} }
/// <summary> /**
/// Compute set of tokens that can follow * Compute set of tokens that can follow {@code s} in the ATN in the
/// <paramref name="s"/> * specified {@code ctx}.
/// in the ATN in the *
/// specified * <p>If {@code ctx} is {@code null} and the end of the rule containing
/// <paramref name="ctx"/> * {@code s} is reached, {@link Token#EPSILON} is added to the result set.
/// . * If {@code ctx} is not {@code null} and the end of the outermost rule is
/// <p>If * reached, {@link Token#EOF} is added to the result set.</p>
/// <paramref name="ctx"/> *
/// is * @param s the ATN state
/// <see langword="null"/> * @param ctx the complete parser context, or {@code null} if the context
/// and the end of the rule containing * should be ignored
/// <paramref name="s"/> *
/// is reached, * @return The set of tokens that can follow {@code s} in the ATN in the
/// <see cref="TokenConstants.EPSILON"/> * specified {@code ctx}.
/// is added to the result set. */
/// If
/// <paramref name="ctx"/>
/// is not
/// <see langword="null"/>
/// and the end of the outermost rule is
/// reached,
/// <see cref="TokenConstants.EOF"/>
/// is added to the result set.</p>
/// </summary>
/// <param name="s">the ATN state</param>
/// <param name="ctx">
/// the complete parser context, or
/// <see langword="null"/>
/// if the context
/// should be ignored
/// </param>
/// <returns>
/// The set of tokens that can follow
/// <paramref name="s"/>
/// in the ATN in the
/// specified
/// <paramref name="ctx"/>
/// .
/// </returns>
[return: NotNull] [return: NotNull]
public virtual IntervalSet Look(ATNState s, RuleContext ctx) public virtual IntervalSet Look(ATNState s, RuleContext ctx)
{ {
return Look(s, null, ctx); return Look(s, null, ctx);
} }
/// <summary> /**
/// Compute set of tokens that can follow * Compute set of tokens that can follow {@code s} in the ATN in the
/// <paramref name="s"/> * specified {@code ctx}.
/// in the ATN in the *
/// specified * <p>If {@code ctx} is {@code null} and the end of the rule containing
/// <paramref name="ctx"/> * {@code s} is reached, {@link Token#EPSILON} is added to the result set.
/// . * If {@code ctx} is not {@code null} and the end of the outermost rule is
/// <p>If * reached, {@link Token#EOF} is added to the result set.</p>
/// <paramref name="ctx"/> *
/// is * @param s the ATN state
/// <see langword="null"/> * @param stopState the ATN state to stop at. This can be a
/// and the end of the rule containing * {@link BlockEndState} to detect epsilon paths through a closure.
/// <paramref name="s"/> * @param ctx the complete parser context, or {@code null} if the context
/// is reached, * should be ignored
/// <see cref="TokenConstants.EPSILON"/> *
/// is added to the result set. * @return The set of tokens that can follow {@code s} in the ATN in the
/// If * specified {@code ctx}.
/// <paramref name="ctx"/> */
/// is not
/// <c>PredictionContext#EMPTY_LOCAL</c>
/// and the end of the outermost rule is
/// reached,
/// <see cref="TokenConstants.EOF"/>
/// is added to the result set.</p>
/// </summary>
/// <param name="s">the ATN state</param>
/// <param name="stopState">
/// the ATN state to stop at. This can be a
/// <see cref="BlockEndState"/>
/// to detect epsilon paths through a closure.
/// </param>
/// <param name="ctx">
/// the complete parser context, or
/// <see langword="null"/>
/// if the context
/// should be ignored
/// </param>
/// <returns>
/// The set of tokens that can follow
/// <paramref name="s"/>
/// in the ATN in the
/// specified
/// <paramref name="ctx"/>
/// .
/// </returns>
[return: NotNull] [return: NotNull]
public virtual IntervalSet Look(ATNState s, ATNState stopState, RuleContext ctx) public virtual IntervalSet Look(ATNState s, ATNState stopState, RuleContext ctx)
{ {
IntervalSet r = new IntervalSet(); IntervalSet r = new IntervalSet();
bool seeThruPreds = true; bool seeThruPreds = true;
PredictionContext lookContext = ctx != null ? PredictionContext.FromRuleContext(s.atn, ctx) : null; PredictionContext lookContext = ctx != null ? PredictionContext.FromRuleContext(s.atn, ctx) : null;
Look(s, stopState, lookContext, r, new HashSet<ATNConfig>(), new BitSet(), seeThruPreds, true); Look_(s, stopState, lookContext, r, new HashSet<ATNConfig>(), new BitSet(), seeThruPreds, true);
return r; return r;
} }
/// <summary> /**
/// Compute set of tokens that can follow * Compute set of tokens that can follow {@code s} in the ATN in the
/// <paramref name="s"/> * specified {@code ctx}.
/// in the ATN in the *
/// specified * <p>If {@code ctx} is {@code null} and {@code stopState} or the end of the
/// <paramref name="ctx"/> * rule containing {@code s} is reached, {@link Token#EPSILON} is added to
/// . * the result set. If {@code ctx} is not {@code null} and {@code addEOF} is
/// <p/> * {@code true} and {@code stopState} or the end of the outermost rule is
/// If * reached, {@link Token#EOF} is added to the result set.</p>
/// <paramref name="ctx"/> *
/// is * @param s the ATN state.
/// <see cref="PredictionContext.EMPTY"/> * @param stopState the ATN state to stop at. This can be a
/// and * {@link BlockEndState} to detect epsilon paths through a closure.
/// <paramref name="stopState"/> * @param ctx The outer context, or {@code null} if the outer context should
/// or the end of the rule containing * not be used.
/// <paramref name="s"/> * @param look The result lookahead set.
/// is reached, * @param lookBusy A set used for preventing epsilon closures in the ATN
/// <see cref="TokenConstants.EPSILON"/> * from causing a stack overflow. Outside code should pass
/// is added to the result set. If * {@code new HashSet<ATNConfig>} for this argument.
/// <paramref name="ctx"/> * @param calledRuleStack A set used for preventing left recursion in the
/// is not * ATN from causing a stack overflow. Outside code should pass
/// <see cref="PredictionContext.EMPTY"/> * {@code new BitSet()} for this argument.
/// and * @param seeThruPreds {@code true} to true semantic predicates as
/// <paramref name="addEOF"/> * implicitly {@code true} and "see through them", otherwise {@code false}
/// is * to treat semantic predicates as opaque and add {@link #HIT_PRED} to the
/// <see langword="true"/> * result if one is encountered.
/// and * @param addEOF Add {@link Token#EOF} to the result if the end of the
/// <paramref name="stopState"/> * outermost context is reached. This parameter has no effect if {@code ctx}
/// or the end of the outermost rule is reached, * is {@code null}.
/// <see cref="TokenConstants.EOF"/> */
/// is added to the result set. protected internal virtual void Look_(ATNState s, ATNState stopState, PredictionContext ctx, IntervalSet look, HashSet<ATNConfig> lookBusy, BitSet calledRuleStack, bool seeThruPreds, bool addEOF)
/// </summary>
/// <param name="s">the ATN state.</param>
/// <param name="stopState">
/// the ATN state to stop at. This can be a
/// <see cref="BlockEndState"/>
/// to detect epsilon paths through a closure.
/// </param>
/// <param name="ctx">
/// The outer context, or
/// <see cref="PredictionContext.EMPTY"/>
/// if
/// the outer context should not be used.
/// </param>
/// <param name="look">The result lookahead set.</param>
/// <param name="lookBusy">
/// A set used for preventing epsilon closures in the ATN
/// from causing a stack overflow. Outside code should pass
/// <c>new HashSet&lt;ATNConfig&gt;</c>
/// for this argument.
/// </param>
/// <param name="calledRuleStack">
/// A set used for preventing left recursion in the
/// ATN from causing a stack overflow. Outside code should pass
/// <c>new BitSet()</c>
/// for this argument.
/// </param>
/// <param name="seeThruPreds">
///
/// <see langword="true"/>
/// to true semantic predicates as
/// implicitly
/// <see langword="true"/>
/// and "see through them", otherwise
/// <see langword="false"/>
/// to treat semantic predicates as opaque and add
/// <see cref="HitPred"/>
/// to the
/// result if one is encountered.
/// </param>
/// <param name="addEOF">
/// Add
/// <see cref="TokenConstants.EOF"/>
/// to the result if the end of the
/// outermost context is reached. This parameter has no effect if
/// <paramref name="ctx"/>
/// is
/// <see cref="PredictionContext.EMPTY"/>
/// .
/// </param>
protected internal virtual void Look(ATNState s, ATNState stopState, PredictionContext ctx, IntervalSet look, HashSet<ATNConfig> lookBusy, BitSet calledRuleStack, bool seeThruPreds, bool addEOF)
{ {
// System.out.println("_LOOK("+s.stateNumber+", ctx="+ctx);
ATNConfig c = new ATNConfig(s, 0, ctx); ATNConfig c = new ATNConfig(s, 0, ctx);
if (!lookBusy.Add(c)) if (!lookBusy.Add(c))
{ {
@ -268,50 +153,51 @@ namespace Antlr4.Runtime.Atn
look.Add(TokenConstants.EPSILON); look.Add(TokenConstants.EPSILON);
return; return;
} }
else if (ctx.IsEmpty && addEOF) {
look.Add(TokenConstants.EOF);
return;
}
}
if (s is RuleStopState)
{
if (ctx == null)
{
look.Add(TokenConstants.EPSILON);
return;
}
else if (ctx.IsEmpty && addEOF) else if (ctx.IsEmpty && addEOF)
{ {
look.Add(TokenConstants.EOF); look.Add(TokenConstants.EOF);
return; return;
} }
if (ctx != PredictionContext.EMPTY) }
{ if (s is RuleStopState)
for (int i = 0; i < ctx.Size; i++) {
{ if (ctx == null)
ATNState returnState = atn.states[ctx.GetReturnState(i)]; {
bool removed = calledRuleStack.Get(returnState.ruleIndex); look.Add(TokenConstants.EPSILON);
try return;
{ }
calledRuleStack.Clear(returnState.ruleIndex); else if (ctx.IsEmpty && addEOF)
Look(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF); {
} look.Add(TokenConstants.EOF);
finally return;
{ }
if (removed) if (ctx != PredictionContext.EMPTY)
{ {
calledRuleStack.Set(returnState.ruleIndex); bool removed = calledRuleStack.Get(s.ruleIndex);
} try
} {
} calledRuleStack.Clear(s.ruleIndex);
return; for (int i = 0; i < ctx.Size; i++)
} {
ATNState returnState = atn.states[ctx.GetReturnState(i)];
Look_(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
}
}
finally
{
if (removed)
{
calledRuleStack.Set(s.ruleIndex);
}
}
return;
}
} }
int n = s.NumberOfTransitions; int n = s.NumberOfTransitions;
for (int i_1 = 0; i_1 < n; i_1++) for (int i_1 = 0; i_1 < n; i_1++)
{ {
Transition t = s.Transition(i_1); Transition t = s.Transition(i_1);
if (t is RuleTransition) if (t.GetType() == typeof(RuleTransition))
{ {
RuleTransition ruleTransition = (RuleTransition)t; RuleTransition ruleTransition = (RuleTransition)t;
if (calledRuleStack.Get(ruleTransition.ruleIndex)) if (calledRuleStack.Get(ruleTransition.ruleIndex))
@ -322,51 +208,42 @@ namespace Antlr4.Runtime.Atn
try try
{ {
calledRuleStack.Set(ruleTransition.target.ruleIndex); calledRuleStack.Set(ruleTransition.target.ruleIndex);
Look(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); Look_(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
} }
finally finally
{ {
calledRuleStack.Clear(ruleTransition.target.ruleIndex); calledRuleStack.Clear(ruleTransition.target.ruleIndex);
} }
} }
else else if (t is AbstractPredicateTransition)
{ {
if (t is AbstractPredicateTransition) if (seeThruPreds)
{ {
if (seeThruPreds) Look_(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
{
Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
}
else
{
look.Add(HitPred);
}
} }
else else
{ {
if (t.IsEpsilon) look.Add(HitPred);
}
}
else if (t.IsEpsilon)
{
Look_(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
}
else if (t.GetType() == typeof(WildcardTransition))
{
look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType));
}
else
{
IntervalSet set = t.Label;
if (set != null)
{
if (t is NotSetTransition)
{ {
Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType));
}
else
{
if (t is WildcardTransition)
{
look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType));
}
else
{
IntervalSet set = t.Label;
if (set != null)
{
if (t is NotSetTransition)
{
set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType));
}
look.AddAll(set);
}
}
} }
look.AddAll(set);
} }
} }
} }

31
runtime/CSharp/DefaultErrorStrategy.cs
View File

@ -42,6 +42,21 @@ namespace Antlr4.Runtime
protected internal IntervalSet lastErrorStates; protected internal IntervalSet lastErrorStates;
/**
* This field is used to propagate information about the lookahead following
* the previous match. Since prediction prefers completing the current rule
* to error recovery efforts, error reporting may occur later than the
* original point where it was discoverable. The original context is used to
* compute the true expected sets as though the reporting occurred as early
* as possible.
*/
protected ParserRuleContext nextTokensContext;
/**
* @see #nextTokensContext
*/
protected int nextTokensState;
/// <summary> /// <summary>
/// <inheritDoc/> /// <inheritDoc/>
/// <p>The default implementation simply calls /// <p>The default implementation simply calls
@ -264,8 +279,22 @@ namespace Antlr4.Runtime
int la = tokens.LA(1); int la = tokens.LA(1);
// try cheaper subset first; might get lucky. seems to shave a wee bit off // try cheaper subset first; might get lucky. seems to shave a wee bit off
var nextTokens = recognizer.Atn.NextTokens(s); var nextTokens = recognizer.Atn.NextTokens(s);
if (nextTokens.Contains(TokenConstants.EPSILON) || nextTokens.Contains(la)) if (nextTokens.Contains(la))
{ {
nextTokensContext = null;
nextTokensState = ATNState.InvalidStateNumber;
return;
}
if (nextTokens.Contains(TokenConstants.EPSILON))
{
if (nextTokensContext == null)
{
// It's possible the next token won't match; information tracked
// by sync is restricted for performance.
nextTokensContext = recognizer.Context;
nextTokensState = recognizer.State;
}
return; return;
} }
switch (s.StateType) switch (s.StateType)

18
runtime/Cpp/runtime/src/atn/LL1Analyzer.cpp
View File

@ -100,18 +100,16 @@ void LL1Analyzer::_LOOK(ATNState *s, ATNState *stopState, Ref<PredictionContext>
} }
if (ctx != PredictionContext::EMPTY) { if (ctx != PredictionContext::EMPTY) {
// run thru all possible stack tops in ctx bool removed = calledRuleStack.test(s->ruleIndex);
calledRuleStack[s->ruleIndex] = false;
auto onExit = finally([removed, &calledRuleStack, s] {
if (removed) {
calledRuleStack.set(s->ruleIndex);
}
});
// run thru all possible stack tops in ctx
for (size_t i = 0; i < ctx->size(); i++) { for (size_t i = 0; i < ctx->size(); i++) {
ATNState *returnState = _atn.states[ctx->getReturnState(i)]; ATNState *returnState = _atn.states[ctx->getReturnState(i)];
bool removed = calledRuleStack.test(returnState->ruleIndex);
auto onExit = finally([removed, &calledRuleStack, returnState] {
if (removed) {
calledRuleStack.set(returnState->ruleIndex);
}
});
calledRuleStack[returnState->ruleIndex] = false;
_LOOK(returnState, stopState, ctx->getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF); _LOOK(returnState, stopState, ctx->getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
} }
return; return;

17
runtime/Go/antlr/ll1_analyzer.go
View File

@ -112,16 +112,6 @@ func (la *LL1Analyzer) Look(s, stopState ATNState, ctx RuleContext) *IntervalSet
func (la *LL1Analyzer) look2(s, stopState ATNState, ctx PredictionContext, look *IntervalSet, lookBusy *Set, calledRuleStack *BitSet, seeThruPreds, addEOF bool, i int) { func (la *LL1Analyzer) look2(s, stopState ATNState, ctx PredictionContext, look *IntervalSet, lookBusy *Set, calledRuleStack *BitSet, seeThruPreds, addEOF bool, i int) {
returnState := la.atn.states[ctx.getReturnState(i)] returnState := la.atn.states[ctx.getReturnState(i)]
removed := calledRuleStack.contains(returnState.GetRuleIndex())
defer func() {
if removed {
calledRuleStack.add(returnState.GetRuleIndex())
}
}()
calledRuleStack.remove(returnState.GetRuleIndex())
la.look1(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF) la.look1(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
} }
@ -158,6 +148,13 @@ func (la *LL1Analyzer) look1(s, stopState ATNState, ctx PredictionContext, look
} }
if ctx != BasePredictionContextEMPTY { if ctx != BasePredictionContextEMPTY {
removed := calledRuleStack.contains(s.GetRuleIndex())
defer func() {
if removed {
calledRuleStack.add(s.GetRuleIndex())
}
}()
calledRuleStack.remove(s.GetRuleIndex())
// run thru all possible stack tops in ctx // run thru all possible stack tops in ctx
for i := 0; i < ctx.length(); i++ { for i := 0; i < ctx.length(); i++ {
returnState := la.atn.states[ctx.getReturnState(i)] returnState := la.atn.states[ctx.getReturnState(i)]

20
runtime/JavaScript/src/antlr4/LL1Analyzer.js
View File

@ -128,17 +128,17 @@ class LL1Analyzer {
return; return;
} }
if (ctx !== PredictionContext.EMPTY) { if (ctx !== PredictionContext.EMPTY) {
// run thru all possible stack tops in ctx const removed = calledRuleStack.contains(s.ruleIndex);
for(let i=0; i<ctx.length; i++) { try {
const returnState = this.atn.states[ctx.getReturnState(i)]; calledRuleStack.remove(s.ruleIndex);
const removed = calledRuleStack.contains(returnState.ruleIndex); // run thru all possible stack tops in ctx
try { for (let i = 0; i < ctx.length; i++) {
calledRuleStack.remove(returnState.ruleIndex); const returnState = this.atn.states[ctx.getReturnState(i)];
this._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF); this._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
} finally { }
if (removed) { }finally {
calledRuleStack.add(returnState.ruleIndex); if (removed) {
} calledRuleStack.add(s.ruleIndex);
} }
} }
return; return;

20
runtime/JavaScript/src/antlr4/error/ErrorStrategy.js
View File

@ -55,6 +55,8 @@ class DefaultErrorStrategy extends ErrorStrategy {
*/ */
this.lastErrorIndex = -1; this.lastErrorIndex = -1;
this.lastErrorStates = null; this.lastErrorStates = null;
this.nextTokensContext = null;
this.nextTokenState = 0;
} }
/** /**
@ -216,11 +218,21 @@ class DefaultErrorStrategy extends ErrorStrategy {
if (this.inErrorRecoveryMode(recognizer)) { if (this.inErrorRecoveryMode(recognizer)) {
return; return;
} }
const s = recognizer._interp.atn.states[recognizer.state] const s = recognizer._interp.atn.states[recognizer.state];
const la = recognizer.getTokenStream().LA(1) const la = recognizer.getTokenStream().LA(1);
// try cheaper subset first; might get lucky. seems to shave a wee bit off // try cheaper subset first; might get lucky. seems to shave a wee bit off
const nextTokens = recognizer.atn.nextTokens(s) const nextTokens = recognizer.atn.nextTokens(s);
if (nextTokens.contains(Token.EPSILON) || nextTokens.contains(la)) { if(nextTokens.contains(la)) {
this.nextTokensContext = null;
this.nextTokenState = ATNState.INVALID_STATE_NUMBER;
return;
} else if (nextTokens.contains(Token.EPSILON)) {
if(this.nextTokensContext === null) {
// It's possible the next token won't match information tracked
// by sync is restricted for performance.
this.nextTokensContext = recognizer._ctx;
this.nextTokensState = recognizer._stateNumber;
}
return; return;
} }
switch (s.stateType) { switch (s.stateType) {

26
runtime/Python2/src/antlr4/LL1Analyzer.py
View File

@ -129,16 +129,16 @@ class LL1Analyzer (object):
return return
if ctx != PredictionContext.EMPTY: if ctx != PredictionContext.EMPTY:
# run thru all possible stack tops in ctx removed = s.ruleIndex in calledRuleStack
for i in range(0, len(ctx)): try:
returnState = self.atn.states[ctx.getReturnState(i)] calledRuleStack.discard(s.ruleIndex)
removed = returnState.ruleIndex in calledRuleStack # run thru all possible stack tops in ctx
try: for i in range(0, len(ctx)):
calledRuleStack.discard(returnState.ruleIndex) returnState = self.atn.states[ctx.getReturnState(i)]
self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF) self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
finally: finally:
if removed: if removed:
calledRuleStack.add(returnState.ruleIndex) calledRuleStack.add(s.ruleIndex)
return return
for t in s.transitions: for t in s.transitions:
@ -163,8 +163,8 @@ class LL1Analyzer (object):
elif type(t) == WildcardTransition: elif type(t) == WildcardTransition:
look.addRange( Interval(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType + 1) ) look.addRange( Interval(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType + 1) )
else: else:
set = t.label set_ = t.label
if set is not None: if set_ is not None:
if isinstance(t, NotSetTransition): if isinstance(t, NotSetTransition):
set = set.complement(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType) set_ = set_.complement(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType)
look.addSet(set) look.addSet(set_)

13
runtime/Python2/src/antlr4/error/ErrorStrategy.py
View File

@ -53,6 +53,8 @@ class DefaultErrorStrategy(ErrorStrategy):
# #
self.lastErrorIndex = -1 self.lastErrorIndex = -1
self.lastErrorStates = None self.lastErrorStates = None
self.nextTokensContext = None
self.nextTokenState = 0
# <p>The default implementation simply calls {@link #endErrorCondition} to # <p>The default implementation simply calls {@link #endErrorCondition} to
# ensure that the handler is not in error recovery mode.</p> # ensure that the handler is not in error recovery mode.</p>
@ -203,7 +205,16 @@ class DefaultErrorStrategy(ErrorStrategy):
la = recognizer.getTokenStream().LA(1) la = recognizer.getTokenStream().LA(1)
# try cheaper subset first; might get lucky. seems to shave a wee bit off # try cheaper subset first; might get lucky. seems to shave a wee bit off
nextTokens = recognizer.atn.nextTokens(s) nextTokens = recognizer.atn.nextTokens(s)
if Token.EPSILON in nextTokens or la in nextTokens: if la in nextTokens:
self.nextTokensContext = None
self.nextTokenState = ATNState.INVALID_STATE_NUMBER
return
elif Token.EPSILON in nextTokens:
if self.nextTokensContext is None:
# It's possible the next token won't match information tracked
# by sync is restricted for performance.
self.nextTokensContext = recognizer._ctx
self.nextTokensState = recognizer._stateNumber
return return
if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START, if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START,

18
runtime/Python3/src/antlr4/LL1Analyzer.py
View File

@ -132,16 +132,16 @@ class LL1Analyzer (object):
return return
if ctx != PredictionContext.EMPTY: if ctx != PredictionContext.EMPTY:
# run thru all possible stack tops in ctx removed = s.ruleIndex in calledRuleStack
for i in range(0, len(ctx)): try:
returnState = self.atn.states[ctx.getReturnState(i)] calledRuleStack.discard(s.ruleIndex)
removed = returnState.ruleIndex in calledRuleStack # run thru all possible stack tops in ctx
try: for i in range(0, len(ctx)):
calledRuleStack.discard(returnState.ruleIndex) returnState = self.atn.states[ctx.getReturnState(i)]
self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF) self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
finally: finally:
if removed: if removed:
calledRuleStack.add(returnState.ruleIndex) calledRuleStack.add(s.ruleIndex)
return return
for t in s.transitions: for t in s.transitions:

13
runtime/Python3/src/antlr4/error/ErrorStrategy.py
View File

@ -58,6 +58,8 @@ class DefaultErrorStrategy(ErrorStrategy):
# #
self.lastErrorIndex = -1 self.lastErrorIndex = -1
self.lastErrorStates = None self.lastErrorStates = None
self.nextTokensContext = None
self.nextTokenState = 0
# <p>The default implementation simply calls {@link #endErrorCondition} to # <p>The default implementation simply calls {@link #endErrorCondition} to
# ensure that the handler is not in error recovery mode.</p> # ensure that the handler is not in error recovery mode.</p>
@ -208,7 +210,16 @@ class DefaultErrorStrategy(ErrorStrategy):
la = recognizer.getTokenStream().LA(1) la = recognizer.getTokenStream().LA(1)
# try cheaper subset first; might get lucky. seems to shave a wee bit off # try cheaper subset first; might get lucky. seems to shave a wee bit off
nextTokens = recognizer.atn.nextTokens(s) nextTokens = recognizer.atn.nextTokens(s)
if Token.EPSILON in nextTokens or la in nextTokens: if la in nextTokens:
self.nextTokensContext = None
self.nextTokenState = ATNState.INVALID_STATE_NUMBER
return
elif Token.EPSILON in nextTokens:
if self.nextTokensContext is None:
# It's possible the next token won't match information tracked
# by sync is restricted for performance.
self.nextTokensContext = recognizer._ctx
self.nextTokensState = recognizer._stateNumber
return return
if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START, if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START,

12
runtime/Swift/Sources/Antlr4/atn/LL1Analyzer.swift
View File

@ -169,16 +169,18 @@ public class LL1Analyzer {
} }
if ctx != PredictionContext.EMPTY { if ctx != PredictionContext.EMPTY {
let removed = try! calledRuleStack.get(s.ruleIndex!)
try! calledRuleStack.clear(s.ruleIndex!)
defer {
if removed {
try! calledRuleStack.set(s.ruleIndex!)
}
}
// run thru all possible stack tops in ctx // run thru all possible stack tops in ctx
let length = ctx.size() let length = ctx.size()
for i in 0..<length { for i in 0..<length {
let returnState = atn.states[(ctx.getReturnState(i))]! let returnState = atn.states[(ctx.getReturnState(i))]!
let removed = try! calledRuleStack.get(returnState.ruleIndex!)
try! calledRuleStack.clear(returnState.ruleIndex!)
_LOOK(returnState, stopState, ctx.getParent(i), look, &lookBusy, calledRuleStack, seeThruPreds, addEOF) _LOOK(returnState, stopState, ctx.getParent(i), look, &lookBusy, calledRuleStack, seeThruPreds, addEOF)
if removed {
try! calledRuleStack.set(returnState.ruleIndex!)
}
} }
return return
} }