Merge pull request #3046 from ericvergnaud/update-LL1Analyzer
Update LL1 analyzer in C#, Python, JavaScript, Cpp and Swift runtimes, aligning on the Java runtime Fixes #3042
This commit is contained in:
commit
b050ac43de
|
@ -100,3 +100,4 @@ javac-services.0.log.lck
|
||||||
|
|
||||||
# Don't ignore python tests
|
# Don't ignore python tests
|
||||||
!runtime/Python3/test/
|
!runtime/Python3/test/
|
||||||
|
Antlr4.sln
|
||||||
|
|
|
@ -3,7 +3,6 @@
|
||||||
* can be found in the LICENSE.txt file in the project root.
|
* can be found in the LICENSE.txt file in the project root.
|
||||||
*/
|
*/
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using Antlr4.Runtime.Atn;
|
|
||||||
using Antlr4.Runtime.Misc;
|
using Antlr4.Runtime.Misc;
|
||||||
using Antlr4.Runtime.Sharpen;
|
using Antlr4.Runtime.Sharpen;
|
||||||
|
|
||||||
|
@ -11,12 +10,9 @@ namespace Antlr4.Runtime.Atn
|
||||||
{
|
{
|
||||||
public class LL1Analyzer
|
public class LL1Analyzer
|
||||||
{
|
{
|
||||||
/// <summary>
|
/** Special value added to the lookahead sets to indicate that we hit
|
||||||
/// Special value added to the lookahead sets to indicate that we hit
|
* a predicate during analysis if {@code seeThruPreds==false}.
|
||||||
/// a predicate during analysis if
|
*/
|
||||||
/// <c>seeThruPreds==false</c>
|
|
||||||
/// .
|
|
||||||
/// </summary>
|
|
||||||
public const int HitPred = TokenConstants.InvalidType;
|
public const int HitPred = TokenConstants.InvalidType;
|
||||||
|
|
||||||
[NotNull]
|
[NotNull]
|
||||||
|
@ -27,25 +23,16 @@ namespace Antlr4.Runtime.Atn
|
||||||
this.atn = atn;
|
this.atn = atn;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/**
|
||||||
/// Calculates the SLL(1) expected lookahead set for each outgoing transition
|
* Calculates the SLL(1) expected lookahead set for each outgoing transition
|
||||||
/// of an
|
* of an {@link ATNState}. The returned array has one element for each
|
||||||
/// <see cref="ATNState"/>
|
* outgoing transition in {@code s}. If the closure from transition
|
||||||
/// . The returned array has one element for each
|
* <em>i</em> leads to a semantic predicate before matching a symbol, the
|
||||||
/// outgoing transition in
|
* element at index <em>i</em> of the result will be {@code null}.
|
||||||
/// <paramref name="s"/>
|
*
|
||||||
/// . If the closure from transition
|
* @param s the ATN state
|
||||||
/// <em>i</em> leads to a semantic predicate before matching a symbol, the
|
* @return the expected symbols for each outgoing transition of {@code s}.
|
||||||
/// element at index <em>i</em> of the result will be
|
*/
|
||||||
/// <see langword="null"/>
|
|
||||||
/// .
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="s">the ATN state</param>
|
|
||||||
/// <returns>
|
|
||||||
/// the expected symbols for each outgoing transition of
|
|
||||||
/// <paramref name="s"/>
|
|
||||||
/// .
|
|
||||||
/// </returns>
|
|
||||||
[return: Nullable]
|
[return: Nullable]
|
||||||
public virtual IntervalSet[] GetDecisionLookahead(ATNState s)
|
public virtual IntervalSet[] GetDecisionLookahead(ATNState s)
|
||||||
{
|
{
|
||||||
|
@ -61,7 +48,7 @@ namespace Antlr4.Runtime.Atn
|
||||||
HashSet<ATNConfig> lookBusy = new HashSet<ATNConfig>();
|
HashSet<ATNConfig> lookBusy = new HashSet<ATNConfig>();
|
||||||
bool seeThruPreds = false;
|
bool seeThruPreds = false;
|
||||||
// fail to get lookahead upon pred
|
// fail to get lookahead upon pred
|
||||||
Look(s.Transition(alt).target, null, PredictionContext.EMPTY, look[alt], lookBusy, new BitSet(), seeThruPreds, false);
|
Look_(s.Transition(alt).target, null, PredictionContext.EMPTY, look[alt], lookBusy, new BitSet(), seeThruPreds, false);
|
||||||
// Wipe out lookahead for this alternative if we found nothing
|
// Wipe out lookahead for this alternative if we found nothing
|
||||||
// or we had a predicate when we !seeThruPreds
|
// or we had a predicate when we !seeThruPreds
|
||||||
if (look[alt].Count == 0 || look[alt].Contains(HitPred))
|
if (look[alt].Count == 0 || look[alt].Contains(HitPred))
|
||||||
|
@ -72,190 +59,88 @@ namespace Antlr4.Runtime.Atn
|
||||||
return look;
|
return look;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/**
|
||||||
/// Compute set of tokens that can follow
|
* Compute set of tokens that can follow {@code s} in the ATN in the
|
||||||
/// <paramref name="s"/>
|
* specified {@code ctx}.
|
||||||
/// in the ATN in the
|
*
|
||||||
/// specified
|
* <p>If {@code ctx} is {@code null} and the end of the rule containing
|
||||||
/// <paramref name="ctx"/>
|
* {@code s} is reached, {@link Token#EPSILON} is added to the result set.
|
||||||
/// .
|
* If {@code ctx} is not {@code null} and the end of the outermost rule is
|
||||||
/// <p>If
|
* reached, {@link Token#EOF} is added to the result set.</p>
|
||||||
/// <paramref name="ctx"/>
|
*
|
||||||
/// is
|
* @param s the ATN state
|
||||||
/// <see langword="null"/>
|
* @param ctx the complete parser context, or {@code null} if the context
|
||||||
/// and the end of the rule containing
|
* should be ignored
|
||||||
/// <paramref name="s"/>
|
*
|
||||||
/// is reached,
|
* @return The set of tokens that can follow {@code s} in the ATN in the
|
||||||
/// <see cref="TokenConstants.EPSILON"/>
|
* specified {@code ctx}.
|
||||||
/// is added to the result set.
|
*/
|
||||||
/// If
|
|
||||||
/// <paramref name="ctx"/>
|
|
||||||
/// is not
|
|
||||||
/// <see langword="null"/>
|
|
||||||
/// and the end of the outermost rule is
|
|
||||||
/// reached,
|
|
||||||
/// <see cref="TokenConstants.EOF"/>
|
|
||||||
/// is added to the result set.</p>
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="s">the ATN state</param>
|
|
||||||
/// <param name="ctx">
|
|
||||||
/// the complete parser context, or
|
|
||||||
/// <see langword="null"/>
|
|
||||||
/// if the context
|
|
||||||
/// should be ignored
|
|
||||||
/// </param>
|
|
||||||
/// <returns>
|
|
||||||
/// The set of tokens that can follow
|
|
||||||
/// <paramref name="s"/>
|
|
||||||
/// in the ATN in the
|
|
||||||
/// specified
|
|
||||||
/// <paramref name="ctx"/>
|
|
||||||
/// .
|
|
||||||
/// </returns>
|
|
||||||
[return: NotNull]
|
[return: NotNull]
|
||||||
public virtual IntervalSet Look(ATNState s, RuleContext ctx)
|
public virtual IntervalSet Look(ATNState s, RuleContext ctx)
|
||||||
{
|
{
|
||||||
return Look(s, null, ctx);
|
return Look(s, null, ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/**
|
||||||
/// Compute set of tokens that can follow
|
* Compute set of tokens that can follow {@code s} in the ATN in the
|
||||||
/// <paramref name="s"/>
|
* specified {@code ctx}.
|
||||||
/// in the ATN in the
|
*
|
||||||
/// specified
|
* <p>If {@code ctx} is {@code null} and the end of the rule containing
|
||||||
/// <paramref name="ctx"/>
|
* {@code s} is reached, {@link Token#EPSILON} is added to the result set.
|
||||||
/// .
|
* If {@code ctx} is not {@code null} and the end of the outermost rule is
|
||||||
/// <p>If
|
* reached, {@link Token#EOF} is added to the result set.</p>
|
||||||
/// <paramref name="ctx"/>
|
*
|
||||||
/// is
|
* @param s the ATN state
|
||||||
/// <see langword="null"/>
|
* @param stopState the ATN state to stop at. This can be a
|
||||||
/// and the end of the rule containing
|
* {@link BlockEndState} to detect epsilon paths through a closure.
|
||||||
/// <paramref name="s"/>
|
* @param ctx the complete parser context, or {@code null} if the context
|
||||||
/// is reached,
|
* should be ignored
|
||||||
/// <see cref="TokenConstants.EPSILON"/>
|
*
|
||||||
/// is added to the result set.
|
* @return The set of tokens that can follow {@code s} in the ATN in the
|
||||||
/// If
|
* specified {@code ctx}.
|
||||||
/// <paramref name="ctx"/>
|
*/
|
||||||
/// is not
|
|
||||||
/// <c>PredictionContext#EMPTY_LOCAL</c>
|
|
||||||
/// and the end of the outermost rule is
|
|
||||||
/// reached,
|
|
||||||
/// <see cref="TokenConstants.EOF"/>
|
|
||||||
/// is added to the result set.</p>
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="s">the ATN state</param>
|
|
||||||
/// <param name="stopState">
|
|
||||||
/// the ATN state to stop at. This can be a
|
|
||||||
/// <see cref="BlockEndState"/>
|
|
||||||
/// to detect epsilon paths through a closure.
|
|
||||||
/// </param>
|
|
||||||
/// <param name="ctx">
|
|
||||||
/// the complete parser context, or
|
|
||||||
/// <see langword="null"/>
|
|
||||||
/// if the context
|
|
||||||
/// should be ignored
|
|
||||||
/// </param>
|
|
||||||
/// <returns>
|
|
||||||
/// The set of tokens that can follow
|
|
||||||
/// <paramref name="s"/>
|
|
||||||
/// in the ATN in the
|
|
||||||
/// specified
|
|
||||||
/// <paramref name="ctx"/>
|
|
||||||
/// .
|
|
||||||
/// </returns>
|
|
||||||
[return: NotNull]
|
[return: NotNull]
|
||||||
public virtual IntervalSet Look(ATNState s, ATNState stopState, RuleContext ctx)
|
public virtual IntervalSet Look(ATNState s, ATNState stopState, RuleContext ctx)
|
||||||
{
|
{
|
||||||
IntervalSet r = new IntervalSet();
|
IntervalSet r = new IntervalSet();
|
||||||
bool seeThruPreds = true;
|
bool seeThruPreds = true;
|
||||||
PredictionContext lookContext = ctx != null ? PredictionContext.FromRuleContext(s.atn, ctx) : null;
|
PredictionContext lookContext = ctx != null ? PredictionContext.FromRuleContext(s.atn, ctx) : null;
|
||||||
Look(s, stopState, lookContext, r, new HashSet<ATNConfig>(), new BitSet(), seeThruPreds, true);
|
Look_(s, stopState, lookContext, r, new HashSet<ATNConfig>(), new BitSet(), seeThruPreds, true);
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/**
|
||||||
/// Compute set of tokens that can follow
|
* Compute set of tokens that can follow {@code s} in the ATN in the
|
||||||
/// <paramref name="s"/>
|
* specified {@code ctx}.
|
||||||
/// in the ATN in the
|
*
|
||||||
/// specified
|
* <p>If {@code ctx} is {@code null} and {@code stopState} or the end of the
|
||||||
/// <paramref name="ctx"/>
|
* rule containing {@code s} is reached, {@link Token#EPSILON} is added to
|
||||||
/// .
|
* the result set. If {@code ctx} is not {@code null} and {@code addEOF} is
|
||||||
/// <p/>
|
* {@code true} and {@code stopState} or the end of the outermost rule is
|
||||||
/// If
|
* reached, {@link Token#EOF} is added to the result set.</p>
|
||||||
/// <paramref name="ctx"/>
|
*
|
||||||
/// is
|
* @param s the ATN state.
|
||||||
/// <see cref="PredictionContext.EMPTY"/>
|
* @param stopState the ATN state to stop at. This can be a
|
||||||
/// and
|
* {@link BlockEndState} to detect epsilon paths through a closure.
|
||||||
/// <paramref name="stopState"/>
|
* @param ctx The outer context, or {@code null} if the outer context should
|
||||||
/// or the end of the rule containing
|
* not be used.
|
||||||
/// <paramref name="s"/>
|
* @param look The result lookahead set.
|
||||||
/// is reached,
|
* @param lookBusy A set used for preventing epsilon closures in the ATN
|
||||||
/// <see cref="TokenConstants.EPSILON"/>
|
* from causing a stack overflow. Outside code should pass
|
||||||
/// is added to the result set. If
|
* {@code new HashSet<ATNConfig>} for this argument.
|
||||||
/// <paramref name="ctx"/>
|
* @param calledRuleStack A set used for preventing left recursion in the
|
||||||
/// is not
|
* ATN from causing a stack overflow. Outside code should pass
|
||||||
/// <see cref="PredictionContext.EMPTY"/>
|
* {@code new BitSet()} for this argument.
|
||||||
/// and
|
* @param seeThruPreds {@code true} to true semantic predicates as
|
||||||
/// <paramref name="addEOF"/>
|
* implicitly {@code true} and "see through them", otherwise {@code false}
|
||||||
/// is
|
* to treat semantic predicates as opaque and add {@link #HIT_PRED} to the
|
||||||
/// <see langword="true"/>
|
* result if one is encountered.
|
||||||
/// and
|
* @param addEOF Add {@link Token#EOF} to the result if the end of the
|
||||||
/// <paramref name="stopState"/>
|
* outermost context is reached. This parameter has no effect if {@code ctx}
|
||||||
/// or the end of the outermost rule is reached,
|
* is {@code null}.
|
||||||
/// <see cref="TokenConstants.EOF"/>
|
*/
|
||||||
/// is added to the result set.
|
protected internal virtual void Look_(ATNState s, ATNState stopState, PredictionContext ctx, IntervalSet look, HashSet<ATNConfig> lookBusy, BitSet calledRuleStack, bool seeThruPreds, bool addEOF)
|
||||||
/// </summary>
|
|
||||||
/// <param name="s">the ATN state.</param>
|
|
||||||
/// <param name="stopState">
|
|
||||||
/// the ATN state to stop at. This can be a
|
|
||||||
/// <see cref="BlockEndState"/>
|
|
||||||
/// to detect epsilon paths through a closure.
|
|
||||||
/// </param>
|
|
||||||
/// <param name="ctx">
|
|
||||||
/// The outer context, or
|
|
||||||
/// <see cref="PredictionContext.EMPTY"/>
|
|
||||||
/// if
|
|
||||||
/// the outer context should not be used.
|
|
||||||
/// </param>
|
|
||||||
/// <param name="look">The result lookahead set.</param>
|
|
||||||
/// <param name="lookBusy">
|
|
||||||
/// A set used for preventing epsilon closures in the ATN
|
|
||||||
/// from causing a stack overflow. Outside code should pass
|
|
||||||
/// <c>new HashSet<ATNConfig></c>
|
|
||||||
/// for this argument.
|
|
||||||
/// </param>
|
|
||||||
/// <param name="calledRuleStack">
|
|
||||||
/// A set used for preventing left recursion in the
|
|
||||||
/// ATN from causing a stack overflow. Outside code should pass
|
|
||||||
/// <c>new BitSet()</c>
|
|
||||||
/// for this argument.
|
|
||||||
/// </param>
|
|
||||||
/// <param name="seeThruPreds">
|
|
||||||
///
|
|
||||||
/// <see langword="true"/>
|
|
||||||
/// to true semantic predicates as
|
|
||||||
/// implicitly
|
|
||||||
/// <see langword="true"/>
|
|
||||||
/// and "see through them", otherwise
|
|
||||||
/// <see langword="false"/>
|
|
||||||
/// to treat semantic predicates as opaque and add
|
|
||||||
/// <see cref="HitPred"/>
|
|
||||||
/// to the
|
|
||||||
/// result if one is encountered.
|
|
||||||
/// </param>
|
|
||||||
/// <param name="addEOF">
|
|
||||||
/// Add
|
|
||||||
/// <see cref="TokenConstants.EOF"/>
|
|
||||||
/// to the result if the end of the
|
|
||||||
/// outermost context is reached. This parameter has no effect if
|
|
||||||
/// <paramref name="ctx"/>
|
|
||||||
/// is
|
|
||||||
/// <see cref="PredictionContext.EMPTY"/>
|
|
||||||
/// .
|
|
||||||
/// </param>
|
|
||||||
protected internal virtual void Look(ATNState s, ATNState stopState, PredictionContext ctx, IntervalSet look, HashSet<ATNConfig> lookBusy, BitSet calledRuleStack, bool seeThruPreds, bool addEOF)
|
|
||||||
{
|
{
|
||||||
// System.out.println("_LOOK("+s.stateNumber+", ctx="+ctx);
|
|
||||||
ATNConfig c = new ATNConfig(s, 0, ctx);
|
ATNConfig c = new ATNConfig(s, 0, ctx);
|
||||||
if (!lookBusy.Add(c))
|
if (!lookBusy.Add(c))
|
||||||
{
|
{
|
||||||
|
@ -268,50 +153,51 @@ namespace Antlr4.Runtime.Atn
|
||||||
look.Add(TokenConstants.EPSILON);
|
look.Add(TokenConstants.EPSILON);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
else if (ctx.IsEmpty && addEOF) {
|
|
||||||
look.Add(TokenConstants.EOF);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (s is RuleStopState)
|
|
||||||
{
|
|
||||||
if (ctx == null)
|
|
||||||
{
|
|
||||||
look.Add(TokenConstants.EPSILON);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
else if (ctx.IsEmpty && addEOF)
|
else if (ctx.IsEmpty && addEOF)
|
||||||
{
|
{
|
||||||
look.Add(TokenConstants.EOF);
|
look.Add(TokenConstants.EOF);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (ctx != PredictionContext.EMPTY)
|
}
|
||||||
{
|
if (s is RuleStopState)
|
||||||
for (int i = 0; i < ctx.Size; i++)
|
{
|
||||||
{
|
if (ctx == null)
|
||||||
ATNState returnState = atn.states[ctx.GetReturnState(i)];
|
{
|
||||||
bool removed = calledRuleStack.Get(returnState.ruleIndex);
|
look.Add(TokenConstants.EPSILON);
|
||||||
try
|
return;
|
||||||
{
|
}
|
||||||
calledRuleStack.Clear(returnState.ruleIndex);
|
else if (ctx.IsEmpty && addEOF)
|
||||||
Look(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
|
{
|
||||||
}
|
look.Add(TokenConstants.EOF);
|
||||||
finally
|
return;
|
||||||
{
|
}
|
||||||
if (removed)
|
if (ctx != PredictionContext.EMPTY)
|
||||||
{
|
{
|
||||||
calledRuleStack.Set(returnState.ruleIndex);
|
bool removed = calledRuleStack.Get(s.ruleIndex);
|
||||||
}
|
try
|
||||||
}
|
{
|
||||||
}
|
calledRuleStack.Clear(s.ruleIndex);
|
||||||
return;
|
for (int i = 0; i < ctx.Size; i++)
|
||||||
}
|
{
|
||||||
|
ATNState returnState = atn.states[ctx.GetReturnState(i)];
|
||||||
|
Look_(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (removed)
|
||||||
|
{
|
||||||
|
calledRuleStack.Set(s.ruleIndex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
int n = s.NumberOfTransitions;
|
int n = s.NumberOfTransitions;
|
||||||
for (int i_1 = 0; i_1 < n; i_1++)
|
for (int i_1 = 0; i_1 < n; i_1++)
|
||||||
{
|
{
|
||||||
Transition t = s.Transition(i_1);
|
Transition t = s.Transition(i_1);
|
||||||
if (t is RuleTransition)
|
if (t.GetType() == typeof(RuleTransition))
|
||||||
{
|
{
|
||||||
RuleTransition ruleTransition = (RuleTransition)t;
|
RuleTransition ruleTransition = (RuleTransition)t;
|
||||||
if (calledRuleStack.Get(ruleTransition.ruleIndex))
|
if (calledRuleStack.Get(ruleTransition.ruleIndex))
|
||||||
|
@ -322,51 +208,42 @@ namespace Antlr4.Runtime.Atn
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
calledRuleStack.Set(ruleTransition.target.ruleIndex);
|
calledRuleStack.Set(ruleTransition.target.ruleIndex);
|
||||||
Look(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
|
Look_(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
calledRuleStack.Clear(ruleTransition.target.ruleIndex);
|
calledRuleStack.Clear(ruleTransition.target.ruleIndex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else if (t is AbstractPredicateTransition)
|
||||||
{
|
{
|
||||||
if (t is AbstractPredicateTransition)
|
if (seeThruPreds)
|
||||||
{
|
{
|
||||||
if (seeThruPreds)
|
Look_(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
|
||||||
{
|
|
||||||
Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
look.Add(HitPred);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
if (t.IsEpsilon)
|
look.Add(HitPred);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (t.IsEpsilon)
|
||||||
|
{
|
||||||
|
Look_(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
|
||||||
|
}
|
||||||
|
else if (t.GetType() == typeof(WildcardTransition))
|
||||||
|
{
|
||||||
|
look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
IntervalSet set = t.Label;
|
||||||
|
if (set != null)
|
||||||
|
{
|
||||||
|
if (t is NotSetTransition)
|
||||||
{
|
{
|
||||||
Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
|
set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType));
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
if (t is WildcardTransition)
|
|
||||||
{
|
|
||||||
look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType));
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
IntervalSet set = t.Label;
|
|
||||||
if (set != null)
|
|
||||||
{
|
|
||||||
if (t is NotSetTransition)
|
|
||||||
{
|
|
||||||
set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType));
|
|
||||||
}
|
|
||||||
look.AddAll(set);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
look.AddAll(set);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -42,6 +42,21 @@ namespace Antlr4.Runtime
|
||||||
|
|
||||||
protected internal IntervalSet lastErrorStates;
|
protected internal IntervalSet lastErrorStates;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This field is used to propagate information about the lookahead following
|
||||||
|
* the previous match. Since prediction prefers completing the current rule
|
||||||
|
* to error recovery efforts, error reporting may occur later than the
|
||||||
|
* original point where it was discoverable. The original context is used to
|
||||||
|
* compute the true expected sets as though the reporting occurred as early
|
||||||
|
* as possible.
|
||||||
|
*/
|
||||||
|
protected ParserRuleContext nextTokensContext;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @see #nextTokensContext
|
||||||
|
*/
|
||||||
|
protected int nextTokensState;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// <inheritDoc/>
|
/// <inheritDoc/>
|
||||||
/// <p>The default implementation simply calls
|
/// <p>The default implementation simply calls
|
||||||
|
@ -264,8 +279,22 @@ namespace Antlr4.Runtime
|
||||||
int la = tokens.LA(1);
|
int la = tokens.LA(1);
|
||||||
// try cheaper subset first; might get lucky. seems to shave a wee bit off
|
// try cheaper subset first; might get lucky. seems to shave a wee bit off
|
||||||
var nextTokens = recognizer.Atn.NextTokens(s);
|
var nextTokens = recognizer.Atn.NextTokens(s);
|
||||||
if (nextTokens.Contains(TokenConstants.EPSILON) || nextTokens.Contains(la))
|
if (nextTokens.Contains(la))
|
||||||
{
|
{
|
||||||
|
nextTokensContext = null;
|
||||||
|
nextTokensState = ATNState.InvalidStateNumber;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nextTokens.Contains(TokenConstants.EPSILON))
|
||||||
|
{
|
||||||
|
if (nextTokensContext == null)
|
||||||
|
{
|
||||||
|
// It's possible the next token won't match; information tracked
|
||||||
|
// by sync is restricted for performance.
|
||||||
|
nextTokensContext = recognizer.Context;
|
||||||
|
nextTokensState = recognizer.State;
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
switch (s.StateType)
|
switch (s.StateType)
|
||||||
|
|
|
@ -100,18 +100,16 @@ void LL1Analyzer::_LOOK(ATNState *s, ATNState *stopState, Ref<PredictionContext>
|
||||||
}
|
}
|
||||||
|
|
||||||
if (ctx != PredictionContext::EMPTY) {
|
if (ctx != PredictionContext::EMPTY) {
|
||||||
// run thru all possible stack tops in ctx
|
bool removed = calledRuleStack.test(s->ruleIndex);
|
||||||
|
calledRuleStack[s->ruleIndex] = false;
|
||||||
|
auto onExit = finally([removed, &calledRuleStack, s] {
|
||||||
|
if (removed) {
|
||||||
|
calledRuleStack.set(s->ruleIndex);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
// run thru all possible stack tops in ctx
|
||||||
for (size_t i = 0; i < ctx->size(); i++) {
|
for (size_t i = 0; i < ctx->size(); i++) {
|
||||||
ATNState *returnState = _atn.states[ctx->getReturnState(i)];
|
ATNState *returnState = _atn.states[ctx->getReturnState(i)];
|
||||||
|
|
||||||
bool removed = calledRuleStack.test(returnState->ruleIndex);
|
|
||||||
auto onExit = finally([removed, &calledRuleStack, returnState] {
|
|
||||||
if (removed) {
|
|
||||||
calledRuleStack.set(returnState->ruleIndex);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
calledRuleStack[returnState->ruleIndex] = false;
|
|
||||||
_LOOK(returnState, stopState, ctx->getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
|
_LOOK(returnState, stopState, ctx->getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
|
|
|
@ -112,16 +112,6 @@ func (la *LL1Analyzer) Look(s, stopState ATNState, ctx RuleContext) *IntervalSet
|
||||||
func (la *LL1Analyzer) look2(s, stopState ATNState, ctx PredictionContext, look *IntervalSet, lookBusy *Set, calledRuleStack *BitSet, seeThruPreds, addEOF bool, i int) {
|
func (la *LL1Analyzer) look2(s, stopState ATNState, ctx PredictionContext, look *IntervalSet, lookBusy *Set, calledRuleStack *BitSet, seeThruPreds, addEOF bool, i int) {
|
||||||
|
|
||||||
returnState := la.atn.states[ctx.getReturnState(i)]
|
returnState := la.atn.states[ctx.getReturnState(i)]
|
||||||
|
|
||||||
removed := calledRuleStack.contains(returnState.GetRuleIndex())
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
if removed {
|
|
||||||
calledRuleStack.add(returnState.GetRuleIndex())
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
calledRuleStack.remove(returnState.GetRuleIndex())
|
|
||||||
la.look1(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
|
la.look1(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -158,6 +148,13 @@ func (la *LL1Analyzer) look1(s, stopState ATNState, ctx PredictionContext, look
|
||||||
}
|
}
|
||||||
|
|
||||||
if ctx != BasePredictionContextEMPTY {
|
if ctx != BasePredictionContextEMPTY {
|
||||||
|
removed := calledRuleStack.contains(s.GetRuleIndex())
|
||||||
|
defer func() {
|
||||||
|
if removed {
|
||||||
|
calledRuleStack.add(s.GetRuleIndex())
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
calledRuleStack.remove(s.GetRuleIndex())
|
||||||
// run thru all possible stack tops in ctx
|
// run thru all possible stack tops in ctx
|
||||||
for i := 0; i < ctx.length(); i++ {
|
for i := 0; i < ctx.length(); i++ {
|
||||||
returnState := la.atn.states[ctx.getReturnState(i)]
|
returnState := la.atn.states[ctx.getReturnState(i)]
|
||||||
|
|
|
@ -128,17 +128,17 @@ class LL1Analyzer {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (ctx !== PredictionContext.EMPTY) {
|
if (ctx !== PredictionContext.EMPTY) {
|
||||||
// run thru all possible stack tops in ctx
|
const removed = calledRuleStack.contains(s.ruleIndex);
|
||||||
for(let i=0; i<ctx.length; i++) {
|
try {
|
||||||
const returnState = this.atn.states[ctx.getReturnState(i)];
|
calledRuleStack.remove(s.ruleIndex);
|
||||||
const removed = calledRuleStack.contains(returnState.ruleIndex);
|
// run thru all possible stack tops in ctx
|
||||||
try {
|
for (let i = 0; i < ctx.length; i++) {
|
||||||
calledRuleStack.remove(returnState.ruleIndex);
|
const returnState = this.atn.states[ctx.getReturnState(i)];
|
||||||
this._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
|
this._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
|
||||||
} finally {
|
}
|
||||||
if (removed) {
|
}finally {
|
||||||
calledRuleStack.add(returnState.ruleIndex);
|
if (removed) {
|
||||||
}
|
calledRuleStack.add(s.ruleIndex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
|
|
|
@ -55,6 +55,8 @@ class DefaultErrorStrategy extends ErrorStrategy {
|
||||||
*/
|
*/
|
||||||
this.lastErrorIndex = -1;
|
this.lastErrorIndex = -1;
|
||||||
this.lastErrorStates = null;
|
this.lastErrorStates = null;
|
||||||
|
this.nextTokensContext = null;
|
||||||
|
this.nextTokenState = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -216,11 +218,21 @@ class DefaultErrorStrategy extends ErrorStrategy {
|
||||||
if (this.inErrorRecoveryMode(recognizer)) {
|
if (this.inErrorRecoveryMode(recognizer)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const s = recognizer._interp.atn.states[recognizer.state]
|
const s = recognizer._interp.atn.states[recognizer.state];
|
||||||
const la = recognizer.getTokenStream().LA(1)
|
const la = recognizer.getTokenStream().LA(1);
|
||||||
// try cheaper subset first; might get lucky. seems to shave a wee bit off
|
// try cheaper subset first; might get lucky. seems to shave a wee bit off
|
||||||
const nextTokens = recognizer.atn.nextTokens(s)
|
const nextTokens = recognizer.atn.nextTokens(s);
|
||||||
if (nextTokens.contains(Token.EPSILON) || nextTokens.contains(la)) {
|
if(nextTokens.contains(la)) {
|
||||||
|
this.nextTokensContext = null;
|
||||||
|
this.nextTokenState = ATNState.INVALID_STATE_NUMBER;
|
||||||
|
return;
|
||||||
|
} else if (nextTokens.contains(Token.EPSILON)) {
|
||||||
|
if(this.nextTokensContext === null) {
|
||||||
|
// It's possible the next token won't match information tracked
|
||||||
|
// by sync is restricted for performance.
|
||||||
|
this.nextTokensContext = recognizer._ctx;
|
||||||
|
this.nextTokensState = recognizer._stateNumber;
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
switch (s.stateType) {
|
switch (s.stateType) {
|
||||||
|
|
|
@ -129,16 +129,16 @@ class LL1Analyzer (object):
|
||||||
return
|
return
|
||||||
|
|
||||||
if ctx != PredictionContext.EMPTY:
|
if ctx != PredictionContext.EMPTY:
|
||||||
# run thru all possible stack tops in ctx
|
removed = s.ruleIndex in calledRuleStack
|
||||||
for i in range(0, len(ctx)):
|
try:
|
||||||
returnState = self.atn.states[ctx.getReturnState(i)]
|
calledRuleStack.discard(s.ruleIndex)
|
||||||
removed = returnState.ruleIndex in calledRuleStack
|
# run thru all possible stack tops in ctx
|
||||||
try:
|
for i in range(0, len(ctx)):
|
||||||
calledRuleStack.discard(returnState.ruleIndex)
|
returnState = self.atn.states[ctx.getReturnState(i)]
|
||||||
self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
|
self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
|
||||||
finally:
|
finally:
|
||||||
if removed:
|
if removed:
|
||||||
calledRuleStack.add(returnState.ruleIndex)
|
calledRuleStack.add(s.ruleIndex)
|
||||||
return
|
return
|
||||||
|
|
||||||
for t in s.transitions:
|
for t in s.transitions:
|
||||||
|
@ -163,8 +163,8 @@ class LL1Analyzer (object):
|
||||||
elif type(t) == WildcardTransition:
|
elif type(t) == WildcardTransition:
|
||||||
look.addRange( Interval(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType + 1) )
|
look.addRange( Interval(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType + 1) )
|
||||||
else:
|
else:
|
||||||
set = t.label
|
set_ = t.label
|
||||||
if set is not None:
|
if set_ is not None:
|
||||||
if isinstance(t, NotSetTransition):
|
if isinstance(t, NotSetTransition):
|
||||||
set = set.complement(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType)
|
set_ = set_.complement(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType)
|
||||||
look.addSet(set)
|
look.addSet(set_)
|
||||||
|
|
|
@ -53,6 +53,8 @@ class DefaultErrorStrategy(ErrorStrategy):
|
||||||
#
|
#
|
||||||
self.lastErrorIndex = -1
|
self.lastErrorIndex = -1
|
||||||
self.lastErrorStates = None
|
self.lastErrorStates = None
|
||||||
|
self.nextTokensContext = None
|
||||||
|
self.nextTokenState = 0
|
||||||
|
|
||||||
# <p>The default implementation simply calls {@link #endErrorCondition} to
|
# <p>The default implementation simply calls {@link #endErrorCondition} to
|
||||||
# ensure that the handler is not in error recovery mode.</p>
|
# ensure that the handler is not in error recovery mode.</p>
|
||||||
|
@ -203,7 +205,16 @@ class DefaultErrorStrategy(ErrorStrategy):
|
||||||
la = recognizer.getTokenStream().LA(1)
|
la = recognizer.getTokenStream().LA(1)
|
||||||
# try cheaper subset first; might get lucky. seems to shave a wee bit off
|
# try cheaper subset first; might get lucky. seems to shave a wee bit off
|
||||||
nextTokens = recognizer.atn.nextTokens(s)
|
nextTokens = recognizer.atn.nextTokens(s)
|
||||||
if Token.EPSILON in nextTokens or la in nextTokens:
|
if la in nextTokens:
|
||||||
|
self.nextTokensContext = None
|
||||||
|
self.nextTokenState = ATNState.INVALID_STATE_NUMBER
|
||||||
|
return
|
||||||
|
elif Token.EPSILON in nextTokens:
|
||||||
|
if self.nextTokensContext is None:
|
||||||
|
# It's possible the next token won't match information tracked
|
||||||
|
# by sync is restricted for performance.
|
||||||
|
self.nextTokensContext = recognizer._ctx
|
||||||
|
self.nextTokensState = recognizer._stateNumber
|
||||||
return
|
return
|
||||||
|
|
||||||
if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START,
|
if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START,
|
||||||
|
|
|
@ -132,16 +132,16 @@ class LL1Analyzer (object):
|
||||||
return
|
return
|
||||||
|
|
||||||
if ctx != PredictionContext.EMPTY:
|
if ctx != PredictionContext.EMPTY:
|
||||||
# run thru all possible stack tops in ctx
|
removed = s.ruleIndex in calledRuleStack
|
||||||
for i in range(0, len(ctx)):
|
try:
|
||||||
returnState = self.atn.states[ctx.getReturnState(i)]
|
calledRuleStack.discard(s.ruleIndex)
|
||||||
removed = returnState.ruleIndex in calledRuleStack
|
# run thru all possible stack tops in ctx
|
||||||
try:
|
for i in range(0, len(ctx)):
|
||||||
calledRuleStack.discard(returnState.ruleIndex)
|
returnState = self.atn.states[ctx.getReturnState(i)]
|
||||||
self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
|
self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
|
||||||
finally:
|
finally:
|
||||||
if removed:
|
if removed:
|
||||||
calledRuleStack.add(returnState.ruleIndex)
|
calledRuleStack.add(s.ruleIndex)
|
||||||
return
|
return
|
||||||
|
|
||||||
for t in s.transitions:
|
for t in s.transitions:
|
||||||
|
|
|
@ -58,6 +58,8 @@ class DefaultErrorStrategy(ErrorStrategy):
|
||||||
#
|
#
|
||||||
self.lastErrorIndex = -1
|
self.lastErrorIndex = -1
|
||||||
self.lastErrorStates = None
|
self.lastErrorStates = None
|
||||||
|
self.nextTokensContext = None
|
||||||
|
self.nextTokenState = 0
|
||||||
|
|
||||||
# <p>The default implementation simply calls {@link #endErrorCondition} to
|
# <p>The default implementation simply calls {@link #endErrorCondition} to
|
||||||
# ensure that the handler is not in error recovery mode.</p>
|
# ensure that the handler is not in error recovery mode.</p>
|
||||||
|
@ -208,7 +210,16 @@ class DefaultErrorStrategy(ErrorStrategy):
|
||||||
la = recognizer.getTokenStream().LA(1)
|
la = recognizer.getTokenStream().LA(1)
|
||||||
# try cheaper subset first; might get lucky. seems to shave a wee bit off
|
# try cheaper subset first; might get lucky. seems to shave a wee bit off
|
||||||
nextTokens = recognizer.atn.nextTokens(s)
|
nextTokens = recognizer.atn.nextTokens(s)
|
||||||
if Token.EPSILON in nextTokens or la in nextTokens:
|
if la in nextTokens:
|
||||||
|
self.nextTokensContext = None
|
||||||
|
self.nextTokenState = ATNState.INVALID_STATE_NUMBER
|
||||||
|
return
|
||||||
|
elif Token.EPSILON in nextTokens:
|
||||||
|
if self.nextTokensContext is None:
|
||||||
|
# It's possible the next token won't match information tracked
|
||||||
|
# by sync is restricted for performance.
|
||||||
|
self.nextTokensContext = recognizer._ctx
|
||||||
|
self.nextTokensState = recognizer._stateNumber
|
||||||
return
|
return
|
||||||
|
|
||||||
if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START,
|
if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START,
|
||||||
|
|
|
@ -169,16 +169,18 @@ public class LL1Analyzer {
|
||||||
}
|
}
|
||||||
|
|
||||||
if ctx != PredictionContext.EMPTY {
|
if ctx != PredictionContext.EMPTY {
|
||||||
|
let removed = try! calledRuleStack.get(s.ruleIndex!)
|
||||||
|
try! calledRuleStack.clear(s.ruleIndex!)
|
||||||
|
defer {
|
||||||
|
if removed {
|
||||||
|
try! calledRuleStack.set(s.ruleIndex!)
|
||||||
|
}
|
||||||
|
}
|
||||||
// run thru all possible stack tops in ctx
|
// run thru all possible stack tops in ctx
|
||||||
let length = ctx.size()
|
let length = ctx.size()
|
||||||
for i in 0..<length {
|
for i in 0..<length {
|
||||||
let returnState = atn.states[(ctx.getReturnState(i))]!
|
let returnState = atn.states[(ctx.getReturnState(i))]!
|
||||||
let removed = try! calledRuleStack.get(returnState.ruleIndex!)
|
|
||||||
try! calledRuleStack.clear(returnState.ruleIndex!)
|
|
||||||
_LOOK(returnState, stopState, ctx.getParent(i), look, &lookBusy, calledRuleStack, seeThruPreds, addEOF)
|
_LOOK(returnState, stopState, ctx.getParent(i), look, &lookBusy, calledRuleStack, seeThruPreds, addEOF)
|
||||||
if removed {
|
|
||||||
try! calledRuleStack.set(returnState.ruleIndex!)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue