From 53b65c015c9aa3c910c059334040aa6bf34fbd99 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 23 Jan 2021 17:04:01 +0800 Subject: [PATCH 1/8] fix #3042 in C# runtime --- .gitignore | 1 + runtime/CSharp/Atn/LL1Analyzer.cs | 401 +++++++++---------------- runtime/CSharp/DefaultErrorStrategy.cs | 31 +- 3 files changed, 170 insertions(+), 263 deletions(-) diff --git a/.gitignore b/.gitignore index 81e058979..00cfcd826 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,4 @@ javac-services.0.log.lck # Don't ignore python tests !runtime/Python3/test/ +Antlr4.sln diff --git a/runtime/CSharp/Atn/LL1Analyzer.cs b/runtime/CSharp/Atn/LL1Analyzer.cs index 06bcebb33..e67933412 100644 --- a/runtime/CSharp/Atn/LL1Analyzer.cs +++ b/runtime/CSharp/Atn/LL1Analyzer.cs @@ -3,7 +3,6 @@ * can be found in the LICENSE.txt file in the project root. */ using System.Collections.Generic; -using Antlr4.Runtime.Atn; using Antlr4.Runtime.Misc; using Antlr4.Runtime.Sharpen; @@ -11,12 +10,9 @@ namespace Antlr4.Runtime.Atn { public class LL1Analyzer { - /// - /// Special value added to the lookahead sets to indicate that we hit - /// a predicate during analysis if - /// seeThruPreds==false - /// . - /// + /** Special value added to the lookahead sets to indicate that we hit + * a predicate during analysis if {@code seeThruPreds==false}. + */ public const int HitPred = TokenConstants.InvalidType; [NotNull] @@ -27,25 +23,16 @@ namespace Antlr4.Runtime.Atn this.atn = atn; } - /// - /// Calculates the SLL(1) expected lookahead set for each outgoing transition - /// of an - /// - /// . The returned array has one element for each - /// outgoing transition in - /// - /// . If the closure from transition - /// i leads to a semantic predicate before matching a symbol, the - /// element at index i of the result will be - /// - /// . - /// - /// the ATN state - /// - /// the expected symbols for each outgoing transition of - /// - /// . - /// + /** + * Calculates the SLL(1) expected lookahead set for each outgoing transition + * of an {@link ATNState}. The returned array has one element for each + * outgoing transition in {@code s}. If the closure from transition + * i leads to a semantic predicate before matching a symbol, the + * element at index i of the result will be {@code null}. + * + * @param s the ATN state + * @return the expected symbols for each outgoing transition of {@code s}. + */ [return: Nullable] public virtual IntervalSet[] GetDecisionLookahead(ATNState s) { @@ -61,7 +48,7 @@ namespace Antlr4.Runtime.Atn HashSet lookBusy = new HashSet(); bool seeThruPreds = false; // fail to get lookahead upon pred - Look(s.Transition(alt).target, null, PredictionContext.EMPTY, look[alt], lookBusy, new BitSet(), seeThruPreds, false); + Look_(s.Transition(alt).target, null, PredictionContext.EMPTY, look[alt], lookBusy, new BitSet(), seeThruPreds, false); // Wipe out lookahead for this alternative if we found nothing // or we had a predicate when we !seeThruPreds if (look[alt].Count == 0 || look[alt].Contains(HitPred)) @@ -72,190 +59,88 @@ namespace Antlr4.Runtime.Atn return look; } - /// - /// Compute set of tokens that can follow - /// - /// in the ATN in the - /// specified - /// - /// . - ///

If - /// - /// is - /// - /// and the end of the rule containing - /// - /// is reached, - /// - /// is added to the result set. - /// If - /// - /// is not - /// - /// and the end of the outermost rule is - /// reached, - /// - /// is added to the result set.

- /// - /// the ATN state - /// - /// the complete parser context, or - /// - /// if the context - /// should be ignored - /// - /// - /// The set of tokens that can follow - /// - /// in the ATN in the - /// specified - /// - /// . - /// + /** + * Compute set of tokens that can follow {@code s} in the ATN in the + * specified {@code ctx}. + * + *

If {@code ctx} is {@code null} and the end of the rule containing + * {@code s} is reached, {@link Token#EPSILON} is added to the result set. + * If {@code ctx} is not {@code null} and the end of the outermost rule is + * reached, {@link Token#EOF} is added to the result set.

+ * + * @param s the ATN state + * @param ctx the complete parser context, or {@code null} if the context + * should be ignored + * + * @return The set of tokens that can follow {@code s} in the ATN in the + * specified {@code ctx}. + */ [return: NotNull] public virtual IntervalSet Look(ATNState s, RuleContext ctx) { return Look(s, null, ctx); } - /// - /// Compute set of tokens that can follow - /// - /// in the ATN in the - /// specified - /// - /// . - ///

If - /// - /// is - /// - /// and the end of the rule containing - /// - /// is reached, - /// - /// is added to the result set. - /// If - /// - /// is not - /// PredictionContext#EMPTY_LOCAL - /// and the end of the outermost rule is - /// reached, - /// - /// is added to the result set.

- /// - /// the ATN state - /// - /// the ATN state to stop at. This can be a - /// - /// to detect epsilon paths through a closure. - /// - /// - /// the complete parser context, or - /// - /// if the context - /// should be ignored - /// - /// - /// The set of tokens that can follow - /// - /// in the ATN in the - /// specified - /// - /// . - /// + /** + * Compute set of tokens that can follow {@code s} in the ATN in the + * specified {@code ctx}. + * + *

If {@code ctx} is {@code null} and the end of the rule containing + * {@code s} is reached, {@link Token#EPSILON} is added to the result set. + * If {@code ctx} is not {@code null} and the end of the outermost rule is + * reached, {@link Token#EOF} is added to the result set.

+ * + * @param s the ATN state + * @param stopState the ATN state to stop at. This can be a + * {@link BlockEndState} to detect epsilon paths through a closure. + * @param ctx the complete parser context, or {@code null} if the context + * should be ignored + * + * @return The set of tokens that can follow {@code s} in the ATN in the + * specified {@code ctx}. + */ [return: NotNull] public virtual IntervalSet Look(ATNState s, ATNState stopState, RuleContext ctx) { IntervalSet r = new IntervalSet(); bool seeThruPreds = true; - PredictionContext lookContext = ctx != null ? PredictionContext.FromRuleContext(s.atn, ctx) : null; - Look(s, stopState, lookContext, r, new HashSet(), new BitSet(), seeThruPreds, true); + PredictionContext lookContext = ctx != null ? PredictionContext.FromRuleContext(s.atn, ctx) : null; + Look_(s, stopState, lookContext, r, new HashSet(), new BitSet(), seeThruPreds, true); return r; } - /// - /// Compute set of tokens that can follow - /// - /// in the ATN in the - /// specified - /// - /// . - ///

- /// If - /// - /// is - /// - /// and - /// - /// or the end of the rule containing - /// - /// is reached, - /// - /// is added to the result set. If - /// - /// is not - /// - /// and - /// - /// is - /// - /// and - /// - /// or the end of the outermost rule is reached, - /// - /// is added to the result set. - ///

- /// the ATN state. - /// - /// the ATN state to stop at. This can be a - /// - /// to detect epsilon paths through a closure. - /// - /// - /// The outer context, or - /// - /// if - /// the outer context should not be used. - /// - /// The result lookahead set. - /// - /// A set used for preventing epsilon closures in the ATN - /// from causing a stack overflow. Outside code should pass - /// new HashSet<ATNConfig> - /// for this argument. - /// - /// - /// A set used for preventing left recursion in the - /// ATN from causing a stack overflow. Outside code should pass - /// new BitSet() - /// for this argument. - /// - /// - /// - /// - /// to true semantic predicates as - /// implicitly - /// - /// and "see through them", otherwise - /// - /// to treat semantic predicates as opaque and add - /// - /// to the - /// result if one is encountered. - /// - /// - /// Add - /// - /// to the result if the end of the - /// outermost context is reached. This parameter has no effect if - /// - /// is - /// - /// . - /// - protected internal virtual void Look(ATNState s, ATNState stopState, PredictionContext ctx, IntervalSet look, HashSet lookBusy, BitSet calledRuleStack, bool seeThruPreds, bool addEOF) + /** + * Compute set of tokens that can follow {@code s} in the ATN in the + * specified {@code ctx}. + * + *

If {@code ctx} is {@code null} and {@code stopState} or the end of the + * rule containing {@code s} is reached, {@link Token#EPSILON} is added to + * the result set. If {@code ctx} is not {@code null} and {@code addEOF} is + * {@code true} and {@code stopState} or the end of the outermost rule is + * reached, {@link Token#EOF} is added to the result set.

+ * + * @param s the ATN state. + * @param stopState the ATN state to stop at. This can be a + * {@link BlockEndState} to detect epsilon paths through a closure. + * @param ctx The outer context, or {@code null} if the outer context should + * not be used. + * @param look The result lookahead set. + * @param lookBusy A set used for preventing epsilon closures in the ATN + * from causing a stack overflow. Outside code should pass + * {@code new HashSet} for this argument. + * @param calledRuleStack A set used for preventing left recursion in the + * ATN from causing a stack overflow. Outside code should pass + * {@code new BitSet()} for this argument. + * @param seeThruPreds {@code true} to true semantic predicates as + * implicitly {@code true} and "see through them", otherwise {@code false} + * to treat semantic predicates as opaque and add {@link #HIT_PRED} to the + * result if one is encountered. + * @param addEOF Add {@link Token#EOF} to the result if the end of the + * outermost context is reached. This parameter has no effect if {@code ctx} + * is {@code null}. + */ + protected internal virtual void Look_(ATNState s, ATNState stopState, PredictionContext ctx, IntervalSet look, HashSet lookBusy, BitSet calledRuleStack, bool seeThruPreds, bool addEOF) { - // System.out.println("_LOOK("+s.stateNumber+", ctx="+ctx); ATNConfig c = new ATNConfig(s, 0, ctx); if (!lookBusy.Add(c)) { @@ -268,50 +153,51 @@ namespace Antlr4.Runtime.Atn look.Add(TokenConstants.EPSILON); return; } - else if (ctx.IsEmpty && addEOF) { - look.Add(TokenConstants.EOF); - return; - } - } - if (s is RuleStopState) - { - if (ctx == null) - { - look.Add(TokenConstants.EPSILON); - return; - } else if (ctx.IsEmpty && addEOF) { look.Add(TokenConstants.EOF); return; } - if (ctx != PredictionContext.EMPTY) - { - for (int i = 0; i < ctx.Size; i++) - { - ATNState returnState = atn.states[ctx.GetReturnState(i)]; - bool removed = calledRuleStack.Get(returnState.ruleIndex); - try - { - calledRuleStack.Clear(returnState.ruleIndex); - Look(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF); - } - finally - { - if (removed) - { - calledRuleStack.Set(returnState.ruleIndex); - } - } - } - return; - } + } + if (s is RuleStopState) + { + if (ctx == null) + { + look.Add(TokenConstants.EPSILON); + return; + } + else if (ctx.IsEmpty && addEOF) + { + look.Add(TokenConstants.EOF); + return; + } + if (ctx != PredictionContext.EMPTY) + { + bool removed = calledRuleStack.Get(s.ruleIndex); + try + { + calledRuleStack.Clear(s.ruleIndex); + for (int i = 0; i < ctx.Size; i++) + { + ATNState returnState = atn.states[ctx.GetReturnState(i)]; + Look_(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF); + } + } + finally + { + if (removed) + { + calledRuleStack.Set(s.ruleIndex); + } + } + return; + } } int n = s.NumberOfTransitions; for (int i_1 = 0; i_1 < n; i_1++) { Transition t = s.Transition(i_1); - if (t is RuleTransition) + if (t.GetType() == typeof(RuleTransition)) { RuleTransition ruleTransition = (RuleTransition)t; if (calledRuleStack.Get(ruleTransition.ruleIndex)) @@ -322,51 +208,42 @@ namespace Antlr4.Runtime.Atn try { calledRuleStack.Set(ruleTransition.target.ruleIndex); - Look(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); + Look_(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); } finally { calledRuleStack.Clear(ruleTransition.target.ruleIndex); } } - else + else if (t is AbstractPredicateTransition) { - if (t is AbstractPredicateTransition) + if (seeThruPreds) { - if (seeThruPreds) - { - Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); - } - else - { - look.Add(HitPred); - } + Look_(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); } else { - if (t.IsEpsilon) + look.Add(HitPred); + } + } + else if (t.IsEpsilon) + { + Look_(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); + } + else if (t.GetType() == typeof(WildcardTransition)) + { + look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); + } + else + { + IntervalSet set = t.Label; + if (set != null) + { + if (t is NotSetTransition) { - Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); - } - else - { - if (t is WildcardTransition) - { - look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); - } - else - { - IntervalSet set = t.Label; - if (set != null) - { - if (t is NotSetTransition) - { - set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); - } - look.AddAll(set); - } - } + set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); } + look.AddAll(set); } } } diff --git a/runtime/CSharp/DefaultErrorStrategy.cs b/runtime/CSharp/DefaultErrorStrategy.cs index 5dee39cdd..f208c610f 100644 --- a/runtime/CSharp/DefaultErrorStrategy.cs +++ b/runtime/CSharp/DefaultErrorStrategy.cs @@ -42,6 +42,21 @@ namespace Antlr4.Runtime protected internal IntervalSet lastErrorStates; + /** + * This field is used to propagate information about the lookahead following + * the previous match. Since prediction prefers completing the current rule + * to error recovery efforts, error reporting may occur later than the + * original point where it was discoverable. The original context is used to + * compute the true expected sets as though the reporting occurred as early + * as possible. + */ + protected ParserRuleContext nextTokensContext; + + /** + * @see #nextTokensContext + */ + protected int nextTokensState; + /// /// ///

The default implementation simply calls @@ -264,8 +279,22 @@ namespace Antlr4.Runtime int la = tokens.LA(1); // try cheaper subset first; might get lucky. seems to shave a wee bit off var nextTokens = recognizer.Atn.NextTokens(s); - if (nextTokens.Contains(TokenConstants.EPSILON) || nextTokens.Contains(la)) + if (nextTokens.Contains(la)) { + nextTokensContext = null; + nextTokensState = ATNState.InvalidStateNumber; + return; + } + + if (nextTokens.Contains(TokenConstants.EPSILON)) + { + if (nextTokensContext == null) + { + // It's possible the next token won't match; information tracked + // by sync is restricted for performance. + nextTokensContext = recognizer.Context; + nextTokensState = recognizer.State; + } return; } switch (s.StateType) From 7a9a26c7ecb9d30f48e18eb097440946f9dd084a Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 23 Jan 2021 17:22:17 +0800 Subject: [PATCH 2/8] fix #3042 in Python 3 runtime --- runtime/Python3/src/antlr4/LL1Analyzer.py | 18 +++++++++--------- .../Python3/src/antlr4/error/ErrorStrategy.py | 13 ++++++++++++- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/runtime/Python3/src/antlr4/LL1Analyzer.py b/runtime/Python3/src/antlr4/LL1Analyzer.py index a200a5d0e..ac149fa2f 100644 --- a/runtime/Python3/src/antlr4/LL1Analyzer.py +++ b/runtime/Python3/src/antlr4/LL1Analyzer.py @@ -132,16 +132,16 @@ class LL1Analyzer (object): return if ctx != PredictionContext.EMPTY: - # run thru all possible stack tops in ctx - for i in range(0, len(ctx)): - returnState = self.atn.states[ctx.getReturnState(i)] - removed = returnState.ruleIndex in calledRuleStack - try: - calledRuleStack.discard(returnState.ruleIndex) + removed = s.ruleIndex in calledRuleStack + try: + calledRuleStack.discard(s.ruleIndex) + # run thru all possible stack tops in ctx + for i in range(0, len(ctx)): + returnState = self.atn.states[ctx.getReturnState(i)] self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF) - finally: - if removed: - calledRuleStack.add(returnState.ruleIndex) + finally: + if removed: + calledRuleStack.add(s.ruleIndex) return for t in s.transitions: diff --git a/runtime/Python3/src/antlr4/error/ErrorStrategy.py b/runtime/Python3/src/antlr4/error/ErrorStrategy.py index d7538b607..0f7caadb2 100644 --- a/runtime/Python3/src/antlr4/error/ErrorStrategy.py +++ b/runtime/Python3/src/antlr4/error/ErrorStrategy.py @@ -58,6 +58,8 @@ class DefaultErrorStrategy(ErrorStrategy): # self.lastErrorIndex = -1 self.lastErrorStates = None + self.nextTokensContext = None + self.nextTokenState = 0 #

The default implementation simply calls {@link #endErrorCondition} to # ensure that the handler is not in error recovery mode.

@@ -208,7 +210,16 @@ class DefaultErrorStrategy(ErrorStrategy): la = recognizer.getTokenStream().LA(1) # try cheaper subset first; might get lucky. seems to shave a wee bit off nextTokens = recognizer.atn.nextTokens(s) - if Token.EPSILON in nextTokens or la in nextTokens: + if la in nextTokens: + self.nextTokensContext = None + self.nextTokenState = ATNState.INVALID_STATE_NUMBER + return + elif Token.EPSILON in nextTokens: + if self.nextTokensContext is None: + # It's possible the next token won't match information tracked + # by sync is restricted for performance. + self.nextTokensContext = recognizer._ctx + self.nextTokensState = recognizer._stateNumber return if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START, From a9f11612dd24f1857d33227ab1671720df14618e Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 23 Jan 2021 17:23:45 +0800 Subject: [PATCH 3/8] fix #3042 in Python 2 runtime --- runtime/Python2/src/antlr4/LL1Analyzer.py | 26 +++++++++---------- .../Python2/src/antlr4/error/ErrorStrategy.py | 13 +++++++++- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/runtime/Python2/src/antlr4/LL1Analyzer.py b/runtime/Python2/src/antlr4/LL1Analyzer.py index 7403f8c78..1c89c492e 100644 --- a/runtime/Python2/src/antlr4/LL1Analyzer.py +++ b/runtime/Python2/src/antlr4/LL1Analyzer.py @@ -129,16 +129,16 @@ class LL1Analyzer (object): return if ctx != PredictionContext.EMPTY: - # run thru all possible stack tops in ctx - for i in range(0, len(ctx)): - returnState = self.atn.states[ctx.getReturnState(i)] - removed = returnState.ruleIndex in calledRuleStack - try: - calledRuleStack.discard(returnState.ruleIndex) + removed = s.ruleIndex in calledRuleStack + try: + calledRuleStack.discard(s.ruleIndex) + # run thru all possible stack tops in ctx + for i in range(0, len(ctx)): + returnState = self.atn.states[ctx.getReturnState(i)] self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF) - finally: - if removed: - calledRuleStack.add(returnState.ruleIndex) + finally: + if removed: + calledRuleStack.add(s.ruleIndex) return for t in s.transitions: @@ -163,8 +163,8 @@ class LL1Analyzer (object): elif type(t) == WildcardTransition: look.addRange( Interval(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType + 1) ) else: - set = t.label - if set is not None: + set_ = t.label + if set_ is not None: if isinstance(t, NotSetTransition): - set = set.complement(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType) - look.addSet(set) + set_ = set_.complement(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType) + look.addSet(set_) diff --git a/runtime/Python2/src/antlr4/error/ErrorStrategy.py b/runtime/Python2/src/antlr4/error/ErrorStrategy.py index c0c4001b3..b70c7df8e 100644 --- a/runtime/Python2/src/antlr4/error/ErrorStrategy.py +++ b/runtime/Python2/src/antlr4/error/ErrorStrategy.py @@ -53,6 +53,8 @@ class DefaultErrorStrategy(ErrorStrategy): # self.lastErrorIndex = -1 self.lastErrorStates = None + self.nextTokensContext = None + self.nextTokenState = 0 #

The default implementation simply calls {@link #endErrorCondition} to # ensure that the handler is not in error recovery mode.

@@ -203,7 +205,16 @@ class DefaultErrorStrategy(ErrorStrategy): la = recognizer.getTokenStream().LA(1) # try cheaper subset first; might get lucky. seems to shave a wee bit off nextTokens = recognizer.atn.nextTokens(s) - if Token.EPSILON in nextTokens or la in nextTokens: + if la in nextTokens: + self.nextTokensContext = None + self.nextTokenState = ATNState.INVALID_STATE_NUMBER + return + elif Token.EPSILON in nextTokens: + if self.nextTokensContext is None: + # It's possible the next token won't match information tracked + # by sync is restricted for performance. + self.nextTokensContext = recognizer._ctx + self.nextTokensState = recognizer._stateNumber return if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START, From 329f1301ea6f413a4038f701086c248e45ab734e Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 23 Jan 2021 17:32:06 +0800 Subject: [PATCH 4/8] fix #3042 in JavaScript runtime --- runtime/JavaScript/src/antlr4/LL1Analyzer.js | 20 +++++++++---------- .../src/antlr4/error/ErrorStrategy.js | 20 +++++++++++++++---- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/runtime/JavaScript/src/antlr4/LL1Analyzer.js b/runtime/JavaScript/src/antlr4/LL1Analyzer.js index e76238dd7..18ed87899 100644 --- a/runtime/JavaScript/src/antlr4/LL1Analyzer.js +++ b/runtime/JavaScript/src/antlr4/LL1Analyzer.js @@ -128,17 +128,17 @@ class LL1Analyzer { return; } if (ctx !== PredictionContext.EMPTY) { - // run thru all possible stack tops in ctx - for(let i=0; i Date: Sat, 23 Jan 2021 18:18:30 +0800 Subject: [PATCH 5/8] fix typo --- runtime/JavaScript/src/antlr4/error/ErrorStrategy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/JavaScript/src/antlr4/error/ErrorStrategy.js b/runtime/JavaScript/src/antlr4/error/ErrorStrategy.js index 44a161014..3da80f28b 100644 --- a/runtime/JavaScript/src/antlr4/error/ErrorStrategy.js +++ b/runtime/JavaScript/src/antlr4/error/ErrorStrategy.js @@ -227,7 +227,7 @@ class DefaultErrorStrategy extends ErrorStrategy { this.nextTokenState = ATNState.INVALID_STATE_NUMBER; return; } else if (nextTokens.contains(Token.EPSILON)) { - if(self.nextTokensContext === null) { + if(this.nextTokensContext === null) { // It's possible the next token won't match information tracked // by sync is restricted for performance. this.nextTokensContext = recognizer._ctx; From 559d5266484c5565d8fe2b15a5a2ed359bd71b03 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 23 Jan 2021 18:47:54 +0800 Subject: [PATCH 6/8] fix #3042 in Cpp runtime --- runtime/Cpp/runtime/src/atn/LL1Analyzer.cpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/runtime/Cpp/runtime/src/atn/LL1Analyzer.cpp b/runtime/Cpp/runtime/src/atn/LL1Analyzer.cpp index d7949cd1e..ddca80088 100755 --- a/runtime/Cpp/runtime/src/atn/LL1Analyzer.cpp +++ b/runtime/Cpp/runtime/src/atn/LL1Analyzer.cpp @@ -100,18 +100,16 @@ void LL1Analyzer::_LOOK(ATNState *s, ATNState *stopState, Ref } if (ctx != PredictionContext::EMPTY) { - // run thru all possible stack tops in ctx + bool removed = calledRuleStack.test(s->ruleIndex); + calledRuleStack[s->ruleIndex] = false; + auto onExit = finally([removed, &calledRuleStack, s] { + if (removed) { + calledRuleStack.set(s->ruleIndex); + } + }); + // run thru all possible stack tops in ctx for (size_t i = 0; i < ctx->size(); i++) { ATNState *returnState = _atn.states[ctx->getReturnState(i)]; - - bool removed = calledRuleStack.test(returnState->ruleIndex); - auto onExit = finally([removed, &calledRuleStack, returnState] { - if (removed) { - calledRuleStack.set(returnState->ruleIndex); - } - }); - - calledRuleStack[returnState->ruleIndex] = false; _LOOK(returnState, stopState, ctx->getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF); } return; From 43226724350f73066de4498ab5f06c1fc56c22ff Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 23 Jan 2021 18:50:21 +0800 Subject: [PATCH 7/8] fix #3042 in Go runtime --- runtime/Go/antlr/ll1_analyzer.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/runtime/Go/antlr/ll1_analyzer.go b/runtime/Go/antlr/ll1_analyzer.go index f5afd09b3..3ebc40a76 100644 --- a/runtime/Go/antlr/ll1_analyzer.go +++ b/runtime/Go/antlr/ll1_analyzer.go @@ -112,16 +112,6 @@ func (la *LL1Analyzer) Look(s, stopState ATNState, ctx RuleContext) *IntervalSet func (la *LL1Analyzer) look2(s, stopState ATNState, ctx PredictionContext, look *IntervalSet, lookBusy *Set, calledRuleStack *BitSet, seeThruPreds, addEOF bool, i int) { returnState := la.atn.states[ctx.getReturnState(i)] - - removed := calledRuleStack.contains(returnState.GetRuleIndex()) - - defer func() { - if removed { - calledRuleStack.add(returnState.GetRuleIndex()) - } - }() - - calledRuleStack.remove(returnState.GetRuleIndex()) la.look1(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF) } @@ -158,6 +148,13 @@ func (la *LL1Analyzer) look1(s, stopState ATNState, ctx PredictionContext, look } if ctx != BasePredictionContextEMPTY { + removed := calledRuleStack.contains(s.GetRuleIndex()) + defer func() { + if removed { + calledRuleStack.add(s.GetRuleIndex()) + } + }() + calledRuleStack.remove(s.GetRuleIndex()) // run thru all possible stack tops in ctx for i := 0; i < ctx.length(); i++ { returnState := la.atn.states[ctx.getReturnState(i)] From 58da376bef2813973cbfdf61f1ac8a25f1db7619 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 23 Jan 2021 19:19:02 +0800 Subject: [PATCH 8/8] fix #3042 in Swift runtime --- runtime/Swift/Sources/Antlr4/atn/LL1Analyzer.swift | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/runtime/Swift/Sources/Antlr4/atn/LL1Analyzer.swift b/runtime/Swift/Sources/Antlr4/atn/LL1Analyzer.swift index 3f594f234..bc179a1d0 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LL1Analyzer.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LL1Analyzer.swift @@ -169,16 +169,18 @@ public class LL1Analyzer { } if ctx != PredictionContext.EMPTY { + let removed = try! calledRuleStack.get(s.ruleIndex!) + try! calledRuleStack.clear(s.ruleIndex!) + defer { + if removed { + try! calledRuleStack.set(s.ruleIndex!) + } + } // run thru all possible stack tops in ctx let length = ctx.size() for i in 0..