From a41a70f71a7201986975a8391250406849f0a5f0 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Tue, 8 Apr 2014 06:21:59 -0500 Subject: [PATCH] Update to the latest release of ANTLR 4 --- reference/antlr4 | 2 +- .../CSharp/Antlr4.Runtime/Atn/ATNConfigSet.cs | 2 +- .../Antlr4.Runtime/Atn/LexerATNSimulator.cs | 31 ++- .../Antlr4.Runtime/Atn/ParserATNSimulator.cs | 193 ++++++++++++++++-- .../CSharp/Antlr4.Runtime/Atn/Transition.cs | 17 +- runtime/CSharp/Antlr4.Runtime/Dfa/DFA.cs | 2 +- 6 files changed, 221 insertions(+), 26 deletions(-) diff --git a/reference/antlr4 b/reference/antlr4 index 8083e0b65..cd9d2e248 160000 --- a/reference/antlr4 +++ b/reference/antlr4 @@ -1 +1 @@ -Subproject commit 8083e0b659bb318eb4454591648a01d28e07f03e +Subproject commit cd9d2e248a9754a601e4a27f32fd889cc9459935 diff --git a/runtime/CSharp/Antlr4.Runtime/Atn/ATNConfigSet.cs b/runtime/CSharp/Antlr4.Runtime/Atn/ATNConfigSet.cs index 9a3d7d943..67929e9a9 100644 --- a/runtime/CSharp/Antlr4.Runtime/Atn/ATNConfigSet.cs +++ b/runtime/CSharp/Antlr4.Runtime/Atn/ATNConfigSet.cs @@ -92,7 +92,7 @@ namespace Antlr4.Runtime.Atn /// true /// , this config set represents configurations where the entire /// outer context has been consumed by the ATN interpreter. This prevents the - /// ParserATNSimulator.Closure(ATNConfigSet, ATNConfigSet, bool, bool, PredictionContextCache) + /// ParserATNSimulator.Closure(ATNConfigSet, ATNConfigSet, bool, bool, PredictionContextCache, bool) /// from pursuing the global FOLLOW when a /// rule stop state is reached with an empty prediction context. ///

diff --git a/runtime/CSharp/Antlr4.Runtime/Atn/LexerATNSimulator.cs b/runtime/CSharp/Antlr4.Runtime/Atn/LexerATNSimulator.cs index ad554fca5..a4df07f73 100644 --- a/runtime/CSharp/Antlr4.Runtime/Atn/LexerATNSimulator.cs +++ b/runtime/CSharp/Antlr4.Runtime/Atn/LexerATNSimulator.cs @@ -367,7 +367,8 @@ namespace Antlr4.Runtime.Atn { lexerActionExecutor = lexerActionExecutor.FixOffsetBeforeMatch(input.Index - startIndex); } - if (Closure(input, c.Transform(target, lexerActionExecutor, true), reach, currentAltReachedAcceptState, true)) + bool treatEofAsEpsilon = t == CharStreamConstants.Eof; + if (Closure(input, c.Transform(target, lexerActionExecutor, true), reach, currentAltReachedAcceptState, true, treatEofAsEpsilon)) { // any remaining configs for this alt have a lower priority than // the one that just reached an accept state. @@ -414,7 +415,7 @@ namespace Antlr4.Runtime.Atn { ATNState target = p.Transition(i).target; ATNConfig c = ATNConfig.Create(target, i + 1, initialContext); - Closure(input, c, configs, false, false); + Closure(input, c, configs, false, false, false); } return configs; } @@ -440,7 +441,7 @@ namespace Antlr4.Runtime.Atn /// false /// . /// - protected internal virtual bool Closure(ICharStream input, ATNConfig config, ATNConfigSet configs, bool currentAltReachedAcceptState, bool speculative) + protected internal virtual bool Closure(ICharStream input, ATNConfig config, ATNConfigSet configs, bool currentAltReachedAcceptState, bool speculative, bool treatEofAsEpsilon) { if (config.State is RuleStopState) { @@ -469,7 +470,7 @@ namespace Antlr4.Runtime.Atn // "pop" return state ATNState returnState = atn.states[returnStateNumber]; ATNConfig c = config.Transform(returnState, newContext, false); - currentAltReachedAcceptState = Closure(input, c, configs, currentAltReachedAcceptState, speculative); + currentAltReachedAcceptState = Closure(input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon); } return currentAltReachedAcceptState; } @@ -485,10 +486,10 @@ namespace Antlr4.Runtime.Atn for (int i_1 = 0; i_1 < p.NumberOfOptimizedTransitions; i_1++) { Transition t = p.GetOptimizedTransition(i_1); - ATNConfig c = GetEpsilonTarget(input, config, t, configs, speculative); + ATNConfig c = GetEpsilonTarget(input, config, t, configs, speculative, treatEofAsEpsilon); if (c != null) { - currentAltReachedAcceptState = Closure(input, c, configs, currentAltReachedAcceptState, speculative); + currentAltReachedAcceptState = Closure(input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon); } } return currentAltReachedAcceptState; @@ -496,7 +497,7 @@ namespace Antlr4.Runtime.Atn // side-effect: can alter configs.hasSemanticContext [Nullable] - protected internal virtual ATNConfig GetEpsilonTarget(ICharStream input, ATNConfig config, Transition t, ATNConfigSet configs, bool speculative) + protected internal virtual ATNConfig GetEpsilonTarget(ICharStream input, ATNConfig config, Transition t, ATNConfigSet configs, bool speculative, bool treatEofAsEpsilon) { ATNConfig c; switch (t.TransitionType) @@ -571,6 +572,22 @@ namespace Antlr4.Runtime.Atn break; } + case TransitionType.Atom: + case TransitionType.Range: + case TransitionType.Set: + { + if (treatEofAsEpsilon) + { + if (t.Matches(CharStreamConstants.Eof, char.MinValue, char.MaxValue)) + { + c = config.Transform(t.target, false); + break; + } + } + c = null; + break; + } + default: { c = null; diff --git a/runtime/CSharp/Antlr4.Runtime/Atn/ParserATNSimulator.cs b/runtime/CSharp/Antlr4.Runtime/Atn/ParserATNSimulator.cs index b48e0d20e..4e24f55c2 100644 --- a/runtime/CSharp/Antlr4.Runtime/Atn/ParserATNSimulator.cs +++ b/runtime/CSharp/Antlr4.Runtime/Atn/ParserATNSimulator.cs @@ -884,21 +884,160 @@ namespace Antlr4.Runtime.Atn } } + ///

+ /// This method is used to improve the localization of error messages by + /// choosing an alternative rather than throwing a + /// Antlr4.Runtime.NoViableAltException + /// in particular prediction scenarios where the + /// ATNSimulator.Error + /// state was reached during ATN simulation. + ///

+ /// The default implementation of this method uses the following + /// algorithm to identify an ATN configuration which successfully parsed the + /// decision entry rule. Choosing such an alternative ensures that the + /// Antlr4.Runtime.ParserRuleContext + /// returned by the calling rule will be complete + /// and valid, and the syntax error will be reported later at a more + /// localized location.

+ /// + ///

+ /// In some scenarios, the algorithm described above could predict an + /// alternative which will result in a + /// Antlr4.Runtime.FailedPredicateException + /// in + /// parser. Specifically, this could occur if the only configuration + /// capable of successfully parsing to the end of the decision rule is + /// blocked by a semantic predicate. By choosing this alternative within + /// AdaptivePredict(Antlr4.Runtime.ITokenStream, int, Antlr4.Runtime.ParserRuleContext) + /// instead of throwing a + /// Antlr4.Runtime.NoViableAltException + /// , the resulting + /// Antlr4.Runtime.FailedPredicateException + /// in the parser will identify the specific + /// predicate which is preventing the parser from successfully parsing the + /// decision rule, which helps developers identify and correct logic errors + /// in semantic predicates. + ///

+ /// + /// + /// The input + /// Antlr4.Runtime.ITokenStream + /// + /// + /// The start index for the current prediction, which is + /// the input index where any semantic context in + /// configs + /// should be + /// evaluated + /// + /// + /// The ATN simulation state immediately before the + /// ATNSimulator.Error + /// state was reached + /// + /// + /// The value to return from + /// AdaptivePredict(Antlr4.Runtime.ITokenStream, int, Antlr4.Runtime.ParserRuleContext) + /// , or + /// ATN.InvalidAltNumber + /// if a suitable alternative was not + /// identified and + /// AdaptivePredict(Antlr4.Runtime.ITokenStream, int, Antlr4.Runtime.ParserRuleContext) + /// should report an error instead. + /// protected internal virtual int HandleNoViableAlt(ITokenStream input, int startIndex, SimulatorState previous) { if (previous.s0 != null) { BitSet alts = new BitSet(); + int maxAlt = 0; foreach (ATNConfig config in previous.s0.configs) { if (config.ReachesIntoOuterContext || config.State is RuleStopState) { alts.Set(config.Alt); + maxAlt = Math.Max(maxAlt, config.Alt); } } - if (!alts.IsEmpty()) + switch (alts.Cardinality()) { - return alts.NextSetBit(0); + case 0: + { + break; + } + + case 1: + { + return alts.NextSetBit(0); + } + + default: + { + if (!previous.s0.configs.HasSemanticContext) + { + // configs doesn't contain any predicates, so the predicate + // filtering code below would be pointless + return alts.NextSetBit(0); + } + ATNConfigSet filteredConfigs = new ATNConfigSet(); + foreach (ATNConfig config_1 in previous.s0.configs) + { + if (config_1.ReachesIntoOuterContext || config_1.State is RuleStopState) + { + filteredConfigs.AddItem(config_1); + } + } + SemanticContext[] altToPred = GetPredsForAmbigAlts(alts, filteredConfigs, maxAlt); + if (altToPred != null) + { + DFAState.PredPrediction[] predicates = GetPredicatePredictions(alts, altToPred); + if (predicates != null) + { + int stopIndex = input.Index; + try + { + input.Seek(startIndex); + BitSet filteredAlts = EvalSemanticContext(predicates, previous.outerContext, false); + if (!filteredAlts.IsEmpty()) + { + return filteredAlts.NextSetBit(0); + } + } + finally + { + input.Seek(stopIndex); + } + } + } + return alts.NextSetBit(0); + } } } throw NoViableAlt(input, previous.outerContext, previous.s0.configs, startIndex); @@ -1039,14 +1178,15 @@ namespace Antlr4.Runtime.Atn } } } - if (optimize_unique_closure && skippedStopStates == null && reachIntermediate.UniqueAlt != ATN.InvalidAltNumber) + if (optimize_unique_closure && skippedStopStates == null && t != TokenConstants.Eof && reachIntermediate.UniqueAlt != ATN.InvalidAltNumber) { reachIntermediate.IsOutermostConfigSet = reach.IsOutermostConfigSet; reach = reachIntermediate; break; } bool collectPredicates = false; - Closure(reachIntermediate, reach, collectPredicates, hasMoreContext, contextCache); + bool treatEofAsEpsilon = t == TokenConstants.Eof; + Closure(reachIntermediate, reach, collectPredicates, hasMoreContext, contextCache, treatEofAsEpsilon); stepIntoGlobal = reach.DipsIntoOuterContext; if (t == IntStreamConstants.Eof) { @@ -1229,7 +1369,7 @@ namespace Antlr4.Runtime.Atn configs.IsOutermostConfigSet = true; } bool collectPredicates = true; - Closure(reachIntermediate, configs, collectPredicates, hasMoreContext, contextCache); + Closure(reachIntermediate, configs, collectPredicates, hasMoreContext, contextCache, false); bool stepIntoGlobal = configs.DipsIntoOuterContext; DFAState next; if (useContext && !enable_global_context_dfa) @@ -1562,7 +1702,7 @@ namespace Antlr4.Runtime.Atn return predictions; } - protected internal virtual void Closure(ATNConfigSet sourceConfigs, ATNConfigSet configs, bool collectPredicates, bool hasMoreContext, PredictionContextCache contextCache) + protected internal virtual void Closure(ATNConfigSet sourceConfigs, ATNConfigSet configs, bool collectPredicates, bool hasMoreContext, PredictionContextCache contextCache, bool treatEofAsEpsilon) { if (contextCache == null) { @@ -1575,13 +1715,13 @@ namespace Antlr4.Runtime.Atn ATNConfigSet intermediate = new ATNConfigSet(); foreach (ATNConfig config in currentConfigs) { - Closure(config, configs, intermediate, closureBusy, collectPredicates, hasMoreContext, contextCache, 0); + Closure(config, configs, intermediate, closureBusy, collectPredicates, hasMoreContext, contextCache, 0, treatEofAsEpsilon); } currentConfigs = intermediate; } } - protected internal virtual void Closure(ATNConfig config, ATNConfigSet configs, ATNConfigSet intermediate, HashSet closureBusy, bool collectPredicates, bool hasMoreContexts, PredictionContextCache contextCache, int depth) + protected internal virtual void Closure(ATNConfig config, ATNConfigSet configs, ATNConfigSet intermediate, HashSet closureBusy, bool collectPredicates, bool hasMoreContexts, PredictionContextCache contextCache, int depth, bool treatEofAsEpsilon) { if (config.State is RuleStopState) { @@ -1601,7 +1741,7 @@ namespace Antlr4.Runtime.Atn // Make sure we track that we are now out of context. c.OuterContextDepth = config.OuterContextDepth; System.Diagnostics.Debug.Assert(depth > int.MinValue); - Closure(c, configs, intermediate, closureBusy, collectPredicates, hasMoreContexts, contextCache, depth - 1); + Closure(c, configs, intermediate, closureBusy, collectPredicates, hasMoreContexts, contextCache, depth - 1, treatEofAsEpsilon); } if (!hasEmpty || !hasMoreContexts) { @@ -1633,11 +1773,13 @@ namespace Antlr4.Runtime.Atn { configs.Add(config, contextCache); } + // make sure to not return here, because EOF transitions can act as + // both epsilon transitions and non-epsilon transitions. for (int i_1 = 0; i_1 < p.NumberOfOptimizedTransitions; i_1++) { Transition t = p.GetOptimizedTransition(i_1); bool continueCollecting = !(t is Antlr4.Runtime.Atn.ActionTransition) && collectPredicates; - ATNConfig c = GetEpsilonTarget(config, t, continueCollecting, depth == 0, contextCache); + ATNConfig c = GetEpsilonTarget(config, t, continueCollecting, depth == 0, contextCache, treatEofAsEpsilon); if (c != null) { if (t is Antlr4.Runtime.Atn.RuleTransition) @@ -1648,6 +1790,11 @@ namespace Antlr4.Runtime.Atn continue; } } + if (!t.IsEpsilon && !closureBusy.AddItem(c)) + { + // avoid infinite recursion for EOF* and EOF+ + continue; + } int newDepth = depth; if (config.State is RuleStopState) { @@ -1694,7 +1841,7 @@ namespace Antlr4.Runtime.Atn } } } - Closure(c, configs, intermediate, closureBusy, continueCollecting, hasMoreContexts, contextCache, newDepth); + Closure(c, configs, intermediate, closureBusy, continueCollecting, hasMoreContexts, contextCache, newDepth, treatEofAsEpsilon); } } } @@ -1710,7 +1857,7 @@ namespace Antlr4.Runtime.Atn } [Nullable] - protected internal virtual ATNConfig GetEpsilonTarget(ATNConfig config, Transition t, bool collectPredicates, bool inContext, PredictionContextCache contextCache) + protected internal virtual ATNConfig GetEpsilonTarget(ATNConfig config, Transition t, bool collectPredicates, bool inContext, PredictionContextCache contextCache, bool treatEofAsEpsilon) { switch (t.TransitionType) { @@ -1739,6 +1886,22 @@ namespace Antlr4.Runtime.Atn return config.Transform(t.target, false); } + case TransitionType.Atom: + case TransitionType.Range: + case TransitionType.Set: + { + // EOF transitions act like epsilon transitions after the first EOF + // transition is traversed + if (treatEofAsEpsilon) + { + if (t.Matches(TokenConstants.Eof, 0, 1)) + { + return config.Transform(t.target, false); + } + } + return null; + } + default: { return null; @@ -1807,9 +1970,9 @@ namespace Antlr4.Runtime.Atn return config.Transform(t.target, newContext, false); } - private sealed class _IComparer_1771 : IComparer + private sealed class _IComparer_1905 : IComparer { - public _IComparer_1771() + public _IComparer_1905() { } @@ -1829,7 +1992,7 @@ namespace Antlr4.Runtime.Atn } } - private static readonly IComparer StateAltSortComparator = new _IComparer_1771(); + private static readonly IComparer StateAltSortComparator = new _IComparer_1905(); private BitSet IsConflicted(ATNConfigSet configset, PredictionContextCache contextCache) { diff --git a/runtime/CSharp/Antlr4.Runtime/Atn/Transition.cs b/runtime/CSharp/Antlr4.Runtime/Atn/Transition.cs index bcfbba5a3..7672d3d92 100644 --- a/runtime/CSharp/Antlr4.Runtime/Atn/Transition.cs +++ b/runtime/CSharp/Antlr4.Runtime/Atn/Transition.cs @@ -70,7 +70,22 @@ namespace Antlr4.Runtime.Atn get; } - /// Are we epsilon, action, sempred? + /// Determines if the transition is an "epsilon" transition. + /// + /// Determines if the transition is an "epsilon" transition. + ///

The default implementation returns + /// false + /// .

+ /// + /// + /// + /// true + /// if traversing this transition in the ATN does not + /// consume an input symbol; otherwise, + /// false + /// if traversing this + /// transition consumes (matches) an input symbol. + /// public virtual bool IsEpsilon { get diff --git a/runtime/CSharp/Antlr4.Runtime/Dfa/DFA.cs b/runtime/CSharp/Antlr4.Runtime/Dfa/DFA.cs index 68d045087..1b7aa7eb1 100644 --- a/runtime/CSharp/Antlr4.Runtime/Dfa/DFA.cs +++ b/runtime/CSharp/Antlr4.Runtime/Dfa/DFA.cs @@ -250,7 +250,7 @@ namespace Antlr4.Runtime.Dfa { if (IsPrecedenceDfa()) { - return s0full.Get().EdgeMap.IsEmpty(); + return !s0full.Get().EdgeMap.IsEmpty(); } return s0full.Get() != null; }