Merge branch 'sharpen'

This commit is contained in:
Sam Harwell 2014-04-08 06:23:33 -05:00
commit f71699e20a
6 changed files with 221 additions and 26 deletions

@ -1 +1 @@
Subproject commit 8083e0b659bb318eb4454591648a01d28e07f03e Subproject commit cd9d2e248a9754a601e4a27f32fd889cc9459935

View File

@ -95,7 +95,7 @@ namespace Antlr4.Runtime.Atn
/// <code>true</code> /// <code>true</code>
/// , this config set represents configurations where the entire /// , this config set represents configurations where the entire
/// outer context has been consumed by the ATN interpreter. This prevents the /// outer context has been consumed by the ATN interpreter. This prevents the
/// <see cref="ParserATNSimulator.Closure(ATNConfigSet, ATNConfigSet, bool, bool, PredictionContextCache)">ParserATNSimulator.Closure(ATNConfigSet, ATNConfigSet, bool, bool, PredictionContextCache)</see> /// <see cref="ParserATNSimulator.Closure(ATNConfigSet, ATNConfigSet, bool, bool, PredictionContextCache, bool)">ParserATNSimulator.Closure(ATNConfigSet, ATNConfigSet, bool, bool, PredictionContextCache, bool)</see>
/// from pursuing the global FOLLOW when a /// from pursuing the global FOLLOW when a
/// rule stop state is reached with an empty prediction context. /// rule stop state is reached with an empty prediction context.
/// <p/> /// <p/>

View File

@ -371,7 +371,8 @@ namespace Antlr4.Runtime.Atn
{ {
lexerActionExecutor = lexerActionExecutor.FixOffsetBeforeMatch(input.Index - startIndex); lexerActionExecutor = lexerActionExecutor.FixOffsetBeforeMatch(input.Index - startIndex);
} }
if (Closure(input, c.Transform(target, lexerActionExecutor, true), reach, currentAltReachedAcceptState, true)) bool treatEofAsEpsilon = t == CharStreamConstants.Eof;
if (Closure(input, c.Transform(target, lexerActionExecutor, true), reach, currentAltReachedAcceptState, true, treatEofAsEpsilon))
{ {
// any remaining configs for this alt have a lower priority than // any remaining configs for this alt have a lower priority than
// the one that just reached an accept state. // the one that just reached an accept state.
@ -418,7 +419,7 @@ namespace Antlr4.Runtime.Atn
{ {
ATNState target = p.Transition(i).target; ATNState target = p.Transition(i).target;
ATNConfig c = ATNConfig.Create(target, i + 1, initialContext); ATNConfig c = ATNConfig.Create(target, i + 1, initialContext);
Closure(input, c, configs, false, false); Closure(input, c, configs, false, false, false);
} }
return configs; return configs;
} }
@ -444,7 +445,7 @@ namespace Antlr4.Runtime.Atn
/// <code>false</code> /// <code>false</code>
/// . /// .
/// </returns> /// </returns>
protected internal virtual bool Closure(ICharStream input, ATNConfig config, ATNConfigSet configs, bool currentAltReachedAcceptState, bool speculative) protected internal virtual bool Closure(ICharStream input, ATNConfig config, ATNConfigSet configs, bool currentAltReachedAcceptState, bool speculative, bool treatEofAsEpsilon)
{ {
if (config.State is RuleStopState) if (config.State is RuleStopState)
{ {
@ -473,7 +474,7 @@ namespace Antlr4.Runtime.Atn
// "pop" return state // "pop" return state
ATNState returnState = atn.states[returnStateNumber]; ATNState returnState = atn.states[returnStateNumber];
ATNConfig c = config.Transform(returnState, newContext, false); ATNConfig c = config.Transform(returnState, newContext, false);
currentAltReachedAcceptState = Closure(input, c, configs, currentAltReachedAcceptState, speculative); currentAltReachedAcceptState = Closure(input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon);
} }
return currentAltReachedAcceptState; return currentAltReachedAcceptState;
} }
@ -489,10 +490,10 @@ namespace Antlr4.Runtime.Atn
for (int i_1 = 0; i_1 < p.NumberOfOptimizedTransitions; i_1++) for (int i_1 = 0; i_1 < p.NumberOfOptimizedTransitions; i_1++)
{ {
Transition t = p.GetOptimizedTransition(i_1); Transition t = p.GetOptimizedTransition(i_1);
ATNConfig c = GetEpsilonTarget(input, config, t, configs, speculative); ATNConfig c = GetEpsilonTarget(input, config, t, configs, speculative, treatEofAsEpsilon);
if (c != null) if (c != null)
{ {
currentAltReachedAcceptState = Closure(input, c, configs, currentAltReachedAcceptState, speculative); currentAltReachedAcceptState = Closure(input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon);
} }
} }
return currentAltReachedAcceptState; return currentAltReachedAcceptState;
@ -500,7 +501,7 @@ namespace Antlr4.Runtime.Atn
// side-effect: can alter configs.hasSemanticContext // side-effect: can alter configs.hasSemanticContext
[return: Nullable] [return: Nullable]
protected internal virtual ATNConfig GetEpsilonTarget(ICharStream input, ATNConfig config, Transition t, ATNConfigSet configs, bool speculative) protected internal virtual ATNConfig GetEpsilonTarget(ICharStream input, ATNConfig config, Transition t, ATNConfigSet configs, bool speculative, bool treatEofAsEpsilon)
{ {
ATNConfig c; ATNConfig c;
switch (t.TransitionType) switch (t.TransitionType)
@ -575,6 +576,22 @@ namespace Antlr4.Runtime.Atn
break; break;
} }
case TransitionType.Atom:
case TransitionType.Range:
case TransitionType.Set:
{
if (treatEofAsEpsilon)
{
if (t.Matches(CharStreamConstants.Eof, char.MinValue, char.MaxValue))
{
c = config.Transform(t.target, false);
break;
}
}
c = null;
break;
}
default: default:
{ {
c = null; c = null;

View File

@ -889,21 +889,160 @@ namespace Antlr4.Runtime.Atn
} }
} }
/// <summary>
/// This method is used to improve the localization of error messages by
/// choosing an alternative rather than throwing a
/// <see cref="Antlr4.Runtime.NoViableAltException">Antlr4.Runtime.NoViableAltException</see>
/// in particular prediction scenarios where the
/// <see cref="ATNSimulator.Error">ATNSimulator.Error</see>
/// state was reached during ATN simulation.
/// <p>
/// The default implementation of this method uses the following
/// algorithm to identify an ATN configuration which successfully parsed the
/// decision entry rule. Choosing such an alternative ensures that the
/// <see cref="Antlr4.Runtime.ParserRuleContext">Antlr4.Runtime.ParserRuleContext</see>
/// returned by the calling rule will be complete
/// and valid, and the syntax error will be reported later at a more
/// localized location.</p>
/// <ul>
/// <li>If no configuration in
/// <code>configs</code>
/// reached the end of the
/// decision rule, return
/// <see cref="ATN.InvalidAltNumber">ATN.InvalidAltNumber</see>
/// .</li>
/// <li>If all configurations in
/// <code>configs</code>
/// which reached the end of the
/// decision rule predict the same alternative, return that alternative.</li>
/// <li>If the configurations in
/// <code>configs</code>
/// which reached the end of the
/// decision rule predict multiple alternatives (call this <em>S</em>),
/// choose an alternative in the following order.
/// <ol>
/// <li>Filter the configurations in
/// <code>configs</code>
/// to only those
/// configurations which remain viable after evaluating semantic predicates.
/// If the set of these filtered configurations which also reached the end of
/// the decision rule is not empty, return the minimum alternative
/// represented in this set.</li>
/// <li>Otherwise, choose the minimum alternative in <em>S</em>.</li>
/// </ol>
/// </li>
/// </ul>
/// <p>
/// In some scenarios, the algorithm described above could predict an
/// alternative which will result in a
/// <see cref="Antlr4.Runtime.FailedPredicateException">Antlr4.Runtime.FailedPredicateException</see>
/// in
/// parser. Specifically, this could occur if the <em>only</em> configuration
/// capable of successfully parsing to the end of the decision rule is
/// blocked by a semantic predicate. By choosing this alternative within
/// <see cref="AdaptivePredict(Antlr4.Runtime.ITokenStream, int, Antlr4.Runtime.ParserRuleContext)">AdaptivePredict(Antlr4.Runtime.ITokenStream, int, Antlr4.Runtime.ParserRuleContext)</see>
/// instead of throwing a
/// <see cref="Antlr4.Runtime.NoViableAltException">Antlr4.Runtime.NoViableAltException</see>
/// , the resulting
/// <see cref="Antlr4.Runtime.FailedPredicateException">Antlr4.Runtime.FailedPredicateException</see>
/// in the parser will identify the specific
/// predicate which is preventing the parser from successfully parsing the
/// decision rule, which helps developers identify and correct logic errors
/// in semantic predicates.
/// </p>
/// </summary>
/// <param name="input">
/// The input
/// <see cref="Antlr4.Runtime.ITokenStream">Antlr4.Runtime.ITokenStream</see>
/// </param>
/// <param name="startIndex">
/// The start index for the current prediction, which is
/// the input index where any semantic context in
/// <code>configs</code>
/// should be
/// evaluated
/// </param>
/// <param name="previous">
/// The ATN simulation state immediately before the
/// <see cref="ATNSimulator.Error">ATNSimulator.Error</see>
/// state was reached
/// </param>
/// <returns>
/// The value to return from
/// <see cref="AdaptivePredict(Antlr4.Runtime.ITokenStream, int, Antlr4.Runtime.ParserRuleContext)">AdaptivePredict(Antlr4.Runtime.ITokenStream, int, Antlr4.Runtime.ParserRuleContext)</see>
/// , or
/// <see cref="ATN.InvalidAltNumber">ATN.InvalidAltNumber</see>
/// if a suitable alternative was not
/// identified and
/// <see cref="AdaptivePredict(Antlr4.Runtime.ITokenStream, int, Antlr4.Runtime.ParserRuleContext)">AdaptivePredict(Antlr4.Runtime.ITokenStream, int, Antlr4.Runtime.ParserRuleContext)</see>
/// should report an error instead.
/// </returns>
protected internal virtual int HandleNoViableAlt(ITokenStream input, int startIndex, SimulatorState previous) protected internal virtual int HandleNoViableAlt(ITokenStream input, int startIndex, SimulatorState previous)
{ {
if (previous.s0 != null) if (previous.s0 != null)
{ {
BitSet alts = new BitSet(); BitSet alts = new BitSet();
int maxAlt = 0;
foreach (ATNConfig config in previous.s0.configs) foreach (ATNConfig config in previous.s0.configs)
{ {
if (config.ReachesIntoOuterContext || config.State is RuleStopState) if (config.ReachesIntoOuterContext || config.State is RuleStopState)
{ {
alts.Set(config.Alt); alts.Set(config.Alt);
maxAlt = Math.Max(maxAlt, config.Alt);
} }
} }
if (!alts.IsEmpty()) switch (alts.Cardinality())
{ {
return alts.NextSetBit(0); case 0:
{
break;
}
case 1:
{
return alts.NextSetBit(0);
}
default:
{
if (!previous.s0.configs.HasSemanticContext)
{
// configs doesn't contain any predicates, so the predicate
// filtering code below would be pointless
return alts.NextSetBit(0);
}
ATNConfigSet filteredConfigs = new ATNConfigSet();
foreach (ATNConfig config_1 in previous.s0.configs)
{
if (config_1.ReachesIntoOuterContext || config_1.State is RuleStopState)
{
filteredConfigs.AddItem(config_1);
}
}
SemanticContext[] altToPred = GetPredsForAmbigAlts(alts, filteredConfigs, maxAlt);
if (altToPred != null)
{
DFAState.PredPrediction[] predicates = GetPredicatePredictions(alts, altToPred);
if (predicates != null)
{
int stopIndex = input.Index;
try
{
input.Seek(startIndex);
BitSet filteredAlts = EvalSemanticContext(predicates, previous.outerContext, false);
if (!filteredAlts.IsEmpty())
{
return filteredAlts.NextSetBit(0);
}
}
finally
{
input.Seek(stopIndex);
}
}
}
return alts.NextSetBit(0);
}
} }
} }
throw NoViableAlt(input, previous.outerContext, previous.s0.configs, startIndex); throw NoViableAlt(input, previous.outerContext, previous.s0.configs, startIndex);
@ -1044,14 +1183,15 @@ namespace Antlr4.Runtime.Atn
} }
} }
} }
if (optimize_unique_closure && skippedStopStates == null && reachIntermediate.UniqueAlt != ATN.InvalidAltNumber) if (optimize_unique_closure && skippedStopStates == null && t != TokenConstants.Eof && reachIntermediate.UniqueAlt != ATN.InvalidAltNumber)
{ {
reachIntermediate.IsOutermostConfigSet = reach.IsOutermostConfigSet; reachIntermediate.IsOutermostConfigSet = reach.IsOutermostConfigSet;
reach = reachIntermediate; reach = reachIntermediate;
break; break;
} }
bool collectPredicates = false; bool collectPredicates = false;
Closure(reachIntermediate, reach, collectPredicates, hasMoreContext, contextCache); bool treatEofAsEpsilon = t == TokenConstants.Eof;
Closure(reachIntermediate, reach, collectPredicates, hasMoreContext, contextCache, treatEofAsEpsilon);
stepIntoGlobal = reach.DipsIntoOuterContext; stepIntoGlobal = reach.DipsIntoOuterContext;
if (t == IntStreamConstants.Eof) if (t == IntStreamConstants.Eof)
{ {
@ -1234,7 +1374,7 @@ namespace Antlr4.Runtime.Atn
configs.IsOutermostConfigSet = true; configs.IsOutermostConfigSet = true;
} }
bool collectPredicates = true; bool collectPredicates = true;
Closure(reachIntermediate, configs, collectPredicates, hasMoreContext, contextCache); Closure(reachIntermediate, configs, collectPredicates, hasMoreContext, contextCache, false);
bool stepIntoGlobal = configs.DipsIntoOuterContext; bool stepIntoGlobal = configs.DipsIntoOuterContext;
DFAState next; DFAState next;
if (useContext && !enable_global_context_dfa) if (useContext && !enable_global_context_dfa)
@ -1571,7 +1711,7 @@ namespace Antlr4.Runtime.Atn
return predictions; return predictions;
} }
protected internal virtual void Closure(ATNConfigSet sourceConfigs, ATNConfigSet configs, bool collectPredicates, bool hasMoreContext, PredictionContextCache contextCache) protected internal virtual void Closure(ATNConfigSet sourceConfigs, ATNConfigSet configs, bool collectPredicates, bool hasMoreContext, PredictionContextCache contextCache, bool treatEofAsEpsilon)
{ {
if (contextCache == null) if (contextCache == null)
{ {
@ -1584,13 +1724,13 @@ namespace Antlr4.Runtime.Atn
ATNConfigSet intermediate = new ATNConfigSet(); ATNConfigSet intermediate = new ATNConfigSet();
foreach (ATNConfig config in currentConfigs) foreach (ATNConfig config in currentConfigs)
{ {
Closure(config, configs, intermediate, closureBusy, collectPredicates, hasMoreContext, contextCache, 0); Closure(config, configs, intermediate, closureBusy, collectPredicates, hasMoreContext, contextCache, 0, treatEofAsEpsilon);
} }
currentConfigs = intermediate; currentConfigs = intermediate;
} }
} }
protected internal virtual void Closure(ATNConfig config, ATNConfigSet configs, ATNConfigSet intermediate, HashSet<ATNConfig> closureBusy, bool collectPredicates, bool hasMoreContexts, PredictionContextCache contextCache, int depth) protected internal virtual void Closure(ATNConfig config, ATNConfigSet configs, ATNConfigSet intermediate, HashSet<ATNConfig> closureBusy, bool collectPredicates, bool hasMoreContexts, PredictionContextCache contextCache, int depth, bool treatEofAsEpsilon)
{ {
if (config.State is RuleStopState) if (config.State is RuleStopState)
{ {
@ -1610,7 +1750,7 @@ namespace Antlr4.Runtime.Atn
// Make sure we track that we are now out of context. // Make sure we track that we are now out of context.
c.OuterContextDepth = config.OuterContextDepth; c.OuterContextDepth = config.OuterContextDepth;
System.Diagnostics.Debug.Assert(depth > int.MinValue); System.Diagnostics.Debug.Assert(depth > int.MinValue);
Closure(c, configs, intermediate, closureBusy, collectPredicates, hasMoreContexts, contextCache, depth - 1); Closure(c, configs, intermediate, closureBusy, collectPredicates, hasMoreContexts, contextCache, depth - 1, treatEofAsEpsilon);
} }
if (!hasEmpty || !hasMoreContexts) if (!hasEmpty || !hasMoreContexts)
{ {
@ -1642,11 +1782,13 @@ namespace Antlr4.Runtime.Atn
{ {
configs.Add(config, contextCache); configs.Add(config, contextCache);
} }
// make sure to not return here, because EOF transitions can act as
// both epsilon transitions and non-epsilon transitions.
for (int i_1 = 0; i_1 < p.NumberOfOptimizedTransitions; i_1++) for (int i_1 = 0; i_1 < p.NumberOfOptimizedTransitions; i_1++)
{ {
Transition t = p.GetOptimizedTransition(i_1); Transition t = p.GetOptimizedTransition(i_1);
bool continueCollecting = !(t is Antlr4.Runtime.Atn.ActionTransition) && collectPredicates; bool continueCollecting = !(t is Antlr4.Runtime.Atn.ActionTransition) && collectPredicates;
ATNConfig c = GetEpsilonTarget(config, t, continueCollecting, depth == 0, contextCache); ATNConfig c = GetEpsilonTarget(config, t, continueCollecting, depth == 0, contextCache, treatEofAsEpsilon);
if (c != null) if (c != null)
{ {
if (t is Antlr4.Runtime.Atn.RuleTransition) if (t is Antlr4.Runtime.Atn.RuleTransition)
@ -1657,6 +1799,11 @@ namespace Antlr4.Runtime.Atn
continue; continue;
} }
} }
if (!t.IsEpsilon && !closureBusy.AddItem(c))
{
// avoid infinite recursion for EOF* and EOF+
continue;
}
int newDepth = depth; int newDepth = depth;
if (config.State is RuleStopState) if (config.State is RuleStopState)
{ {
@ -1703,7 +1850,7 @@ namespace Antlr4.Runtime.Atn
} }
} }
} }
Closure(c, configs, intermediate, closureBusy, continueCollecting, hasMoreContexts, contextCache, newDepth); Closure(c, configs, intermediate, closureBusy, continueCollecting, hasMoreContexts, contextCache, newDepth, treatEofAsEpsilon);
} }
} }
} }
@ -1719,7 +1866,7 @@ namespace Antlr4.Runtime.Atn
} }
[return: Nullable] [return: Nullable]
protected internal virtual ATNConfig GetEpsilonTarget(ATNConfig config, Transition t, bool collectPredicates, bool inContext, PredictionContextCache contextCache) protected internal virtual ATNConfig GetEpsilonTarget(ATNConfig config, Transition t, bool collectPredicates, bool inContext, PredictionContextCache contextCache, bool treatEofAsEpsilon)
{ {
switch (t.TransitionType) switch (t.TransitionType)
{ {
@ -1748,6 +1895,22 @@ namespace Antlr4.Runtime.Atn
return config.Transform(t.target, false); return config.Transform(t.target, false);
} }
case TransitionType.Atom:
case TransitionType.Range:
case TransitionType.Set:
{
// EOF transitions act like epsilon transitions after the first EOF
// transition is traversed
if (treatEofAsEpsilon)
{
if (t.Matches(TokenConstants.Eof, 0, 1))
{
return config.Transform(t.target, false);
}
}
return null;
}
default: default:
{ {
return null; return null;
@ -1816,9 +1979,9 @@ namespace Antlr4.Runtime.Atn
return config.Transform(t.target, newContext, false); return config.Transform(t.target, newContext, false);
} }
private sealed class _IComparer_1771 : IComparer<ATNConfig> private sealed class _IComparer_1905 : IComparer<ATNConfig>
{ {
public _IComparer_1771() public _IComparer_1905()
{ {
} }
@ -1838,7 +2001,7 @@ namespace Antlr4.Runtime.Atn
} }
} }
private static readonly IComparer<ATNConfig> StateAltSortComparator = new _IComparer_1771(); private static readonly IComparer<ATNConfig> StateAltSortComparator = new _IComparer_1905();
private BitSet IsConflicted(ATNConfigSet configset, PredictionContextCache contextCache) private BitSet IsConflicted(ATNConfigSet configset, PredictionContextCache contextCache)
{ {

View File

@ -71,7 +71,22 @@ namespace Antlr4.Runtime.Atn
get; get;
} }
/// <summary>Are we epsilon, action, sempred?</summary> /// <summary>Determines if the transition is an "epsilon" transition.</summary>
/// <remarks>
/// Determines if the transition is an "epsilon" transition.
/// <p>The default implementation returns
/// <code>false</code>
/// .</p>
/// </remarks>
/// <returns>
///
/// <code>true</code>
/// if traversing this transition in the ATN does not
/// consume an input symbol; otherwise,
/// <code>false</code>
/// if traversing this
/// transition consumes (matches) an input symbol.
/// </returns>
public virtual bool IsEpsilon public virtual bool IsEpsilon
{ {
get get

View File

@ -252,7 +252,7 @@ namespace Antlr4.Runtime.Dfa
{ {
if (IsPrecedenceDfa()) if (IsPrecedenceDfa())
{ {
return s0full.Get().EdgeMap.Count == 0; return s0full.Get().EdgeMap.Count != 0;
} }
return s0full.Get() != null; return s0full.Get() != null;
} }