jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

In lexer, configs which never reached a non-greedy decision state are fully greedy (unordered alternatives, longest match)

This commit is contained in:
Sam Harwell 2012-10-28 21:43:12 -05:00
parent 871db85ecb
commit ee64790739
2 changed files with 60 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java
View File

@ -37,11 +37,14 @@ public class LexerATNConfig extends ATNConfig {
/** Capture lexer action we traverse */
public int lexerActionIndex = -1;
private final boolean passedThroughNonGreedyDecision;
public LexerATNConfig(@NotNull ATNState state,
int alt,
@Nullable PredictionContext context)
{
super(state, alt, context, SemanticContext.NONE);
this.passedThroughNonGreedyDecision = false;
}
public LexerATNConfig(@NotNull ATNState state,
@ -51,17 +54,20 @@ public class LexerATNConfig extends ATNConfig {
{
super(state, alt, context, SemanticContext.NONE);
this.lexerActionIndex = actionIndex;
this.passedThroughNonGreedyDecision = false;
}
public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state) {
super(c, state, c.context, c.semanticContext);
this.lexerActionIndex = c.lexerActionIndex;
this.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state);
}
public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state,
@NotNull SemanticContext semanticContext) {
super(c, state, c.context, semanticContext);
this.lexerActionIndex = c.lexerActionIndex;
this._passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state);
}
public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state,
@ -69,11 +75,46 @@ public class LexerATNConfig extends ATNConfig {
{
super(c, state, c.context, c.semanticContext);
this.lexerActionIndex = actionIndex;
this.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state);
}
public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state,
@Nullable PredictionContext context) {
super(c, state, context, c.semanticContext);
this.lexerActionIndex = c.lexerActionIndex;
this.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state);
}
public final boolean hasPassedThroughNonGreedyDecision() {
return passedThroughNonGreedyDecision;
}
@Override
public int hashCode() {
int hashCode = super.hashCode();
hashCode = 35 * hashCode ^ (passedThroughNonGreedyDecision ? 1 : 0);
return hashCode;
}
@Override
public boolean equals(ATNConfig other) {
if (this == other) {
return true;
}
else if (!(other instanceof LexerATNConfig)) {
return false;
}
LexerATNConfig lexerOther = (LexerATNConfig)other;
if (passedThroughNonGreedyDecision != lexerOther.passedThroughNonGreedyDecision) {
return false;
}
return super.equals(other);
}
private static boolean checkNonGreedyDecision(LexerATNConfig source, ATNState target) {
return source.passedThroughNonGreedyDecision
|| target instanceof DecisionState && ((DecisionState)target).nonGreedy;
}
}

36
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -384,14 +384,16 @@ public class LexerATNSimulator extends ATNSimulator {
}
/** Given a starting configuration set, figure out all ATN configurations
* we can reach upon input t. Parameter reach is a return parameter.
* we can reach upon input {@code t}. Parameter {@code reach} is a return
* parameter.
*/
protected void getReachableConfigSet(ATNConfigSet closure, ATNConfigSet reach, int t) {
// this is used to skip processing for configs which have a lower priority
// than a config that already reached an accept state for the same rule
int skipAlt = ATN.INVALID_ALT_NUMBER;
for (ATNConfig c : closure) {
if (c.alt == skipAlt) {
boolean currentAltReachedAcceptState = c.alt == skipAlt;
if (currentAltReachedAcceptState && ((LexerATNConfig)c).hasPassedThroughNonGreedyDecision()) {
continue;
}
@ -404,7 +406,7 @@ public class LexerATNSimulator extends ATNSimulator {
Transition trans = c.state.transition(ti);
ATNState target = getReachableTarget(trans, t);
if ( target!=null ) {
if (closure(new LexerATNConfig((LexerATNConfig)c, target), reach)) {
if (closure(new LexerATNConfig((LexerATNConfig)c, target), reach, currentAltReachedAcceptState)) {
// any remaining configs for this alt have a lower priority than
// the one that just reached an accept state.
skipAlt = c.alt;
@ -481,7 +483,7 @@ public class LexerATNSimulator extends ATNSimulator {
for (int i=0; i<p.getNumberOfTransitions(); i++) {
ATNState target = p.transition(i).target;
LexerATNConfig c = new LexerATNConfig(target, i+1, initialContext);
closure(c, configs);
closure(c, configs, false);
}
return configs;
}
@ -496,7 +498,7 @@ public class LexerATNSimulator extends ATNSimulator {
* @return {@code true} if an accept state is reached, otherwise
* {@code false}.
*/
protected boolean closure(@NotNull LexerATNConfig config, @NotNull ATNConfigSet configs) {
protected boolean closure(@NotNull LexerATNConfig config, @NotNull ATNConfigSet configs, boolean currentAltReachedAcceptState) {
if ( debug ) {
System.out.println("closure("+config.toString(recog, true)+")");
}
@ -514,12 +516,12 @@ public class LexerATNSimulator extends ATNSimulator {
if ( config.context == null || config.context.hasEmptyPath() ) {
if (config.context == null || config.context.isEmpty()) {
configs.add(config);
return true;
}
else {
configs.add(new LexerATNConfig(config, config.state, PredictionContext.EMPTY));
currentAltReachedAcceptState = true;
}
return true;
}
if ( config.context!=null && !config.context.isEmpty() ) {
@ -540,18 +542,19 @@ public class LexerATNSimulator extends ATNSimulator {
RuleTransition rt = (RuleTransition)invokingState.transition(0);
ATNState retState = rt.followState;
LexerATNConfig c = new LexerATNConfig(retState, config.alt, newContext);
if (closure(c, configs)) {
return true;
}
currentAltReachedAcceptState = closure(c, configs, currentAltReachedAcceptState);
}
}
}
return false;
return currentAltReachedAcceptState;
}
// optimization
if ( !config.state.onlyHasEpsilonTransitions() ) {
configs.add(config);
if ( !config.state.onlyHasEpsilonTransitions() ) {
if (!currentAltReachedAcceptState || !config.hasPassedThroughNonGreedyDecision()) {
configs.add(config);
}
}
ATNState p = config.state;
@ -559,13 +562,11 @@ public class LexerATNSimulator extends ATNSimulator {
Transition t = p.transition(i);
LexerATNConfig c = getEpsilonTarget(config, t, configs);
if ( c!=null ) {
if (closure(c, configs)) {
return true;
}
currentAltReachedAcceptState = closure(c, configs, currentAltReachedAcceptState);
}
}
return false;
return currentAltReachedAcceptState;
}
// side-effect: can alter configs.hasSemanticContext
@ -582,6 +583,7 @@ public class LexerATNSimulator extends ATNSimulator {
SingletonPredictionContext.create(config.context, p.stateNumber);
c = new LexerATNConfig(config, t.target, newContext);
break;
case Transition.PREDICATE:
// if (recog == null) {
// System.out.format("Predicates cannot be evaluated without a recognizer; assuming true.\n");