From ee647907391fbdbc0fa6e64624aee89ced3a3197 Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Sun, 28 Oct 2012 21:43:12 -0500
Subject: [PATCH] In lexer, configs which never reached a non-greedy decision
 state are fully greedy (unordered alternatives, longest match)

---
 .../antlr/v4/runtime/atn/LexerATNConfig.java  | 41 +++++++++++++++++++
 .../v4/runtime/atn/LexerATNSimulator.java     | 36 ++++++++--------
 2 files changed, 60 insertions(+), 17 deletions(-)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java
index ca25d07d4..7e211076b 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java
@@ -37,11 +37,14 @@ public class LexerATNConfig extends ATNConfig {
 	/** Capture lexer action we traverse */
 	public int lexerActionIndex = -1;
 
+	private final boolean passedThroughNonGreedyDecision;
+
 	public LexerATNConfig(@NotNull ATNState state,
 						  int alt,
 						  @Nullable PredictionContext context)
 	{
 		super(state, alt, context, SemanticContext.NONE);
+		this.passedThroughNonGreedyDecision = false;
 	}
 
 	public LexerATNConfig(@NotNull ATNState state,
@@ -51,17 +54,20 @@ public class LexerATNConfig extends ATNConfig {
 	{
 		super(state, alt, context, SemanticContext.NONE);
 		this.lexerActionIndex = actionIndex;
+		this.passedThroughNonGreedyDecision = false;
 	}
 
 	public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state) {
 		super(c, state, c.context, c.semanticContext);
 		this.lexerActionIndex = c.lexerActionIndex;
+		this.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state);
 	}
 
 	public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state,
 						  @NotNull SemanticContext semanticContext) {
 		super(c, state, c.context, semanticContext);
 		this.lexerActionIndex = c.lexerActionIndex;
+		this._passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state);
 	}
 
 	public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state,
@@ -69,11 +75,46 @@ public class LexerATNConfig extends ATNConfig {
 	{
 		super(c, state, c.context, c.semanticContext);
 		this.lexerActionIndex = actionIndex;
+		this.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state);
 	}
 
 	public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state,
 						  @Nullable PredictionContext context) {
 		super(c, state, context, c.semanticContext);
 		this.lexerActionIndex = c.lexerActionIndex;
+		this.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state);
+	}
+
+	public final boolean hasPassedThroughNonGreedyDecision() {
+		return passedThroughNonGreedyDecision;
+	}
+
+	@Override
+	public int hashCode() {
+		int hashCode = super.hashCode();
+		hashCode = 35 * hashCode ^ (passedThroughNonGreedyDecision ? 1 : 0);
+		return hashCode;
+	}
+
+	@Override
+	public boolean equals(ATNConfig other) {
+		if (this == other) {
+			return true;
+		}
+		else if (!(other instanceof LexerATNConfig)) {
+			return false;
+		}
+
+		LexerATNConfig lexerOther = (LexerATNConfig)other;
+		if (passedThroughNonGreedyDecision != lexerOther.passedThroughNonGreedyDecision) {
+			return false;
+		}
+
+		return super.equals(other);
+	}
+
+	private static boolean checkNonGreedyDecision(LexerATNConfig source, ATNState target) {
+		return source.passedThroughNonGreedyDecision
+			|| target instanceof DecisionState && ((DecisionState)target).nonGreedy;
 	}
 }
diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
index 8eebdd5b5..fb3512879 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
@@ -384,14 +384,16 @@ public class LexerATNSimulator extends ATNSimulator {
 	}
 
 	/** Given a starting configuration set, figure out all ATN configurations
-	 *  we can reach upon input t. Parameter reach is a return parameter.
+	 *  we can reach upon input {@code t}. Parameter {@code reach} is a return
+	 *  parameter.
 	 */
 	protected void getReachableConfigSet(ATNConfigSet closure, ATNConfigSet reach, int t) {
 		// this is used to skip processing for configs which have a lower priority
 		// than a config that already reached an accept state for the same rule
 		int skipAlt = ATN.INVALID_ALT_NUMBER;
 		for (ATNConfig c : closure) {
-			if (c.alt == skipAlt) {
+			boolean currentAltReachedAcceptState = c.alt == skipAlt;
+			if (currentAltReachedAcceptState && ((LexerATNConfig)c).hasPassedThroughNonGreedyDecision()) {
 				continue;
 			}
 
@@ -404,7 +406,7 @@ public class LexerATNSimulator extends ATNSimulator {
 				Transition trans = c.state.transition(ti);
 				ATNState target = getReachableTarget(trans, t);
 				if ( target!=null ) {
-					if (closure(new LexerATNConfig((LexerATNConfig)c, target), reach)) {
+					if (closure(new LexerATNConfig((LexerATNConfig)c, target), reach, currentAltReachedAcceptState)) {
 						// any remaining configs for this alt have a lower priority than
 						// the one that just reached an accept state.
 						skipAlt = c.alt;
@@ -481,7 +483,7 @@ public class LexerATNSimulator extends ATNSimulator {
 		for (int i=0; i<p.getNumberOfTransitions(); i++) {
 			ATNState target = p.transition(i).target;
 			LexerATNConfig c = new LexerATNConfig(target, i+1, initialContext);
-			closure(c, configs);
+			closure(c, configs, false);
 		}
 		return configs;
 	}
@@ -496,7 +498,7 @@ public class LexerATNSimulator extends ATNSimulator {
 	 * @return {@code true} if an accept state is reached, otherwise
 	 * {@code false}.
 	 */
-	protected boolean closure(@NotNull LexerATNConfig config, @NotNull ATNConfigSet configs) {
+	protected boolean closure(@NotNull LexerATNConfig config, @NotNull ATNConfigSet configs, boolean currentAltReachedAcceptState) {
 		if ( debug ) {
 			System.out.println("closure("+config.toString(recog, true)+")");
 		}
@@ -514,12 +516,12 @@ public class LexerATNSimulator extends ATNSimulator {
 			if ( config.context == null || config.context.hasEmptyPath() ) {
 				if (config.context == null || config.context.isEmpty()) {
 					configs.add(config);
+					return true;
 				}
 				else {
 					configs.add(new LexerATNConfig(config, config.state, PredictionContext.EMPTY));
+					currentAltReachedAcceptState = true;
 				}
-
-				return true;
 			}
 
 			if ( config.context!=null && !config.context.isEmpty() ) {
@@ -540,18 +542,19 @@ public class LexerATNSimulator extends ATNSimulator {
 						RuleTransition rt = (RuleTransition)invokingState.transition(0);
 						ATNState retState = rt.followState;
 						LexerATNConfig c = new LexerATNConfig(retState, config.alt, newContext);
-						if (closure(c, configs)) {
-							return true;
-						}
+						currentAltReachedAcceptState = closure(c, configs, currentAltReachedAcceptState);
 					}
 				}
 			}
-			return false;
+
+			return currentAltReachedAcceptState;
 		}
 
 		// optimization
-		if ( !config.state.onlyHasEpsilonTransitions() )	{
-			configs.add(config);
+		if ( !config.state.onlyHasEpsilonTransitions() ) {
+			if (!currentAltReachedAcceptState || !config.hasPassedThroughNonGreedyDecision()) {
+				configs.add(config);
+			}
 		}
 
 		ATNState p = config.state;
@@ -559,13 +562,11 @@ public class LexerATNSimulator extends ATNSimulator {
 			Transition t = p.transition(i);
 			LexerATNConfig c = getEpsilonTarget(config, t, configs);
 			if ( c!=null ) {
-				if (closure(c, configs)) {
-					return true;
-				}
+				currentAltReachedAcceptState = closure(c, configs, currentAltReachedAcceptState);
 			}
 		}
 
-		return false;
+		return currentAltReachedAcceptState;
 	}
 
 	// side-effect: can alter configs.hasSemanticContext
@@ -582,6 +583,7 @@ public class LexerATNSimulator extends ATNSimulator {
 					SingletonPredictionContext.create(config.context, p.stateNumber);
 				c = new LexerATNConfig(config, t.target, newContext);
 				break;
+
 			case Transition.PREDICATE:
 //				if (recog == null) {
 //					System.out.format("Predicates cannot be evaluated without a recognizer; assuming true.\n");