Initial implementation of a "precedence DFA" (fixes #400)

runtime/Java/src/org/antlr/v4/runtime/Parser.java

@@ -647,6 +647,20 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
 		_ctx = localctx;
 	}
 
+	/**
+	 * Get the precedence level for the top-most precedence rule.
+	 *
+	 * @return The precedence level for the top-most precedence rule, or -1 if
+	 * the parser context is not nested within a precedence rule.
+	 */
+	public final int getPrecedence() {
+		if (_precedenceStack.isEmpty()) {
+			return -1;
+		}
+
+		return _precedenceStack.peek();
+	}
+
 	/**
 	 * @deprecated Use
 	 * {@link #enterRecursionRule(ParserRuleContext, int, int, int)} instead.

runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java

@@ -325,23 +325,51 @@ public class ParserATNSimulator extends ATNSimulator {
 		// Now we are certain to have a specific decision's DFA
 		// But, do we still need an initial state?
 		try {
-			if ( dfa.s0==null ) {
+			DFAState s0;
+			if (dfa.isPrecedenceDfa()) {
+				s0 = dfa.getPrecedenceStartState(parser.getPrecedence());
+			}
+			else {
+				s0 = dfa.s0;
+			}
+
+			if (s0 == null) {
 				if ( outerContext ==null ) outerContext = ParserRuleContext.EMPTY;
 				if ( debug || debug_list_atn_decisions )  {
 					System.out.println("predictATN decision "+ dfa.decision+
 									   " exec LA(1)=="+ getLookaheadName(input) +
 									   ", outerContext="+ outerContext.toString(parser));
 				}
+
+				if (!dfa.isPrecedenceDfa() && dfa.atnStartState instanceof StarLoopEntryState) {
+					if (atn.ruleToStartState[dfa.atnStartState.ruleIndex].isPrecedenceRule) {
+						ATNState maybeLoopEndState = dfa.atnStartState.transition(dfa.atnStartState.getNumberOfTransitions() - 1).target;
+						if (maybeLoopEndState instanceof LoopEndState) {
+							if (maybeLoopEndState.epsilonOnlyTransitions && maybeLoopEndState.transition(0).target instanceof RuleStopState) {
+								dfa.setPrecedenceDfa(true);
+							}
+						}
+					}
+				}
+
 				boolean fullCtx = false;
 				ATNConfigSet s0_closure =
 					computeStartState(dfa.atnStartState,
 									  ParserRuleContext.EMPTY,
 									  fullCtx);
-				dfa.s0 = addDFAState(dfa, new DFAState(s0_closure));
+
+				if (dfa.isPrecedenceDfa()) {
+					s0_closure = applyPrecedenceFilter(s0_closure);
+					s0 = addDFAState(dfa, new DFAState(s0_closure));
+					dfa.setPrecedenceStartState(parser.getPrecedence(), s0);
+				}
+				else {
+					s0 = addDFAState(dfa, new DFAState(s0_closure));
+					dfa.s0 = s0;
+				}
 			}
 
-			// We can start with an existing DFA
+			int alt = execATN(dfa, s0, input, index, outerContext);
-			int alt = execATN(dfa, dfa.s0, input, index, outerContext);
 			if ( debug ) System.out.println("DFA after predictATN: "+ dfa.toString(parser.getTokenNames()));
 			return alt;
 		}
@@ -907,6 +935,48 @@ public class ParserATNSimulator extends ATNSimulator {
 		return configs;
 	}
 
+	@NotNull
+	protected ATNConfigSet applyPrecedenceFilter(@NotNull ATNConfigSet configs) {
+		Set<Integer> statesFromAlt1 = new HashSet<Integer>();
+		ATNConfigSet configSet = new ATNConfigSet(configs.fullCtx);
+		for (ATNConfig config : configs) {
+			// handle alt 1 first
+			if (config.alt != 1) {
+				continue;
+			}
+
+			SemanticContext updatedContext = config.semanticContext.evalPrecedence(parser, _outerContext);
+			if (updatedContext == null) {
+				// the configuration was eliminated
+				continue;
+			}
+
+			statesFromAlt1.add(config.state.stateNumber);
+			if (updatedContext != config.semanticContext) {
+				configSet.add(new ATNConfig(config, updatedContext), mergeCache);
+			}
+			else {
+				configSet.add(config, mergeCache);
+			}
+		}
+
+		for (ATNConfig config : configs) {
+			if (config.alt == 1) {
+				// already handled
+				continue;
+			}
+
+			if (statesFromAlt1.contains(config.state.stateNumber)) {
+				// eliminated
+				continue;
+			}
+
+			configSet.add(config, mergeCache);
+		}
+
+		return configSet;
+	}
+
 	@Nullable
 	protected ATNState getReachableTarget(@NotNull Transition trans, int ttype) {
 		if (trans.matches(ttype, 0, atn.maxTokenType)) {

runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java

@@ -72,6 +72,28 @@ public abstract class SemanticContext {
 	 */
     public abstract boolean eval(Recognizer<?,?> parser, RuleContext outerContext);
 
+	/**
+	 * Evaluate the precedence predicates for the context and reduce the result.
+	 *
+	 * @param parser The parser instance.
+	 * @param outerContext The current parser context object.
+	 * @return The simplified semantic context after precedence predicates are
+	 * evaluated, which will be one of the following values.
+	 * <ul>
+	 * <li>{@link #NONE}: if the predicate simplifies to {@code true} after
+	 * precedence predicates are evaluated.</li>
+	 * <li>{@code null}: if the predicate simplifies to {@code false} after
+	 * precedence predicates are evaluated.</li>
+	 * <li>{@code this}: if the semantic context is not changed as a result of
+	 * precedence predicate evaluation.</li>
+	 * <li>A non-{@code null} {@link SemanticContext}: the new simplified
+	 * semantic context after precedence predicates are evaluated.</li>
+	 * </ul>
+	 */
+	public SemanticContext evalPrecedence(Recognizer<?,?> parser, RuleContext outerContext) {
+		return this;
+	}
+
     public static class Predicate extends SemanticContext {
         public final int ruleIndex;
        	public final int predIndex;
@@ -137,6 +159,16 @@ public abstract class SemanticContext {
 			return parser.precpred(outerContext, precedence);
 		}
 
+		@Override
+		public SemanticContext evalPrecedence(Recognizer<?, ?> parser, RuleContext outerContext) {
+			if (parser.precpred(outerContext, precedence)) {
+				return SemanticContext.NONE;
+			}
+			else {
+				return null;
+			}
+		}
+
 		@Override
 		public int compareTo(PrecedencePredicate o) {
 			return precedence - o.precedence;
@@ -210,6 +242,40 @@ public abstract class SemanticContext {
 			return true;
         }
 
+		@Override
+		public SemanticContext evalPrecedence(Recognizer<?, ?> parser, RuleContext outerContext) {
+			boolean differs = false;
+			List<SemanticContext> operands = new ArrayList<SemanticContext>();
+			for (SemanticContext context : opnds) {
+				SemanticContext evaluated = context.evalPrecedence(parser, outerContext);
+				differs |= (evaluated != context);
+				if (evaluated == null) {
+					// The AND context is false if any element is false
+					return null;
+				}
+				else if (evaluated != NONE) {
+					// Reduce the result by skipping true elements
+					operands.add(evaluated);
+				}
+			}
+
+			if (!differs) {
+				return this;
+			}
+
+			if (operands.isEmpty()) {
+				// all elements were true, so the AND context is true
+				return NONE;
+			}
+
+			SemanticContext result = operands.get(0);
+			for (int i = 1; i < operands.size(); i++) {
+				result = SemanticContext.and(result, operands.get(i));
+			}
+
+			return result;
+		}
+
 		@Override
 		public String toString() {
 			return Utils.join(Arrays.asList(opnds).iterator(), "&&");
@@ -257,6 +323,40 @@ public abstract class SemanticContext {
 			return false;
         }
 
+		@Override
+		public SemanticContext evalPrecedence(Recognizer<?, ?> parser, RuleContext outerContext) {
+			boolean differs = false;
+			List<SemanticContext> operands = new ArrayList<SemanticContext>();
+			for (SemanticContext context : opnds) {
+				SemanticContext evaluated = context.evalPrecedence(parser, outerContext);
+				differs |= (evaluated != context);
+				if (evaluated == NONE) {
+					// The OR context is true if any element is true
+					return NONE;
+				}
+				else if (evaluated != null) {
+					// Reduce the result by skipping false elements
+					operands.add(evaluated);
+				}
+			}
+
+			if (!differs) {
+				return this;
+			}
+
+			if (operands.isEmpty()) {
+				// all elements were false, so the OR context is false
+				return null;
+			}
+
+			SemanticContext result = operands.get(0);
+			for (int i = 1; i < operands.size(); i++) {
+				result = SemanticContext.or(result, operands.get(i));
+			}
+
+			return result;
+		}
+
         @Override
         public String toString() {
 			return Utils.join(Arrays.asList(opnds).iterator(), "||");

runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java

@@ -29,11 +29,14 @@
  */
 package org.antlr.v4.runtime.dfa;
 
+import org.antlr.v4.runtime.atn.ATNConfigSet;
 import org.antlr.v4.runtime.atn.DecisionState;
 import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.misc.Nullable;
+import org.antlr.v4.runtime.Parser;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -48,7 +51,7 @@ public class DFA {
     @NotNull
 	public final Map<DFAState, DFAState> states = new HashMap<DFAState, DFAState>();
 	@Nullable
-	public DFAState s0;
+	public volatile DFAState s0;
 
 	public final int decision;
 
@@ -56,6 +59,13 @@ public class DFA {
 	@NotNull
 	public final DecisionState atnStartState;
 
+	/**
+	 * {@code true} if this DFA is for a precedence decision; otherwise,
+	 * {@code false}. This is the backing field for {@link #isPrecedenceDfa},
+	 * {@link #setPrecedenceDfa}, {@link #hasPrecedenceEdge}.
+	 */
+	private volatile boolean precedenceDfa;
+
 	public DFA(@NotNull DecisionState atnStartState) {
 		this(atnStartState, 0);
 	}
@@ -65,6 +75,112 @@ public class DFA {
 		this.decision = decision;
 	}
 
+	/**
+	 * Gets whether this DFA is a precedence DFA. Precedence DFAs use a special
+	 * start state {@link #s0} which is not stored in {@link #states}. The
+	 * {@link DFAState#edges} array for this start state contains outgoing edges
+	 * supplying individual start states corresponding to specific precedence
+	 * values.
+	 *
+	 * @return {@code true} if this is a precedence DFA; otherwise,
+	 * {@code false}.
+	 * @see Parser#getPrecedence()
+	 */
+	public final boolean isPrecedenceDfa() {
+		return precedenceDfa;
+	}
+
+	/**
+	 * Get the start state for a specific precedence value.
+	 *
+	 * @param precedence The current precedence.
+	 * @return The start state corresponding to the specified precedence, or
+	 * {@code null} if no start state exists for the specified precedence.
+	 *
+	 * @throws IllegalStateException if this is not a precedence DFA.
+	 * @see #isPrecedenceDfa()
+	 */
+	@SuppressWarnings("null")
+	public final DFAState getPrecedenceStartState(int precedence) {
+		if (!isPrecedenceDfa()) {
+			throw new IllegalStateException("Only precedence DFAs may contain a precedence start state.");
+		}
+
+		// s0.edges is never null for a precedence DFA
+		if (precedence < 0 || precedence >= s0.edges.length) {
+			return null;
+		}
+
+		return s0.edges[precedence];
+	}
+
+	/**
+	 * Set the start state for a specific precedence value.
+	 *
+	 * @param precedence The current precedence.
+	 * @param startState The start state corresponding to the specified
+	 * precedence.
+	 *
+	 * @throws IllegalStateException if this is not a precedence DFA.
+	 * @see #isPrecedenceDfa()
+	 */
+	@SuppressWarnings({"SynchronizeOnNonFinalField", "null"})
+	public final void setPrecedenceStartState(int precedence, DFAState startState) {
+		if (!isPrecedenceDfa()) {
+			throw new IllegalStateException("Only precedence DFAs may contain a precedence start state.");
+		}
+
+		if (precedence < 0) {
+			return;
+		}
+
+		// synchronization on s0 here is ok. when the DFA is turned into a
+		// precedence DFA, s0 will be initialized once and not updated again
+		synchronized (s0) {
+			// s0.edges is never null for a precedence DFA
+			if (precedence >= s0.edges.length) {
+				s0.edges = Arrays.copyOf(s0.edges, precedence + 1);
+			}
+
+			s0.edges[precedence] = startState;
+		}
+	}
+
+	/**
+	 * Sets whether this is a precedence DFA. If the specified value differs
+	 * from the current DFA configuration, the following actions are taken;
+	 * otherwise no changes are made to the current DFA.
+	 *
+	 * <ul>
+	 * <li>The {@link #states} map is cleared</li>
+	 * <li>If {@code precedenceDfa} is {@code false}, the initial state
+	 * {@link #s0} is set to {@code null}; otherwise, it is initialized to a new
+	 * {@link DFAState} with an empty outgoing {@link DFAState#edges} array to
+	 * store the start states for individual precedence values.</li>
+	 * <li>The {@link #precedenceDfa} field is updated</li>
+	 * </ul>
+	 *
+	 * @param precedenceDfa {@code true} if this is a precedence DFA; otherwise,
+	 * {@code false}
+	 */
+	public final synchronized void setPrecedenceDfa(boolean precedenceDfa) {
+		if (this.precedenceDfa != precedenceDfa) {
+			this.states.clear();
+			if (precedenceDfa) {
+				DFAState precedenceState = new DFAState(new ATNConfigSet());
+				precedenceState.edges = new DFAState[0];
+				precedenceState.isAcceptState = false;
+				precedenceState.requiresFullContext = false;
+				this.s0 = precedenceState;
+			}
+			else {
+				this.s0 = null;
+			}
+
+			this.precedenceDfa = precedenceDfa;
+		}
+	}
+
 	/**
 	 * Return a list of all states in this DFA, ordered by state number.
 	 */

tool/test/org/antlr/v4/test/TestLeftRecursion.java

@@ -394,17 +394,10 @@ public class TestLeftRecursion extends BaseTest {
 		assertNull(stderrDuringParse);
 
 		result = execParser("Expr.g4", grammar, "ExprParser", "ExprLexer", "prog", "a+b*2\n", true);
-		assertEquals("line 1:1 reportAttemptingFullContext d=3 (expr), input='+'\n" +
+		assertNull(stderrDuringParse);
-					 "line 1:1 reportContextSensitivity d=3 (expr), input='+'\n" +
-					 "line 1:3 reportAttemptingFullContext d=3 (expr), input='*'\n",
-					 stderrDuringParse);
 
 		result = execParser("Expr.g4", grammar, "ExprParser", "ExprLexer", "prog", "(1+2)*3\n", true);
-		assertEquals("line 1:2 reportAttemptingFullContext d=3 (expr), input='+'\n" +
+		assertNull(stderrDuringParse);
-					 "line 1:2 reportContextSensitivity d=3 (expr), input='+'\n" +
-					 "line 1:5 reportAttemptingFullContext d=3 (expr), input='*'\n" +
-					 "line 1:5 reportContextSensitivity d=3 (expr), input='*'\n",
-					 stderrDuringParse);
 	}
 
 	@Test public void testCheckForNonLeftRecursiveRule() throws Exception {

tool/test/org/antlr/v4/test/TestParseErrors.java

@@ -320,27 +320,15 @@ public class TestParseErrors extends BaseTest {
 
 		result = execParser("T.g4", grammar, "TParser", "TLexer", "start", "xx", true);
 		assertEquals("", result);
-		assertEquals(
+		assertNull(this.stderrDuringParse);
-			"line 1:1 reportAttemptingFullContext d=0 (expr), input='x'\n" +
-			"line 1:1 reportContextSensitivity d=0 (expr), input='x'\n",
-			this.stderrDuringParse);
 
 		result = execParser("T.g4", grammar, "TParser", "TLexer", "start", "xxx", true);
 		assertEquals("", result);
-		assertEquals(
+		assertNull(this.stderrDuringParse);
-			"line 1:1 reportAttemptingFullContext d=0 (expr), input='x'\n" +
-			"line 1:1 reportContextSensitivity d=0 (expr), input='x'\n" +
-			"line 1:2 reportAttemptingFullContext d=0 (expr), input='x'\n",
-			this.stderrDuringParse);
 
 		result = execParser("T.g4", grammar, "TParser", "TLexer", "start", "xxxx", true);
 		assertEquals("", result);
-		assertEquals(
+		assertNull(this.stderrDuringParse);
-			"line 1:1 reportAttemptingFullContext d=0 (expr), input='x'\n" +
-			"line 1:1 reportContextSensitivity d=0 (expr), input='x'\n" +
-			"line 1:2 reportAttemptingFullContext d=0 (expr), input='x'\n" +
-			"line 1:3 reportAttemptingFullContext d=0 (expr), input='x'\n",
-			this.stderrDuringParse);
 	}
 
 	/**