Merge pull request #597 from parrt/track-all-preds

Track all preds

runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java

@@ -1265,7 +1265,7 @@ public class ParserATNSimulator extends ATNSimulator {
 		ATNConfigSet failed = new ATNConfigSet(configs.fullCtx);
 		for (ATNConfig c : configs) {
 			if ( c.semanticContext!=SemanticContext.NONE ) {
-				boolean predicateEvaluationResult = c.semanticContext.eval(parser, outerContext);
+				boolean predicateEvaluationResult = evalSemanticContext(c.semanticContext, outerContext, c.alt, configs.fullCtx);
 				if ( predicateEvaluationResult ) {
 					succeeded.add(c);
 				}
@@ -1300,7 +1300,8 @@ public class ParserATNSimulator extends ATNSimulator {
 				continue;
 			}
 
-			boolean predicateEvaluationResult = pair.pred.eval(parser, outerContext);
+			boolean fullCtx = false; // in dfa
+			boolean predicateEvaluationResult = evalSemanticContext(pair.pred, outerContext, pair.alt, fullCtx);
 			if ( debug || dfa_debug ) {
 				System.out.println("eval pred "+pair+"="+predicateEvaluationResult);
 			}
@@ -1317,6 +1318,37 @@ public class ParserATNSimulator extends ATNSimulator {
 		return predictions;
 	}
 
+	/**
+	 * Evaluate a semantic context within a specific parser context.
+	 *
+	 * <p>
+	 * This method might not be called for every semantic context evaluated
+	 * during the prediction process. In particular, the following restrictions
+	 * are allowed:</p>
+	 *
+	 * <ul>
+	 * <li>Precedence predicates (represented by
+	 * {@link SemanticContext.PrecedencePredicate}) may or may not be evaluated
+	 * through this method.</li>
+	 * <li>Operator predicates (represented by {@link SemanticContext.AND} and
+	 * {@link SemanticContext.OR}) may be evaluated as a single semantic
+	 * context, rather than evaluating the operands individually.
+	 * Implementations which require evaluation results from individual
+	 * predicates should override this method to explicitly handle evaluation of
+	 * the operands within operator predicates.</li>
+	 * </ul>
+	 *
+	 * @param pred The semantic context to evaluate
+	 * @param parserCallStack The parser context in which to evaluate the
+	 * semantic context
+	 * @param alt The alternative which is guarded by {@code pred}
+	 * @param fullCtx {@code true} if the evaluation is occurring during LL
+	 * prediction; otherwise, {@code false} if the evaluation is occurring
+	 * during SLL prediction
+	 */
+	protected boolean evalSemanticContext(@NotNull SemanticContext pred, ParserRuleContext parserCallStack, int alt, boolean fullCtx) {
+		return pred.eval(parser, parserCallStack);
+	}
 
 	/* TODO: If we are doing predicates, there is no point in pursuing
 		 closure operations if we reach a DFA state that uniquely predicts
@@ -1544,7 +1576,7 @@ public class ParserATNSimulator extends ATNSimulator {
 				// later during conflict resolution.
 				int currentPosition = _input.index();
 				_input.seek(_startIndex);
-				boolean predSucceeds = pt.getPredicate().eval(parser, _outerContext);
+				boolean predSucceeds = evalSemanticContext(pt.getPredicate(), _outerContext, config.alt, fullCtx);
 				_input.seek(currentPosition);
 				if ( predSucceeds ) {
 					c = new ATNConfig(config, pt.target); // no pred context
@@ -1592,7 +1624,7 @@ public class ParserATNSimulator extends ATNSimulator {
 				// later during conflict resolution.
 				int currentPosition = _input.index();
 				_input.seek(_startIndex);
-				boolean predSucceeds = pt.getPredicate().eval(parser, _outerContext);
+				boolean predSucceeds = evalSemanticContext(pt.getPredicate(), _outerContext, config.alt, fullCtx);
 				_input.seek(currentPosition);
 				if ( predSucceeds ) {
 					c = new ATNConfig(config, pt.target); // no pred context

runtime/Java/src/org/antlr/v4/runtime/atn/PredicateEvalInfo.java

@@ -30,12 +30,12 @@
 
 package org.antlr.v4.runtime.atn;
 
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.Recognizer;
+import org.antlr.v4.runtime.RuleContext;
 import org.antlr.v4.runtime.TokenStream;
-import org.antlr.v4.runtime.dfa.DFAState;
 import org.antlr.v4.runtime.misc.NotNull;
 
-import java.util.BitSet;
-
 /**
  * This class represents profiling event information for semantic predicate
  * evaluations which occur during prediction.
@@ -44,52 +44,53 @@ import java.util.BitSet;
  */
 public class PredicateEvalInfo extends DecisionEventInfo {
 	/**
-	 * The DFA state at which predicate evaluation is required in order to
+	 * The semantic context which was evaluated.
-	 * continue.
 	 */
-	public final DFAState dfaState;
+	public final SemanticContext semctx;
 	/**
-	 * The results of evaluating specific semantic contexts. The elements of
+	 * The alternative number for the decision which is guarded by the semantic
-	 * this array correspond to the elements in {@link DFAState#predicates}, and
+	 * context {@link #semctx}. Note that other ATN
-	 * the value of each element is the result of evaluating the semantic
+	 * configurations may predict the same alternative which are guarded by
-	 * context {@link DFAState.PredPrediction#pred}.
+	 * other semantic contexts and/or {@link SemanticContext#NONE}.
 	 */
-	public final boolean[] evalResults;
+	public final int predictedAlt;
 	/**
-	 * A {@link BitSet} identifying the represented alternatives of
+	 * The result of evaluating the semantic context {@link #semctx}.
-	 * {@link #dfaState} which remain viable following the evaluation of
-	 * semantic predicates.
 	 */
-	public final BitSet predictions;
+	public final boolean evalResult;
 
 	/**
 	 * Constructs a new instance of the {@link PredicateEvalInfo} class with the
 	 * specified detailed predicate evaluation information.
 	 *
-	 * @param dfaState The DFA state containing information about the semantic
-	 * predicates to evaluate during the prediction process
 	 * @param decision The decision number
 	 * @param input The input token stream
 	 * @param startIndex The start index for the current prediction
 	 * @param stopIndex The index at which the predicate evaluation was
-	 * triggered. Note that the input stream may be reset to other locations for
+	 * triggered. Note that the input stream may be reset to other positions for
 	 * the actual evaluation of individual predicates.
-	 * @param evalResults The results of evaluating specific semantic contexts.
+	 * @param semctx The semantic context which was evaluated
-	 * The elements of this array correspond to the elements in
+	 * @param evalResult The results of evaluating the semantic context
-	 * {@link DFAState#predicates}, and the value of each element is the result
+	 * @param predictedAlt The alternative number for the decision which is
-	 * of evaluating the semantic context {@link DFAState.PredPrediction#pred}.
+	 * guarded by the semantic context {@code semctx}. See {@link #predictedAlt}
-	 * @param predictions A {@link BitSet} identifying the represented
+	 * for more information.
-	 * alternatives of {@code dfaState} which remain viable following the
+	 * @param fullCtx {@code true} if the semantic context was
-	 * evaluation of semantic predicates
+	 * evaluated during LL prediction; otherwise, {@code false} if the semantic
+	 * context was evaluated during SLL prediction
+	 *
+	 * @see ParserATNSimulator#evalSemanticContext(SemanticContext, ParserRuleContext, int, boolean)
+	 * @see SemanticContext#eval(Recognizer, RuleContext)
 	 */
-	public PredicateEvalInfo(@NotNull DFAState dfaState, int decision,
+	public PredicateEvalInfo(int decision,
 							 @NotNull TokenStream input, int startIndex, int stopIndex,
-							 @NotNull boolean[] evalResults,
+							 @NotNull SemanticContext semctx,
-							 @NotNull BitSet predictions)
+							 boolean evalResult,
+							 int predictedAlt,
+							 boolean fullCtx)
 	{
-		super(decision, dfaState.configs, input, startIndex, stopIndex, dfaState.requiresFullContext);
+		super(decision, new ATNConfigSet(), input, startIndex, stopIndex, fullCtx);
-		this.dfaState = dfaState;
+		this.semctx = semctx;
-		this.evalResults = evalResults;
+		this.evalResult = evalResult;
-		this.predictions = predictions;
+		this.predictedAlt = predictedAlt;
 	}
 }

runtime/Java/src/org/antlr/v4/runtime/atn/ProfilingATNSimulator.java

@@ -162,40 +162,17 @@ public class ProfilingATNSimulator extends ParserATNSimulator {
 	}
 
 	@Override
-	protected BitSet evalSemanticContext(DFAState.PredPrediction[] predPredictions,
+	protected boolean evalSemanticContext(SemanticContext pred, ParserRuleContext parserCallStack, int alt, boolean fullCtx) {
-										 ParserRuleContext outerContext,
+		boolean result = super.evalSemanticContext(pred, parserCallStack, alt, fullCtx);
-										 boolean complete) {
+		if (!(pred instanceof SemanticContext.PrecedencePredicate)) {
-		/* Force complete prediction for the purpose of gathering statistical
+			boolean fullContext = _llStopIndex >= 0;
-		 * results. If the caller requested incomplete evaluation, the result is
+			int stopIndex = fullContext ? _llStopIndex : _sllStopIndex;
-		 * modified before returning to behave as though incomplete evaluation
+			decisions[currentDecision].predicateEvals.add(
-		 * was used.
+				new PredicateEvalInfo(currentDecision, _input, _startIndex, stopIndex, pred, result, alt, fullCtx)
-		 */
+			);
-		BitSet predictions = super.evalSemanticContext(predPredictions, outerContext, true);
-		// must re-evaluate all preds as predictions can't map back to pred eval uniquely
-		int n = predPredictions.length;
-		boolean[] results = new boolean[n];
-		int i = 0;
-		// FOR INTERPRETER, these are all true unless precedence preds!
-		for (DFAState.PredPrediction pair : predPredictions) {
-			if ( pair.pred!=SemanticContext.NONE ) {
-				results[i] = pair.pred.eval(parser, outerContext);
-			}
-			i++;
 		}
 
-		boolean fullContext = _llStopIndex >= 0;
+		return result;
-		int stopIndex = fullContext ? _llStopIndex : _sllStopIndex;
-		decisions[currentDecision].predicateEvals.add(
-			new PredicateEvalInfo(currentState, currentDecision, _input, _startIndex, stopIndex, results, predictions)
-		);
-
-		if (!complete && !predictions.isEmpty()) {
-			int minimum = predictions.nextSetBit(0);
-			predictions = new BitSet();
-			predictions.set(minimum);
-		}
-
-		return predictions;
 	}
 
 	@Override

runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java

@@ -72,13 +72,13 @@ public abstract class SemanticContext {
 	 * prediction, so we passed in the outer context here in case of context
 	 * dependent predicate evaluation.</p>
 	 */
-    public abstract boolean eval(Recognizer<?,?> parser, RuleContext outerContext);
+    public abstract boolean eval(Recognizer<?,?> parser, RuleContext parserCallStack);
 
 	/**
 	 * Evaluate the precedence predicates for the context and reduce the result.
 	 *
 	 * @param parser The parser instance.
-	 * @param outerContext The current parser context object.
+	 * @param parserCallStack
 	 * @return The simplified semantic context after precedence predicates are
 	 * evaluated, which will be one of the following values.
 	 * <ul>
@@ -92,7 +92,7 @@ public abstract class SemanticContext {
 	 * semantic context after precedence predicates are evaluated.</li>
 	 * </ul>
 	 */
-	public SemanticContext evalPrecedence(Recognizer<?,?> parser, RuleContext outerContext) {
+	public SemanticContext evalPrecedence(Recognizer<?,?> parser, RuleContext parserCallStack) {
 		return this;
 	}
 
@@ -114,8 +114,8 @@ public abstract class SemanticContext {
         }
 
         @Override
-        public boolean eval(Recognizer<?,?> parser, RuleContext outerContext) {
+        public boolean eval(Recognizer<?,?> parser, RuleContext parserCallStack) {
-            RuleContext localctx = isCtxDependent ? outerContext : null;
+            RuleContext localctx = isCtxDependent ? parserCallStack : null;
             return parser.sempred(localctx, ruleIndex, predIndex);
         }
 
@@ -157,13 +157,13 @@ public abstract class SemanticContext {
 		}
 
 		@Override
-		public boolean eval(Recognizer<?, ?> parser, RuleContext outerContext) {
+		public boolean eval(Recognizer<?, ?> parser, RuleContext parserCallStack) {
-			return parser.precpred(outerContext, precedence);
+			return parser.precpred(parserCallStack, precedence);
 		}
 
 		@Override
-		public SemanticContext evalPrecedence(Recognizer<?, ?> parser, RuleContext outerContext) {
+		public SemanticContext evalPrecedence(Recognizer<?, ?> parser, RuleContext parserCallStack) {
-			if (parser.precpred(outerContext, precedence)) {
+			if (parser.precpred(parserCallStack, precedence)) {
 				return SemanticContext.NONE;
 			}
 			else {
@@ -269,19 +269,19 @@ public abstract class SemanticContext {
 		 * unordered.</p>
 		 */
 		@Override
-		public boolean eval(Recognizer<?,?> parser, RuleContext outerContext) {
+		public boolean eval(Recognizer<?,?> parser, RuleContext parserCallStack) {
 			for (SemanticContext opnd : opnds) {
-				if ( !opnd.eval(parser, outerContext) ) return false;
+				if ( !opnd.eval(parser, parserCallStack) ) return false;
 			}
 			return true;
         }
 
 		@Override
-		public SemanticContext evalPrecedence(Recognizer<?, ?> parser, RuleContext outerContext) {
+		public SemanticContext evalPrecedence(Recognizer<?, ?> parser, RuleContext parserCallStack) {
 			boolean differs = false;
 			List<SemanticContext> operands = new ArrayList<SemanticContext>();
 			for (SemanticContext context : opnds) {
-				SemanticContext evaluated = context.evalPrecedence(parser, outerContext);
+				SemanticContext evaluated = context.evalPrecedence(parser, parserCallStack);
 				differs |= (evaluated != context);
 				if (evaluated == null) {
 					// The AND context is false if any element is false
@@ -366,19 +366,19 @@ public abstract class SemanticContext {
 		 * unordered.</p>
 		 */
 		@Override
-        public boolean eval(Recognizer<?,?> parser, RuleContext outerContext) {
+        public boolean eval(Recognizer<?,?> parser, RuleContext parserCallStack) {
 			for (SemanticContext opnd : opnds) {
-				if ( opnd.eval(parser, outerContext) ) return true;
+				if ( opnd.eval(parser, parserCallStack) ) return true;
 			}
 			return false;
         }
 
 		@Override
-		public SemanticContext evalPrecedence(Recognizer<?, ?> parser, RuleContext outerContext) {
+		public SemanticContext evalPrecedence(Recognizer<?, ?> parser, RuleContext parserCallStack) {
 			boolean differs = false;
 			List<SemanticContext> operands = new ArrayList<SemanticContext>();
 			for (SemanticContext context : opnds) {
-				SemanticContext evaluated = context.evalPrecedence(parser, outerContext);
+				SemanticContext evaluated = context.evalPrecedence(parser, parserCallStack);
 				differs |= (evaluated != context);
 				if (evaluated == NONE) {
 					// The OR context is true if any element is true