From 266f7276d1fe13221d5a2e137f4800013cc5fe22 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Thu, 9 Jan 2014 18:11:17 -0600 Subject: [PATCH] Initial implementation of a "precedence DFA" (fixes #400) --- .../Java/src/org/antlr/v4/runtime/Parser.java | 14 +++ .../v4/runtime/atn/ParserATNSimulator.java | 78 +++++++++++- .../antlr/v4/runtime/atn/SemanticContext.java | 100 +++++++++++++++ .../src/org/antlr/v4/runtime/dfa/DFA.java | 118 +++++++++++++++++- .../org/antlr/v4/test/TestLeftRecursion.java | 11 +- .../org/antlr/v4/test/TestParseErrors.java | 18 +-- 6 files changed, 310 insertions(+), 29 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/Parser.java b/runtime/Java/src/org/antlr/v4/runtime/Parser.java index e2e73c4ed..89ddf0843 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Parser.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Parser.java @@ -647,6 +647,20 @@ public abstract class Parser extends Recognizer { _ctx = localctx; } + /** + * Get the precedence level for the top-most precedence rule. + * + * @return The precedence level for the top-most precedence rule, or -1 if + * the parser context is not nested within a precedence rule. + */ + public final int getPrecedence() { + if (_precedenceStack.isEmpty()) { + return -1; + } + + return _precedenceStack.peek(); + } + /** * @deprecated Use * {@link #enterRecursionRule(ParserRuleContext, int, int, int)} instead. diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 1ef4b4143..a1c4cdb6a 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -325,23 +325,51 @@ public class ParserATNSimulator extends ATNSimulator { // Now we are certain to have a specific decision's DFA // But, do we still need an initial state? try { - if ( dfa.s0==null ) { + DFAState s0; + if (dfa.isPrecedenceDfa()) { + s0 = dfa.getPrecedenceStartState(parser.getPrecedence()); + } + else { + s0 = dfa.s0; + } + + if (s0 == null) { if ( outerContext ==null ) outerContext = ParserRuleContext.EMPTY; if ( debug || debug_list_atn_decisions ) { System.out.println("predictATN decision "+ dfa.decision+ " exec LA(1)=="+ getLookaheadName(input) + ", outerContext="+ outerContext.toString(parser)); } + + if (!dfa.isPrecedenceDfa() && dfa.atnStartState instanceof StarLoopEntryState) { + if (atn.ruleToStartState[dfa.atnStartState.ruleIndex].isPrecedenceRule) { + ATNState maybeLoopEndState = dfa.atnStartState.transition(dfa.atnStartState.getNumberOfTransitions() - 1).target; + if (maybeLoopEndState instanceof LoopEndState) { + if (maybeLoopEndState.epsilonOnlyTransitions && maybeLoopEndState.transition(0).target instanceof RuleStopState) { + dfa.setPrecedenceDfa(true); + } + } + } + } + boolean fullCtx = false; ATNConfigSet s0_closure = computeStartState(dfa.atnStartState, ParserRuleContext.EMPTY, fullCtx); - dfa.s0 = addDFAState(dfa, new DFAState(s0_closure)); + + if (dfa.isPrecedenceDfa()) { + s0_closure = applyPrecedenceFilter(s0_closure); + s0 = addDFAState(dfa, new DFAState(s0_closure)); + dfa.setPrecedenceStartState(parser.getPrecedence(), s0); + } + else { + s0 = addDFAState(dfa, new DFAState(s0_closure)); + dfa.s0 = s0; + } } - // We can start with an existing DFA - int alt = execATN(dfa, dfa.s0, input, index, outerContext); + int alt = execATN(dfa, s0, input, index, outerContext); if ( debug ) System.out.println("DFA after predictATN: "+ dfa.toString(parser.getTokenNames())); return alt; } @@ -907,6 +935,48 @@ public class ParserATNSimulator extends ATNSimulator { return configs; } + @NotNull + protected ATNConfigSet applyPrecedenceFilter(@NotNull ATNConfigSet configs) { + Set statesFromAlt1 = new HashSet(); + ATNConfigSet configSet = new ATNConfigSet(configs.fullCtx); + for (ATNConfig config : configs) { + // handle alt 1 first + if (config.alt != 1) { + continue; + } + + SemanticContext updatedContext = config.semanticContext.evalPrecedence(parser, _outerContext); + if (updatedContext == null) { + // the configuration was eliminated + continue; + } + + statesFromAlt1.add(config.state.stateNumber); + if (updatedContext != config.semanticContext) { + configSet.add(new ATNConfig(config, updatedContext), mergeCache); + } + else { + configSet.add(config, mergeCache); + } + } + + for (ATNConfig config : configs) { + if (config.alt == 1) { + // already handled + continue; + } + + if (statesFromAlt1.contains(config.state.stateNumber)) { + // eliminated + continue; + } + + configSet.add(config, mergeCache); + } + + return configSet; + } + @Nullable protected ATNState getReachableTarget(@NotNull Transition trans, int ttype) { if (trans.matches(ttype, 0, atn.maxTokenType)) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java index 66ca13256..8d841f88a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java @@ -72,6 +72,28 @@ public abstract class SemanticContext { */ public abstract boolean eval(Recognizer parser, RuleContext outerContext); + /** + * Evaluate the precedence predicates for the context and reduce the result. + * + * @param parser The parser instance. + * @param outerContext The current parser context object. + * @return The simplified semantic context after precedence predicates are + * evaluated, which will be one of the following values. + *
    + *
  • {@link #NONE}: if the predicate simplifies to {@code true} after + * precedence predicates are evaluated.
    • + *
  • {@code null}: if the predicate simplifies to {@code false} after + * precedence predicates are evaluated.
    • + *
  • {@code this}: if the semantic context is not changed as a result of + * precedence predicate evaluation.
    • + *
  • A non-{@code null} {@link SemanticContext}: the new simplified + * semantic context after precedence predicates are evaluated.
    • + *
+ */ + public SemanticContext evalPrecedence(Recognizer parser, RuleContext outerContext) { + return this; + } + public static class Predicate extends SemanticContext { public final int ruleIndex; public final int predIndex; @@ -137,6 +159,16 @@ public abstract class SemanticContext { return parser.precpred(outerContext, precedence); } + @Override + public SemanticContext evalPrecedence(Recognizer parser, RuleContext outerContext) { + if (parser.precpred(outerContext, precedence)) { + return SemanticContext.NONE; + } + else { + return null; + } + } + @Override public int compareTo(PrecedencePredicate o) { return precedence - o.precedence; @@ -210,6 +242,40 @@ public abstract class SemanticContext { return true; } + @Override + public SemanticContext evalPrecedence(Recognizer parser, RuleContext outerContext) { + boolean differs = false; + List operands = new ArrayList(); + for (SemanticContext context : opnds) { + SemanticContext evaluated = context.evalPrecedence(parser, outerContext); + differs |= (evaluated != context); + if (evaluated == null) { + // The AND context is false if any element is false + return null; + } + else if (evaluated != NONE) { + // Reduce the result by skipping true elements + operands.add(evaluated); + } + } + + if (!differs) { + return this; + } + + if (operands.isEmpty()) { + // all elements were true, so the AND context is true + return NONE; + } + + SemanticContext result = operands.get(0); + for (int i = 1; i < operands.size(); i++) { + result = SemanticContext.and(result, operands.get(i)); + } + + return result; + } + @Override public String toString() { return Utils.join(Arrays.asList(opnds).iterator(), "&&"); @@ -257,6 +323,40 @@ public abstract class SemanticContext { return false; } + @Override + public SemanticContext evalPrecedence(Recognizer parser, RuleContext outerContext) { + boolean differs = false; + List operands = new ArrayList(); + for (SemanticContext context : opnds) { + SemanticContext evaluated = context.evalPrecedence(parser, outerContext); + differs |= (evaluated != context); + if (evaluated == NONE) { + // The OR context is true if any element is true + return NONE; + } + else if (evaluated != null) { + // Reduce the result by skipping false elements + operands.add(evaluated); + } + } + + if (!differs) { + return this; + } + + if (operands.isEmpty()) { + // all elements were false, so the OR context is false + return null; + } + + SemanticContext result = operands.get(0); + for (int i = 1; i < operands.size(); i++) { + result = SemanticContext.or(result, operands.get(i)); + } + + return result; + } + @Override public String toString() { return Utils.join(Arrays.asList(opnds).iterator(), "||"); diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java index 091a07161..791cf9fbe 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java @@ -29,11 +29,14 @@ */ package org.antlr.v4.runtime.dfa; +import org.antlr.v4.runtime.atn.ATNConfigSet; import org.antlr.v4.runtime.atn.DecisionState; import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.Nullable; +import org.antlr.v4.runtime.Parser; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -48,7 +51,7 @@ public class DFA { @NotNull public final Map states = new HashMap(); @Nullable - public DFAState s0; + public volatile DFAState s0; public final int decision; @@ -56,6 +59,13 @@ public class DFA { @NotNull public final DecisionState atnStartState; + /** + * {@code true} if this DFA is for a precedence decision; otherwise, + * {@code false}. This is the backing field for {@link #isPrecedenceDfa}, + * {@link #setPrecedenceDfa}, {@link #hasPrecedenceEdge}. + */ + private volatile boolean precedenceDfa; + public DFA(@NotNull DecisionState atnStartState) { this(atnStartState, 0); } @@ -65,6 +75,112 @@ public class DFA { this.decision = decision; } + /** + * Gets whether this DFA is a precedence DFA. Precedence DFAs use a special + * start state {@link #s0} which is not stored in {@link #states}. The + * {@link DFAState#edges} array for this start state contains outgoing edges + * supplying individual start states corresponding to specific precedence + * values. + * + * @return {@code true} if this is a precedence DFA; otherwise, + * {@code false}. + * @see Parser#getPrecedence() + */ + public final boolean isPrecedenceDfa() { + return precedenceDfa; + } + + /** + * Get the start state for a specific precedence value. + * + * @param precedence The current precedence. + * @return The start state corresponding to the specified precedence, or + * {@code null} if no start state exists for the specified precedence. + * + * @throws IllegalStateException if this is not a precedence DFA. + * @see #isPrecedenceDfa() + */ + @SuppressWarnings("null") + public final DFAState getPrecedenceStartState(int precedence) { + if (!isPrecedenceDfa()) { + throw new IllegalStateException("Only precedence DFAs may contain a precedence start state."); + } + + // s0.edges is never null for a precedence DFA + if (precedence < 0 || precedence >= s0.edges.length) { + return null; + } + + return s0.edges[precedence]; + } + + /** + * Set the start state for a specific precedence value. + * + * @param precedence The current precedence. + * @param startState The start state corresponding to the specified + * precedence. + * + * @throws IllegalStateException if this is not a precedence DFA. + * @see #isPrecedenceDfa() + */ + @SuppressWarnings({"SynchronizeOnNonFinalField", "null"}) + public final void setPrecedenceStartState(int precedence, DFAState startState) { + if (!isPrecedenceDfa()) { + throw new IllegalStateException("Only precedence DFAs may contain a precedence start state."); + } + + if (precedence < 0) { + return; + } + + // synchronization on s0 here is ok. when the DFA is turned into a + // precedence DFA, s0 will be initialized once and not updated again + synchronized (s0) { + // s0.edges is never null for a precedence DFA + if (precedence >= s0.edges.length) { + s0.edges = Arrays.copyOf(s0.edges, precedence + 1); + } + + s0.edges[precedence] = startState; + } + } + + /** + * Sets whether this is a precedence DFA. If the specified value differs + * from the current DFA configuration, the following actions are taken; + * otherwise no changes are made to the current DFA. + * + *
    + *
  • The {@link #states} map is cleared
    • + *
  • If {@code precedenceDfa} is {@code false}, the initial state + * {@link #s0} is set to {@code null}; otherwise, it is initialized to a new + * {@link DFAState} with an empty outgoing {@link DFAState#edges} array to + * store the start states for individual precedence values.
    • + *
  • The {@link #precedenceDfa} field is updated
    • + *
+ * + * @param precedenceDfa {@code true} if this is a precedence DFA; otherwise, + * {@code false} + */ + public final synchronized void setPrecedenceDfa(boolean precedenceDfa) { + if (this.precedenceDfa != precedenceDfa) { + this.states.clear(); + if (precedenceDfa) { + DFAState precedenceState = new DFAState(new ATNConfigSet()); + precedenceState.edges = new DFAState[0]; + precedenceState.isAcceptState = false; + precedenceState.requiresFullContext = false; + this.s0 = precedenceState; + } + else { + this.s0 = null; + } + + this.precedenceDfa = precedenceDfa; + } + } + /** * Return a list of all states in this DFA, ordered by state number. */ diff --git a/tool/test/org/antlr/v4/test/TestLeftRecursion.java b/tool/test/org/antlr/v4/test/TestLeftRecursion.java index 798864420..7dd6b2432 100644 --- a/tool/test/org/antlr/v4/test/TestLeftRecursion.java +++ b/tool/test/org/antlr/v4/test/TestLeftRecursion.java @@ -394,17 +394,10 @@ public class TestLeftRecursion extends BaseTest { assertNull(stderrDuringParse); result = execParser("Expr.g4", grammar, "ExprParser", "ExprLexer", "prog", "a+b*2\n", true); - assertEquals("line 1:1 reportAttemptingFullContext d=3 (expr), input='+'\n" + - "line 1:1 reportContextSensitivity d=3 (expr), input='+'\n" + - "line 1:3 reportAttemptingFullContext d=3 (expr), input='*'\n", - stderrDuringParse); + assertNull(stderrDuringParse); result = execParser("Expr.g4", grammar, "ExprParser", "ExprLexer", "prog", "(1+2)*3\n", true); - assertEquals("line 1:2 reportAttemptingFullContext d=3 (expr), input='+'\n" + - "line 1:2 reportContextSensitivity d=3 (expr), input='+'\n" + - "line 1:5 reportAttemptingFullContext d=3 (expr), input='*'\n" + - "line 1:5 reportContextSensitivity d=3 (expr), input='*'\n", - stderrDuringParse); + assertNull(stderrDuringParse); } @Test public void testCheckForNonLeftRecursiveRule() throws Exception { diff --git a/tool/test/org/antlr/v4/test/TestParseErrors.java b/tool/test/org/antlr/v4/test/TestParseErrors.java index c4af68c9c..22625358b 100644 --- a/tool/test/org/antlr/v4/test/TestParseErrors.java +++ b/tool/test/org/antlr/v4/test/TestParseErrors.java @@ -320,27 +320,15 @@ public class TestParseErrors extends BaseTest { result = execParser("T.g4", grammar, "TParser", "TLexer", "start", "xx", true); assertEquals("", result); - assertEquals( - "line 1:1 reportAttemptingFullContext d=0 (expr), input='x'\n" + - "line 1:1 reportContextSensitivity d=0 (expr), input='x'\n", - this.stderrDuringParse); + assertNull(this.stderrDuringParse); result = execParser("T.g4", grammar, "TParser", "TLexer", "start", "xxx", true); assertEquals("", result); - assertEquals( - "line 1:1 reportAttemptingFullContext d=0 (expr), input='x'\n" + - "line 1:1 reportContextSensitivity d=0 (expr), input='x'\n" + - "line 1:2 reportAttemptingFullContext d=0 (expr), input='x'\n", - this.stderrDuringParse); + assertNull(this.stderrDuringParse); result = execParser("T.g4", grammar, "TParser", "TLexer", "start", "xxxx", true); assertEquals("", result); - assertEquals( - "line 1:1 reportAttemptingFullContext d=0 (expr), input='x'\n" + - "line 1:1 reportContextSensitivity d=0 (expr), input='x'\n" + - "line 1:2 reportAttemptingFullContext d=0 (expr), input='x'\n" + - "line 1:3 reportAttemptingFullContext d=0 (expr), input='x'\n", - this.stderrDuringParse); + assertNull(this.stderrDuringParse); } /**