Initial implementation of a "precedence DFA" (fixes #400)
This commit is contained in:
parent
f51aaae2f5
commit
266f7276d1
|
@ -647,6 +647,20 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
|
|||
_ctx = localctx;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the precedence level for the top-most precedence rule.
|
||||
*
|
||||
* @return The precedence level for the top-most precedence rule, or -1 if
|
||||
* the parser context is not nested within a precedence rule.
|
||||
*/
|
||||
public final int getPrecedence() {
|
||||
if (_precedenceStack.isEmpty()) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
return _precedenceStack.peek();
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Use
|
||||
* {@link #enterRecursionRule(ParserRuleContext, int, int, int)} instead.
|
||||
|
|
|
@ -325,23 +325,51 @@ public class ParserATNSimulator extends ATNSimulator {
|
|||
// Now we are certain to have a specific decision's DFA
|
||||
// But, do we still need an initial state?
|
||||
try {
|
||||
if ( dfa.s0==null ) {
|
||||
DFAState s0;
|
||||
if (dfa.isPrecedenceDfa()) {
|
||||
s0 = dfa.getPrecedenceStartState(parser.getPrecedence());
|
||||
}
|
||||
else {
|
||||
s0 = dfa.s0;
|
||||
}
|
||||
|
||||
if (s0 == null) {
|
||||
if ( outerContext ==null ) outerContext = ParserRuleContext.EMPTY;
|
||||
if ( debug || debug_list_atn_decisions ) {
|
||||
System.out.println("predictATN decision "+ dfa.decision+
|
||||
" exec LA(1)=="+ getLookaheadName(input) +
|
||||
", outerContext="+ outerContext.toString(parser));
|
||||
}
|
||||
|
||||
if (!dfa.isPrecedenceDfa() && dfa.atnStartState instanceof StarLoopEntryState) {
|
||||
if (atn.ruleToStartState[dfa.atnStartState.ruleIndex].isPrecedenceRule) {
|
||||
ATNState maybeLoopEndState = dfa.atnStartState.transition(dfa.atnStartState.getNumberOfTransitions() - 1).target;
|
||||
if (maybeLoopEndState instanceof LoopEndState) {
|
||||
if (maybeLoopEndState.epsilonOnlyTransitions && maybeLoopEndState.transition(0).target instanceof RuleStopState) {
|
||||
dfa.setPrecedenceDfa(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
boolean fullCtx = false;
|
||||
ATNConfigSet s0_closure =
|
||||
computeStartState(dfa.atnStartState,
|
||||
ParserRuleContext.EMPTY,
|
||||
fullCtx);
|
||||
dfa.s0 = addDFAState(dfa, new DFAState(s0_closure));
|
||||
|
||||
if (dfa.isPrecedenceDfa()) {
|
||||
s0_closure = applyPrecedenceFilter(s0_closure);
|
||||
s0 = addDFAState(dfa, new DFAState(s0_closure));
|
||||
dfa.setPrecedenceStartState(parser.getPrecedence(), s0);
|
||||
}
|
||||
else {
|
||||
s0 = addDFAState(dfa, new DFAState(s0_closure));
|
||||
dfa.s0 = s0;
|
||||
}
|
||||
}
|
||||
|
||||
// We can start with an existing DFA
|
||||
int alt = execATN(dfa, dfa.s0, input, index, outerContext);
|
||||
int alt = execATN(dfa, s0, input, index, outerContext);
|
||||
if ( debug ) System.out.println("DFA after predictATN: "+ dfa.toString(parser.getTokenNames()));
|
||||
return alt;
|
||||
}
|
||||
|
@ -907,6 +935,48 @@ public class ParserATNSimulator extends ATNSimulator {
|
|||
return configs;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
protected ATNConfigSet applyPrecedenceFilter(@NotNull ATNConfigSet configs) {
|
||||
Set<Integer> statesFromAlt1 = new HashSet<Integer>();
|
||||
ATNConfigSet configSet = new ATNConfigSet(configs.fullCtx);
|
||||
for (ATNConfig config : configs) {
|
||||
// handle alt 1 first
|
||||
if (config.alt != 1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
SemanticContext updatedContext = config.semanticContext.evalPrecedence(parser, _outerContext);
|
||||
if (updatedContext == null) {
|
||||
// the configuration was eliminated
|
||||
continue;
|
||||
}
|
||||
|
||||
statesFromAlt1.add(config.state.stateNumber);
|
||||
if (updatedContext != config.semanticContext) {
|
||||
configSet.add(new ATNConfig(config, updatedContext), mergeCache);
|
||||
}
|
||||
else {
|
||||
configSet.add(config, mergeCache);
|
||||
}
|
||||
}
|
||||
|
||||
for (ATNConfig config : configs) {
|
||||
if (config.alt == 1) {
|
||||
// already handled
|
||||
continue;
|
||||
}
|
||||
|
||||
if (statesFromAlt1.contains(config.state.stateNumber)) {
|
||||
// eliminated
|
||||
continue;
|
||||
}
|
||||
|
||||
configSet.add(config, mergeCache);
|
||||
}
|
||||
|
||||
return configSet;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
protected ATNState getReachableTarget(@NotNull Transition trans, int ttype) {
|
||||
if (trans.matches(ttype, 0, atn.maxTokenType)) {
|
||||
|
|
|
@ -72,6 +72,28 @@ public abstract class SemanticContext {
|
|||
*/
|
||||
public abstract boolean eval(Recognizer<?,?> parser, RuleContext outerContext);
|
||||
|
||||
/**
|
||||
* Evaluate the precedence predicates for the context and reduce the result.
|
||||
*
|
||||
* @param parser The parser instance.
|
||||
* @param outerContext The current parser context object.
|
||||
* @return The simplified semantic context after precedence predicates are
|
||||
* evaluated, which will be one of the following values.
|
||||
* <ul>
|
||||
* <li>{@link #NONE}: if the predicate simplifies to {@code true} after
|
||||
* precedence predicates are evaluated.</li>
|
||||
* <li>{@code null}: if the predicate simplifies to {@code false} after
|
||||
* precedence predicates are evaluated.</li>
|
||||
* <li>{@code this}: if the semantic context is not changed as a result of
|
||||
* precedence predicate evaluation.</li>
|
||||
* <li>A non-{@code null} {@link SemanticContext}: the new simplified
|
||||
* semantic context after precedence predicates are evaluated.</li>
|
||||
* </ul>
|
||||
*/
|
||||
public SemanticContext evalPrecedence(Recognizer<?,?> parser, RuleContext outerContext) {
|
||||
return this;
|
||||
}
|
||||
|
||||
public static class Predicate extends SemanticContext {
|
||||
public final int ruleIndex;
|
||||
public final int predIndex;
|
||||
|
@ -137,6 +159,16 @@ public abstract class SemanticContext {
|
|||
return parser.precpred(outerContext, precedence);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SemanticContext evalPrecedence(Recognizer<?, ?> parser, RuleContext outerContext) {
|
||||
if (parser.precpred(outerContext, precedence)) {
|
||||
return SemanticContext.NONE;
|
||||
}
|
||||
else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(PrecedencePredicate o) {
|
||||
return precedence - o.precedence;
|
||||
|
@ -210,6 +242,40 @@ public abstract class SemanticContext {
|
|||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SemanticContext evalPrecedence(Recognizer<?, ?> parser, RuleContext outerContext) {
|
||||
boolean differs = false;
|
||||
List<SemanticContext> operands = new ArrayList<SemanticContext>();
|
||||
for (SemanticContext context : opnds) {
|
||||
SemanticContext evaluated = context.evalPrecedence(parser, outerContext);
|
||||
differs |= (evaluated != context);
|
||||
if (evaluated == null) {
|
||||
// The AND context is false if any element is false
|
||||
return null;
|
||||
}
|
||||
else if (evaluated != NONE) {
|
||||
// Reduce the result by skipping true elements
|
||||
operands.add(evaluated);
|
||||
}
|
||||
}
|
||||
|
||||
if (!differs) {
|
||||
return this;
|
||||
}
|
||||
|
||||
if (operands.isEmpty()) {
|
||||
// all elements were true, so the AND context is true
|
||||
return NONE;
|
||||
}
|
||||
|
||||
SemanticContext result = operands.get(0);
|
||||
for (int i = 1; i < operands.size(); i++) {
|
||||
result = SemanticContext.and(result, operands.get(i));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return Utils.join(Arrays.asList(opnds).iterator(), "&&");
|
||||
|
@ -257,6 +323,40 @@ public abstract class SemanticContext {
|
|||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SemanticContext evalPrecedence(Recognizer<?, ?> parser, RuleContext outerContext) {
|
||||
boolean differs = false;
|
||||
List<SemanticContext> operands = new ArrayList<SemanticContext>();
|
||||
for (SemanticContext context : opnds) {
|
||||
SemanticContext evaluated = context.evalPrecedence(parser, outerContext);
|
||||
differs |= (evaluated != context);
|
||||
if (evaluated == NONE) {
|
||||
// The OR context is true if any element is true
|
||||
return NONE;
|
||||
}
|
||||
else if (evaluated != null) {
|
||||
// Reduce the result by skipping false elements
|
||||
operands.add(evaluated);
|
||||
}
|
||||
}
|
||||
|
||||
if (!differs) {
|
||||
return this;
|
||||
}
|
||||
|
||||
if (operands.isEmpty()) {
|
||||
// all elements were false, so the OR context is false
|
||||
return null;
|
||||
}
|
||||
|
||||
SemanticContext result = operands.get(0);
|
||||
for (int i = 1; i < operands.size(); i++) {
|
||||
result = SemanticContext.or(result, operands.get(i));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return Utils.join(Arrays.asList(opnds).iterator(), "||");
|
||||
|
|
|
@ -29,11 +29,14 @@
|
|||
*/
|
||||
package org.antlr.v4.runtime.dfa;
|
||||
|
||||
import org.antlr.v4.runtime.atn.ATNConfigSet;
|
||||
import org.antlr.v4.runtime.atn.DecisionState;
|
||||
import org.antlr.v4.runtime.misc.NotNull;
|
||||
import org.antlr.v4.runtime.misc.Nullable;
|
||||
import org.antlr.v4.runtime.Parser;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
|
@ -48,7 +51,7 @@ public class DFA {
|
|||
@NotNull
|
||||
public final Map<DFAState, DFAState> states = new HashMap<DFAState, DFAState>();
|
||||
@Nullable
|
||||
public DFAState s0;
|
||||
public volatile DFAState s0;
|
||||
|
||||
public final int decision;
|
||||
|
||||
|
@ -56,6 +59,13 @@ public class DFA {
|
|||
@NotNull
|
||||
public final DecisionState atnStartState;
|
||||
|
||||
/**
|
||||
* {@code true} if this DFA is for a precedence decision; otherwise,
|
||||
* {@code false}. This is the backing field for {@link #isPrecedenceDfa},
|
||||
* {@link #setPrecedenceDfa}, {@link #hasPrecedenceEdge}.
|
||||
*/
|
||||
private volatile boolean precedenceDfa;
|
||||
|
||||
public DFA(@NotNull DecisionState atnStartState) {
|
||||
this(atnStartState, 0);
|
||||
}
|
||||
|
@ -65,6 +75,112 @@ public class DFA {
|
|||
this.decision = decision;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets whether this DFA is a precedence DFA. Precedence DFAs use a special
|
||||
* start state {@link #s0} which is not stored in {@link #states}. The
|
||||
* {@link DFAState#edges} array for this start state contains outgoing edges
|
||||
* supplying individual start states corresponding to specific precedence
|
||||
* values.
|
||||
*
|
||||
* @return {@code true} if this is a precedence DFA; otherwise,
|
||||
* {@code false}.
|
||||
* @see Parser#getPrecedence()
|
||||
*/
|
||||
public final boolean isPrecedenceDfa() {
|
||||
return precedenceDfa;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the start state for a specific precedence value.
|
||||
*
|
||||
* @param precedence The current precedence.
|
||||
* @return The start state corresponding to the specified precedence, or
|
||||
* {@code null} if no start state exists for the specified precedence.
|
||||
*
|
||||
* @throws IllegalStateException if this is not a precedence DFA.
|
||||
* @see #isPrecedenceDfa()
|
||||
*/
|
||||
@SuppressWarnings("null")
|
||||
public final DFAState getPrecedenceStartState(int precedence) {
|
||||
if (!isPrecedenceDfa()) {
|
||||
throw new IllegalStateException("Only precedence DFAs may contain a precedence start state.");
|
||||
}
|
||||
|
||||
// s0.edges is never null for a precedence DFA
|
||||
if (precedence < 0 || precedence >= s0.edges.length) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return s0.edges[precedence];
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the start state for a specific precedence value.
|
||||
*
|
||||
* @param precedence The current precedence.
|
||||
* @param startState The start state corresponding to the specified
|
||||
* precedence.
|
||||
*
|
||||
* @throws IllegalStateException if this is not a precedence DFA.
|
||||
* @see #isPrecedenceDfa()
|
||||
*/
|
||||
@SuppressWarnings({"SynchronizeOnNonFinalField", "null"})
|
||||
public final void setPrecedenceStartState(int precedence, DFAState startState) {
|
||||
if (!isPrecedenceDfa()) {
|
||||
throw new IllegalStateException("Only precedence DFAs may contain a precedence start state.");
|
||||
}
|
||||
|
||||
if (precedence < 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// synchronization on s0 here is ok. when the DFA is turned into a
|
||||
// precedence DFA, s0 will be initialized once and not updated again
|
||||
synchronized (s0) {
|
||||
// s0.edges is never null for a precedence DFA
|
||||
if (precedence >= s0.edges.length) {
|
||||
s0.edges = Arrays.copyOf(s0.edges, precedence + 1);
|
||||
}
|
||||
|
||||
s0.edges[precedence] = startState;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets whether this is a precedence DFA. If the specified value differs
|
||||
* from the current DFA configuration, the following actions are taken;
|
||||
* otherwise no changes are made to the current DFA.
|
||||
*
|
||||
* <ul>
|
||||
* <li>The {@link #states} map is cleared</li>
|
||||
* <li>If {@code precedenceDfa} is {@code false}, the initial state
|
||||
* {@link #s0} is set to {@code null}; otherwise, it is initialized to a new
|
||||
* {@link DFAState} with an empty outgoing {@link DFAState#edges} array to
|
||||
* store the start states for individual precedence values.</li>
|
||||
* <li>The {@link #precedenceDfa} field is updated</li>
|
||||
* </ul>
|
||||
*
|
||||
* @param precedenceDfa {@code true} if this is a precedence DFA; otherwise,
|
||||
* {@code false}
|
||||
*/
|
||||
public final synchronized void setPrecedenceDfa(boolean precedenceDfa) {
|
||||
if (this.precedenceDfa != precedenceDfa) {
|
||||
this.states.clear();
|
||||
if (precedenceDfa) {
|
||||
DFAState precedenceState = new DFAState(new ATNConfigSet());
|
||||
precedenceState.edges = new DFAState[0];
|
||||
precedenceState.isAcceptState = false;
|
||||
precedenceState.requiresFullContext = false;
|
||||
this.s0 = precedenceState;
|
||||
}
|
||||
else {
|
||||
this.s0 = null;
|
||||
}
|
||||
|
||||
this.precedenceDfa = precedenceDfa;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a list of all states in this DFA, ordered by state number.
|
||||
*/
|
||||
|
|
|
@ -394,17 +394,10 @@ public class TestLeftRecursion extends BaseTest {
|
|||
assertNull(stderrDuringParse);
|
||||
|
||||
result = execParser("Expr.g4", grammar, "ExprParser", "ExprLexer", "prog", "a+b*2\n", true);
|
||||
assertEquals("line 1:1 reportAttemptingFullContext d=3 (expr), input='+'\n" +
|
||||
"line 1:1 reportContextSensitivity d=3 (expr), input='+'\n" +
|
||||
"line 1:3 reportAttemptingFullContext d=3 (expr), input='*'\n",
|
||||
stderrDuringParse);
|
||||
assertNull(stderrDuringParse);
|
||||
|
||||
result = execParser("Expr.g4", grammar, "ExprParser", "ExprLexer", "prog", "(1+2)*3\n", true);
|
||||
assertEquals("line 1:2 reportAttemptingFullContext d=3 (expr), input='+'\n" +
|
||||
"line 1:2 reportContextSensitivity d=3 (expr), input='+'\n" +
|
||||
"line 1:5 reportAttemptingFullContext d=3 (expr), input='*'\n" +
|
||||
"line 1:5 reportContextSensitivity d=3 (expr), input='*'\n",
|
||||
stderrDuringParse);
|
||||
assertNull(stderrDuringParse);
|
||||
}
|
||||
|
||||
@Test public void testCheckForNonLeftRecursiveRule() throws Exception {
|
||||
|
|
|
@ -320,27 +320,15 @@ public class TestParseErrors extends BaseTest {
|
|||
|
||||
result = execParser("T.g4", grammar, "TParser", "TLexer", "start", "xx", true);
|
||||
assertEquals("", result);
|
||||
assertEquals(
|
||||
"line 1:1 reportAttemptingFullContext d=0 (expr), input='x'\n" +
|
||||
"line 1:1 reportContextSensitivity d=0 (expr), input='x'\n",
|
||||
this.stderrDuringParse);
|
||||
assertNull(this.stderrDuringParse);
|
||||
|
||||
result = execParser("T.g4", grammar, "TParser", "TLexer", "start", "xxx", true);
|
||||
assertEquals("", result);
|
||||
assertEquals(
|
||||
"line 1:1 reportAttemptingFullContext d=0 (expr), input='x'\n" +
|
||||
"line 1:1 reportContextSensitivity d=0 (expr), input='x'\n" +
|
||||
"line 1:2 reportAttemptingFullContext d=0 (expr), input='x'\n",
|
||||
this.stderrDuringParse);
|
||||
assertNull(this.stderrDuringParse);
|
||||
|
||||
result = execParser("T.g4", grammar, "TParser", "TLexer", "start", "xxxx", true);
|
||||
assertEquals("", result);
|
||||
assertEquals(
|
||||
"line 1:1 reportAttemptingFullContext d=0 (expr), input='x'\n" +
|
||||
"line 1:1 reportContextSensitivity d=0 (expr), input='x'\n" +
|
||||
"line 1:2 reportAttemptingFullContext d=0 (expr), input='x'\n" +
|
||||
"line 1:3 reportAttemptingFullContext d=0 (expr), input='x'\n",
|
||||
this.stderrDuringParse);
|
||||
assertNull(this.stderrDuringParse);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
Loading…
Reference in New Issue