forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

interim pred overhaul 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9479]
This commit is contained in:
parrt 2011-11-28 09:37:29 -08:00
parent eb6443b32b
commit e288ff3039
7 changed files with 327 additions and 583 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java
View File

@ -29,9 +29,10 @@
package org.antlr.v4.runtime.atn; package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable; import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.*;
/** An ATN state, predicted alt, and syntactic/semantic context. /** An ATN state, predicted alt, and syntactic/semantic context.
* The syntactic context is a pointer into the rule invocation * The syntactic context is a pointer into the rule invocation
@ -61,16 +62,7 @@ public class ATNConfig {
* otherwise predicates would not get executed again (DFAs don't * otherwise predicates would not get executed again (DFAs don't
* have predicated edges in v4). * have predicated edges in v4).
*/ */
public boolean traversedPredicate; public boolean traversedPredicate; // TODO: don't need
/**
* Indicates that we have reached this ATN configuration after
* traversing a non-forced action transition. We do not execute
* predicates after such actions because the predicates could be
* functions of the side effects. Force actions must be either side
* effect free or automatically undone as the parse continues.
*/
public boolean traversedAction;
/** /**
* We cannot execute predicates dependent upon local context unless * We cannot execute predicates dependent upon local context unless
@ -85,6 +77,18 @@ public class ATNConfig {
*/ */
public int reachesIntoOuterContext; public int reachesIntoOuterContext;
@NotNull
public SemanticContext semanticContext = SemanticContext.NONE;
/** This bit is used to indicate a semantic predicate will be
* used to resolve the conflict. Essentially, this is used
* as an "ignore" bit so that upon a set of conflicting configurations,
* such as (s|2|p) and (s|3|q), I can set (s|3) to resolved=true (and any
* other configuration associated with alt 3) to make it look like that set
* uniquely predicts an alt.
*/
protected boolean resolveWithPredicate;
public ATNConfig(@NotNull ATNState state, public ATNConfig(@NotNull ATNState state,
int alt, int alt,
@Nullable RuleContext context) @Nullable RuleContext context)
@ -94,26 +98,36 @@ public class ATNConfig {
this.context = context; this.context = context;
} }
public ATNConfig(@NotNull ATNConfig c) { // public ATNConfig(@NotNull ATNConfig c) {
this(c, c.state, c.context); // this(c, c.state, c.context, c.semanticContext);
} // }
public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state) { public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state) {
this(c, state, c.context); this(c, state, c.context, c.semanticContext);
}
public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, SemanticContext semanticContext) {
this(c, state, c.context, semanticContext);
} }
public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, @Nullable RuleContext context) { public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, @Nullable RuleContext context) {
this(c, state, context, c.semanticContext);
}
public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, @Nullable RuleContext context,
SemanticContext semanticContext)
{
this.state = state; this.state = state;
this.alt = c.alt; this.alt = c.alt;
this.context = context; this.context = context;
this.traversedPredicate = c.traversedPredicate; this.traversedPredicate = c.traversedPredicate;
this.traversedAction = c.traversedAction;
this.reachesIntoOuterContext = c.reachesIntoOuterContext; this.reachesIntoOuterContext = c.reachesIntoOuterContext;
this.semanticContext = semanticContext;
} }
public ATNConfig(@NotNull ATNConfig c, @Nullable RuleContext context) { // public ATNConfig(@NotNull ATNConfig c, @Nullable RuleContext context) {
this(c, c.state, context); // this(c, c.state, context);
} // }
/** An ATN configuration is equal to another if both have /** An ATN configuration is equal to another if both have
* the same state, they predict the same alternative, and * the same state, they predict the same alternative, and
@ -160,6 +174,10 @@ public class ATNConfig {
buf.append("|"); buf.append("|");
buf.append(context); buf.append(context);
} }
if ( semanticContext!=null ) {
buf.append("|");
buf.append(semanticContext);
}
if ( reachesIntoOuterContext>0 ) { if ( reachesIntoOuterContext>0 ) {
buf.append("|up=").append(reachesIntoOuterContext); buf.append("|up=").append(reachesIntoOuterContext);
} }

186
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -43,8 +43,8 @@ import org.stringtemplate.v4.misc.MultiMap;
import java.util.*; import java.util.*;
public class ParserATNSimulator<Symbol> extends ATNSimulator { public class ParserATNSimulator<Symbol> extends ATNSimulator {
public static boolean debug = false; public static boolean debug = true;
public static boolean dfa_debug = false; public static boolean dfa_debug = true;
public static int ATN_failover = 0; public static int ATN_failover = 0;
public static int predict_calls = 0; public static int predict_calls = 0;
@ -161,7 +161,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
return execATN(input, dfa, input.index(), s0_closure, false); return execATN(input, dfa, input.index(), s0_closure, false);
} }
public int execDFA(@NotNull SymbolStream<Symbol> input, @NotNull DFA dfa, @NotNull DFAState s0, @Nullable RuleContext outerContext) { public int execDFA(@NotNull SymbolStream<Symbol> input, @NotNull DFA dfa, @NotNull DFAState s0,
@Nullable RuleContext outerContext)
{
// dump(dfa); // dump(dfa);
if ( outerContext==null ) outerContext = RuleContext.EMPTY; if ( outerContext==null ) outerContext = RuleContext.EMPTY;
this.outerContext = outerContext; this.outerContext = outerContext;
@ -278,8 +280,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
return prevAccept.alt; return prevAccept.alt;
} }
DecisionState decState = null; @NotNull DecisionState decState = null;
if ( atn.decisionToState.size()>0 ) decState = atn.decisionToState.get(dfa.decision); //if ( atn.decisionToState.size()>0 )
decState = atn.decisionToState.get(dfa.decision);
if ( debug ) System.out.println("decision state = "+decState); if ( debug ) System.out.println("decision state = "+decState);
prevAccept = null; prevAccept = null;
@ -298,7 +301,8 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
ATNState target = getReachableTarget(trans, t); ATNState target = getReachableTarget(trans, t);
if ( target!=null ) { if ( target!=null ) {
Set<ATNConfig> closureBusy = new HashSet<ATNConfig>(); Set<ATNConfig> closureBusy = new HashSet<ATNConfig>();
closure(new ATNConfig(c, target), reach, decState, closureBusy); closure(new ATNConfig(c, target), reach, decState, closureBusy,
c.semanticContext, false);
} }
} }
} }
@ -316,6 +320,17 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
getInputString(input, startIndex)); getInputString(input, startIndex));
System.out.println("REACH="+reach); System.out.println("REACH="+reach);
} }
// can we resolve with predicates?
// TODO: warn if we have uncovered alts?
SemanticContext[] altToPred = getPredsForAmbigAlts(decState, ambigAlts, reach);
if ( altToPred!=null ) {
int uniqueAlt = evalSemanticContext(ambigAlts, altToPred);
DFAState accept = addDFAEdge(dfa, closure, t, reach);
makeAcceptState(accept, altToPred);
return uniqueAlt;
}
dfa.conflict = true; // at least one DFA state is ambiguous dfa.conflict = true; // at least one DFA state is ambiguous
if ( !userWantsCtxSensitive ) reportConflict(startIndex, input.index(), ambigAlts, reach); if ( !userWantsCtxSensitive ) reportConflict(startIndex, input.index(), ambigAlts, reach);
@ -348,8 +363,8 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
if ( debug ) System.out.println("PREDICT alt "+uniqueAlt+ if ( debug ) System.out.println("PREDICT alt "+uniqueAlt+
" decision "+dfa.decision+ " decision "+dfa.decision+
" at index "+input.index()); " at index "+input.index());
addDFAEdge(dfa, closure, t, reach); DFAState accept = addDFAEdge(dfa, closure, t, reach);
makeAcceptState(dfa, reach, uniqueAlt); makeAcceptState(accept, uniqueAlt);
return uniqueAlt; return uniqueAlt;
} }
@ -391,6 +406,31 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
return prevAccept.alt; return prevAccept.alt;
} }
// eval preds first in alt order for ambigAlts;
// if all false, then pick first unpredicated alt in ambigAlts
public int evalSemanticContext(Set<Integer> ambigAlts, SemanticContext[] altToPred) {
int uniqueAlt = ATN.INVALID_ALT_NUMBER;
int firstUnpredicatedAmbigAlt = ATN.INVALID_ALT_NUMBER;
for (int i = 1; i < altToPred.length; i++) {
SemanticContext or = altToPred[i];
if ( or==null || or==SemanticContext.NONE ) {
if ( ambigAlts.contains(i) ) firstUnpredicatedAmbigAlt = i;
continue;
}
System.out.println("eval "+or+"="+or.eval(parser, outerContext));
if ( or.eval(parser, outerContext) ) {
System.out.println("PREDICT "+i);
uniqueAlt = i;
break;
}
}
if ( uniqueAlt==ATN.INVALID_ALT_NUMBER ) {
System.out.println("PREDICT firstUnpredicatedAmbigAlt "+firstUnpredicatedAmbigAlt);
uniqueAlt = firstUnpredicatedAmbigAlt;
}
return uniqueAlt;
}
protected int resolveToMinAlt(@NotNull OrderedHashSet<ATNConfig> reach, @NotNull Set<Integer> ambigAlts) { protected int resolveToMinAlt(@NotNull OrderedHashSet<ATNConfig> reach, @NotNull Set<Integer> ambigAlts) {
int min = getMinAlt(ambigAlts); int min = getMinAlt(ambigAlts);
// create DFA accept state for resolved alt // create DFA accept state for resolved alt
@ -474,7 +514,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
} }
@NotNull @NotNull
public OrderedHashSet<ATNConfig> computeStartState(int decision, @NotNull ATNState p, @Nullable RuleContext ctx) { public OrderedHashSet<ATNConfig> computeStartState(int decision, @NotNull ATNState p,
@Nullable RuleContext ctx)
{
RuleContext initialContext = ctx; // always at least the implicit call to start rule RuleContext initialContext = ctx; // always at least the implicit call to start rule
OrderedHashSet<ATNConfig> configs = new OrderedHashSet<ATNConfig>(); OrderedHashSet<ATNConfig> configs = new OrderedHashSet<ATNConfig>();
prevAccept = null; // might reach end rule; track prevAccept = null; // might reach end rule; track
@ -487,7 +529,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
ATNState target = p.transition(i).target; ATNState target = p.transition(i).target;
ATNConfig c = new ATNConfig(target, i+1, initialContext); ATNConfig c = new ATNConfig(target, i+1, initialContext);
Set<ATNConfig> closureBusy = new HashSet<ATNConfig>(); Set<ATNConfig> closureBusy = new HashSet<ATNConfig>();
closure(c, configs, decState, closureBusy); closure(c, configs, decState, closureBusy, SemanticContext.NONE, true);
} }
return configs; return configs;
@ -528,7 +570,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
protected void closure(@NotNull ATNConfig config, protected void closure(@NotNull ATNConfig config,
@NotNull OrderedHashSet<ATNConfig> configs, @NotNull OrderedHashSet<ATNConfig> configs,
@Nullable DecisionState decState, @Nullable DecisionState decState,
@NotNull Set<ATNConfig> closureBusy) @NotNull Set<ATNConfig> closureBusy,
@NotNull SemanticContext semanticContext,
boolean collectPredicates)
{ {
if ( debug ) System.out.println("closure("+config+")"); if ( debug ) System.out.println("closure("+config+")");
@ -547,7 +591,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
// gotten that context AFTER having fallen off a rule. // gotten that context AFTER having fallen off a rule.
// Make sure we track that we are now out of context. // Make sure we track that we are now out of context.
c.reachesIntoOuterContext = config.reachesIntoOuterContext; c.reachesIntoOuterContext = config.reachesIntoOuterContext;
closure(c, configs, decState, closureBusy); closure(c, configs, decState, closureBusy, semanticContext, collectPredicates);
return; return;
} }
else { else {
@ -571,8 +615,14 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
for (int i=0; i<p.getNumberOfTransitions(); i++) { for (int i=0; i<p.getNumberOfTransitions(); i++) {
Transition t = p.transition(i); Transition t = p.transition(i);
boolean ignorePreds = config.traversedAction; boolean continueCollecting = !(t instanceof ActionTransition) && collectPredicates;
ATNConfig c = getEpsilonTarget(config, t, ignorePreds); if ( debug ) {
if ( t instanceof ActionTransition && collectPredicates ) {
System.out.println("pruning future pred eval derived from s"+
config.state.stateNumber);
}
}
ATNConfig c = getEpsilonTarget(config, t, continueCollecting);
if ( c!=null ) { if ( c!=null ) {
if ( config.state instanceof RuleStopState ) { if ( config.state instanceof RuleStopState ) {
// fell off end of rule. // fell off end of rule.
@ -581,7 +631,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
// preds if this is > 0. // preds if this is > 0.
c.reachesIntoOuterContext++; c.reachesIntoOuterContext++;
} }
closure(c, configs, decState, closureBusy); closure(c, configs, decState, closureBusy, semanticContext, continueCollecting);
} }
} }
} }
@ -593,12 +643,12 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
} }
@Nullable @Nullable
public ATNConfig getEpsilonTarget(@NotNull ATNConfig config, @NotNull Transition t, boolean ignorePreds) { public ATNConfig getEpsilonTarget(@NotNull ATNConfig config, @NotNull Transition t, boolean collectPredicates) {
if ( t instanceof RuleTransition ) { if ( t instanceof RuleTransition ) {
return ruleTransition(config, t); return ruleTransition(config, t);
} }
else if ( t instanceof PredicateTransition ) { else if ( t instanceof PredicateTransition ) {
return predTransition(config, (PredicateTransition)t, ignorePreds); return predTransition(config, (PredicateTransition)t, collectPredicates);
} }
else if ( t instanceof ActionTransition ) { else if ( t instanceof ActionTransition ) {
return actionTransition(config, (ActionTransition)t); return actionTransition(config, (ActionTransition)t);
@ -612,22 +662,16 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
@NotNull @NotNull
public ATNConfig actionTransition(@NotNull ATNConfig config, @NotNull ActionTransition t) { public ATNConfig actionTransition(@NotNull ATNConfig config, @NotNull ActionTransition t) {
if ( debug ) System.out.println("ACTION edge "+t.ruleIndex+":"+t.actionIndex); if ( debug ) System.out.println("ACTION edge "+t.ruleIndex+":"+t.actionIndex);
if ( debug && !config.traversedAction ) { return new ATNConfig(config, t.target);
System.out.println("pruning future pred eval derived from s"+
config.state.stateNumber);
}
ATNConfig c = new ATNConfig(config, t.target);
c.traversedAction = true;
return c;
} }
@Nullable @Nullable
public ATNConfig predTransition(@NotNull ATNConfig config, @NotNull PredicateTransition pt, public ATNConfig predTransition(@NotNull ATNConfig config, @NotNull PredicateTransition pt,
boolean ignorePreds) boolean collectPredicates)
{ {
if ( debug ) { if ( debug ) {
System.out.println("PRED (ignore="+ignorePreds+") "+pt.ruleIndex+":"+pt.predIndex+ System.out.println("PRED (collectPredicates="+collectPredicates+") "+
pt.ruleIndex+":"+pt.predIndex+
", ctx dependent="+pt.isCtxDependent); ", ctx dependent="+pt.isCtxDependent);
if ( parser != null ) { if ( parser != null ) {
System.out.println("context surrounding pred is "+ System.out.println("context surrounding pred is "+
@ -641,27 +685,40 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
// the outer context. // the outer context.
boolean inContext = boolean inContext =
config.context==RuleContext.EMPTY && config.reachesIntoOuterContext==0; config.context==RuleContext.EMPTY && config.reachesIntoOuterContext==0;
RuleContext ctx = null; // RuleContext ctx = null;
if ( inContext ) ctx = outerContext; // if ( inContext ) ctx = outerContext;
ATNConfig c = null;
if ( parser==null || !collectPredicates ) {
c = new ATNConfig(config, pt.target);
}
else if ( !pt.isCtxDependent || (pt.isCtxDependent&&inContext) ) {
SemanticContext newSemCtx = new SemanticContext.AND(config.semanticContext, pt.getPredicate());
c = new ATNConfig(config, pt.target, newSemCtx);
}
return c;
// We see through the predicate if: // We see through the predicate if:
// 0) we have no parser to eval preds
// 1) we are ignoring them // 1) we are ignoring them
// 2) we aren't ignoring them and it is not context dependent and // 2) we aren't ignoring them and it is not context dependent and
// pred is true // pred is true
// 3) we aren't ignoring them, it is context dependent, but // 3) we aren't ignoring them, it is context dependent, but
// we know the context and pred is true // we know the context and pred is true
// 4) we aren't ignoring them, it is context dependent, but we don't know context // 4) we aren't ignoring them, it is context dependent, but we don't know context
ATNConfig c = null; // ATNConfig c = null;
boolean seeThroughPred = // boolean seeThroughPred =
ignorePreds || // parser==null ||
(!ignorePreds&&!pt.isCtxDependent&&parser.sempred(ctx, pt.ruleIndex, pt.predIndex))|| // !collectPredicates ||
(!ignorePreds&&pt.isCtxDependent&&inContext&&parser.sempred(ctx, pt.ruleIndex, pt.predIndex)|| // (collectPredicates&&!pt.isCtxDependent&&parser.sempred(ctx, pt.ruleIndex, pt.predIndex))||
(!ignorePreds&&pt.isCtxDependent&&!inContext)); // (collectPredicates&&pt.isCtxDependent&&inContext&&parser.sempred(ctx, pt.ruleIndex, pt.predIndex)||
if ( seeThroughPred ) { // (collectPredicates&&pt.isCtxDependent&&!inContext));
c = new ATNConfig(config, pt.target); // if ( seeThroughPred ) {
c.traversedPredicate = true; // c = new ATNConfig(config, pt.target, pt.getSemanticContext());
} // c.traversedPredicate = true;
return c; // }
// return c;
} }
@NotNull @NotNull
@ -718,7 +775,6 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
public Set<Integer> getAmbiguousAlts(@NotNull OrderedHashSet<ATNConfig> configs) { public Set<Integer> getAmbiguousAlts(@NotNull OrderedHashSet<ATNConfig> configs) {
// System.err.println("check ambiguous "+configs); // System.err.println("check ambiguous "+configs);
Set<Integer> ambigAlts = null; Set<Integer> ambigAlts = null;
int numConfigs = configs.size();
// First get a list of configurations for each state. // First get a list of configurations for each state.
// Most of the time, each state will have one associated configuration. // Most of the time, each state will have one associated configuration.
MultiMap<Integer, ATNConfig> stateToConfigListMap = MultiMap<Integer, ATNConfig> stateToConfigListMap =
@ -759,6 +815,29 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
return ambigAlts; return ambigAlts;
} }
public SemanticContext[] getPredsForAmbigAlts(@NotNull DecisionState decState,
@NotNull Set<Integer> ambigAlts,
@NotNull OrderedHashSet<ATNConfig> configs)
{
// REACH=[1|1|[]|0:0, 1|2|[]|0:1]
if ( debug ) System.out.println("getPredsForAmbigAlts decision "+decState.decision);
int nalts = decState.getNumberOfTransitions();
SemanticContext[] altToPred = new SemanticContext[nalts +1];
for (int alt : ambigAlts) { altToPred[alt] = SemanticContext.NONE; }
boolean atLeastOne = false;
for (ATNConfig c : configs) {
if ( c.semanticContext!=SemanticContext.NONE && ambigAlts.contains(c.alt) ) {
altToPred[c.alt] = new SemanticContext.OR(altToPred[c.alt], c.semanticContext);
c.resolveWithPredicate = true;
atLeastOne = true;
}
}
// nonambig alts are null in altToPred
if ( !atLeastOne ) altToPred = null;
System.out.println(Arrays.toString(altToPred));
return altToPred;
}
public static int getMinAlt(@NotNull Set<Integer> ambigAlts) { public static int getMinAlt(@NotNull Set<Integer> ambigAlts) {
int min = Integer.MAX_VALUE; int min = Integer.MAX_VALUE;
for (int alt : ambigAlts) { for (int alt : ambigAlts) {
@ -783,9 +862,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
int t, int t,
@NotNull OrderedHashSet<ATNConfig> q) @NotNull OrderedHashSet<ATNConfig> q)
{ {
// System.out.println("MOVE "+p+" -> "+q+" upon "+getTokenName(t));
DFAState from = addDFAState(dfa, p); DFAState from = addDFAState(dfa, p);
DFAState to = addDFAState(dfa, q); DFAState to = addDFAState(dfa, q);
if ( debug ) System.out.println("EDGE "+from+" -> "+to+" upon "+getTokenName(t));
addDFAEdge(from, t, to); addDFAEdge(from, t, to);
return to; return to;
} }
@ -807,28 +886,33 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
DFAState newState = proposed; DFAState newState = proposed;
boolean traversedPredicate = false; // boolean traversedPredicate = false;
for (ATNConfig c : configs) { // for (ATNConfig c : configs) {
if ( c.traversedPredicate ) {traversedPredicate = true; break;} // if ( c.traversedPredicate ) {traversedPredicate = true; break;}
} // }
//
if ( traversedPredicate ) return null; // cannot cache // if ( traversedPredicate ) return null; // cannot cache
newState.stateNumber = dfa.states.size(); newState.stateNumber = dfa.states.size();
newState.configs = new OrderedHashSet<ATNConfig>(); newState.configs = new OrderedHashSet<ATNConfig>();
newState.configs.addAll(configs); newState.configs.addAll(configs);
dfa.states.put(newState, newState); dfa.states.put(newState, newState);
if ( debug ) System.out.println("adding new DFA state: "+newState);
return newState; return newState;
} }
public void makeAcceptState(@NotNull DFA dfa, @NotNull OrderedHashSet<ATNConfig> reach, int uniqueAlt) { public void makeAcceptState(@NotNull DFAState accept, int uniqueAlt) {
DFAState accept = dfa.states.get(new DFAState(reach));
if ( accept==null ) return;
accept.isAcceptState = true; accept.isAcceptState = true;
accept.prediction = uniqueAlt; accept.prediction = uniqueAlt;
accept.complete = true; accept.complete = true;
} }
public void makeAcceptState(@NotNull DFAState accept, @NotNull SemanticContext[] altToPred) {
accept.isAcceptState = true;
accept.complete = true;
accept.altToPred = altToPred;
}
@NotNull @NotNull
public String getTokenName(int t) { public String getTokenName(int t) {
if ( t==-1 ) return "EOF"; if ( t==-1 ) return "EOF";

4
runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java
View File

@ -52,6 +52,10 @@ public class PredicateTransition extends Transition {
@Override @Override
public boolean isEpsilon() { return true; } public boolean isEpsilon() { return true; }
public SemanticContext.Predicate getPredicate() {
return new SemanticContext.Predicate(ruleIndex, predIndex, isCtxDependent);
}
@Override @Override
@NotNull @NotNull
public String toString() { public String toString() {

499
runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java
View File

@ -29,49 +29,26 @@
package org.antlr.v4.runtime.atn; package org.antlr.v4.runtime.atn;
import java.util.*; import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.RuleContext;
/** A binary tree structure used to record the semantic context in which /** A tree structure used to record the semantic context in which
* an NFA configuration is valid. It's either a single predicate or * an ATN configuration is valid. It's either a single predicate,
* a tree representing an operation tree such as: p1&&p2 or p1||p2. * a conjunction p1&&p2, or a sum of products p1||p2.
* *
* For ATN o-p1->o-p2->o, create tree AND(p1,p2). * I have scoped the AND, OR, and Predicate subclasses of
* For ATN (1)-p1->(2)
* | ^
* | |
* (3)-p2----
* we will have to combine p1 and p2 into DFA state as we will be
* adding NFA configurations for state 2 with two predicates p1,p2.
* So, set context for combined NFA config for state 2: OR(p1,p2).
*
* I have scoped the AND, NOT, OR, and Predicate subclasses of
* SemanticContext within the scope of this outer class. * SemanticContext within the scope of this outer class.
*
* Pulled from v3; lots of fixes by Sam Harwell in v3 I notice.
*/ */
public abstract class SemanticContext { public abstract class SemanticContext {
/** Create a default value for the semantic context shared among all public static final SemanticContext NONE = new Predicate();
* ATNConfigs that do not have an actual semantic context.
* This prevents lots of if!=null type checks all over; it represents public abstract boolean eval(Recognizer<?,?> parser, RuleContext ctx);
* just an empty set of predicates.
*/
public static final SemanticContext EMPTY_SEMANTIC_CONTEXT = new Predicate();
public static class Predicate extends SemanticContext { public static class Predicate extends SemanticContext {
public final int ruleIndex; public final int ruleIndex;
public final int predIndex; public final int predIndex;
public final boolean isCtxDependent; // e.g., $i ref in pred public final boolean isCtxDependent; // e.g., $i ref in pred
// public static final int INVALID_PRED_VALUE = -2;
// public static final int FALSE_PRED = 0;
// public static final int TRUE_PRED = ~0;
/** sometimes predicates are known to be true or false; we need
* a way to represent this without resorting to a target language
* value like true or TRUE.
*/
// protected int constantValue = INVALID_PRED_VALUE;
protected Predicate() { protected Predicate() {
this.ruleIndex = -1; this.ruleIndex = -1;
this.predIndex = -1; this.predIndex = -1;
@ -90,36 +67,16 @@ public abstract class SemanticContext {
this.isCtxDependent = p.isCtxDependent; this.isCtxDependent = p.isCtxDependent;
} }
public boolean equals(Object o) { public boolean eval(Recognizer<?,?> parser, RuleContext ctx) {
if ( !(o instanceof Predicate) ) { return parser.sempred(ctx, ruleIndex, predIndex);
return false;
} }
if ( this != o ) return false;
Predicate other = (Predicate)o;
return this.ruleIndex == other.ruleIndex &&
this.predIndex == other.predIndex;
}
public int hashCode() {
return ruleIndex+predIndex;
}
public String toString() { public String toString() {
return ruleIndex+":"+predIndex; return ruleIndex+":"+predIndex;
} }
} }
public static class TruePredicate extends Predicate { public static class TruePredicate extends Predicate {
@Override
public int hashCode() {
return 1;
}
@Override
public boolean equals(Object o) {
return this == o || o instanceof TruePredicate;
}
@Override @Override
public String toString() { public String toString() {
return "true"; // not used for code gen, just DOT and print outs return "true"; // not used for code gen, just DOT and print outs
@ -127,415 +84,73 @@ public abstract class SemanticContext {
} }
public static class FalsePredicate extends Predicate { public static class FalsePredicate extends Predicate {
@Override
public int hashCode() {
return 0;
}
@Override
public boolean equals(Object o) {
return this == o || o instanceof FalsePredicate;
}
@Override @Override
public String toString() { public String toString() {
return "false"; // not used for code gen, just DOT and print outs return "false"; // not used for code gen, just DOT and print outs
} }
} }
public static abstract class CommutativePredicate extends SemanticContext { public static class AND extends SemanticContext {
protected final Set<SemanticContext> operands = new HashSet<SemanticContext>(); public SemanticContext a;
protected int hashCode; public SemanticContext b;
public AND() { }
public CommutativePredicate(SemanticContext a, SemanticContext b) {
if (a.getClass() == this.getClass()){
CommutativePredicate predicate = (CommutativePredicate)a;
operands.addAll(predicate.operands);
}
else {
operands.add(a);
}
if (b.getClass() == this.getClass()){
CommutativePredicate predicate = (CommutativePredicate)b;
operands.addAll(predicate.operands);
}
else {
operands.add(b);
}
hashCode = calculateHashCode();
}
public CommutativePredicate(HashSet<SemanticContext> contexts){
for (SemanticContext context : contexts){
if (context.getClass() == this.getClass()){
CommutativePredicate predicate = (CommutativePredicate)context;
operands.addAll(predicate.operands);
}
else {
operands.add(context);
}
}
hashCode = calculateHashCode();
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj.getClass() == this.getClass()) {
CommutativePredicate commutative = (CommutativePredicate)obj;
Set<SemanticContext> otherOperands = commutative.operands;
if (operands.size() != otherOperands.size())
return false;
return operands.containsAll(otherOperands);
}
if (obj instanceof NOT) {
NOT not = (NOT)obj;
if (not.ctx instanceof CommutativePredicate && not.ctx.getClass() != this.getClass()) {
Set<SemanticContext> otherOperands = ((CommutativePredicate)not.ctx).operands;
if (operands.size() != otherOperands.size())
return false;
ArrayList<SemanticContext> temp = new ArrayList<SemanticContext>(operands.size());
for (SemanticContext context : otherOperands) {
temp.add(not(context));
}
return operands.containsAll(temp);
}
}
return false;
}
@Override
public int hashCode(){
return hashCode;
}
@Override
public String toString() {
StringBuffer buf = new StringBuffer();
buf.append("(");
int i = 0;
for (SemanticContext semctx : operands) {
if ( i>0 ) {
buf.append(getOperandString());
}
buf.append(semctx.toString());
i++;
}
buf.append(")");
return buf.toString();
}
public abstract String getOperandString();
public abstract SemanticContext combinePredicates(SemanticContext left, SemanticContext right);
public int calculateHashCode() {
int hashcode = 0;
for (SemanticContext context : operands) {
hashcode = hashcode ^ context.hashCode();
}
return hashcode;
}
}
public static class AND extends CommutativePredicate {
public AND(SemanticContext a, SemanticContext b) { public AND(SemanticContext a, SemanticContext b) {
super(a,b); this.a = a;
this.b = b;
} }
public AND(HashSet<SemanticContext> contexts) { public boolean eval(Recognizer<?,?> parser, RuleContext ctx) {
super(contexts); if ( a == NONE ) return b.eval(parser, ctx);
if ( b == NONE ) return a.eval(parser, ctx);
return a.eval(parser, ctx) && b.eval(parser, ctx);
} }
@Override public String toString() {
public String getOperandString() { if ( a == NONE ) return b.toString();
return "&&"; if ( b == NONE ) return a.toString();
return a+"&&"+b;
}
} }
@Override public static class OR extends SemanticContext {
public SemanticContext combinePredicates(SemanticContext left, SemanticContext right) { public SemanticContext a;
return SemanticContext.and(left, right); public SemanticContext b;
} public OR() { }
// @Override
// public int calculateHashCode() {
// int hashcode = 0;
// for (SemanticContext context : operands) {
// hashcode = hashcode ^ context.hashCode();
// }
//
// return hashcode;
// }
}
public static class OR extends CommutativePredicate {
public OR(SemanticContext a, SemanticContext b) { public OR(SemanticContext a, SemanticContext b) {
super(a,b); this.a = a;
this.b = b;
} }
public OR(HashSet<SemanticContext> contexts) { public boolean eval(Recognizer<?,?> parser, RuleContext ctx) {
super(contexts); if ( a == NONE ) return b.eval(parser, ctx);
} if ( b == NONE ) return a.eval(parser, ctx);
return a.eval(parser, ctx) && b.eval(parser, ctx);
@Override
public String getOperandString() {
return "||";
}
@Override
public SemanticContext combinePredicates(SemanticContext left, SemanticContext right) {
return SemanticContext.or(left, right);
}
//
// @Override
// public int calculateHashCode() {
// int hashcode = 0;
// for (SemanticContext context : operands) {
// hashcode = ~hashcode ^ context.hashCode();
// }
//
// return hashcode;
// }
}
public static class NOT extends SemanticContext {
protected SemanticContext ctx;
public NOT(SemanticContext ctx) {
this.ctx = ctx;
}
@Override
public boolean equals(Object object) {
if ( !(object instanceof NOT) ) {
return false;
}
return this.ctx.equals(((NOT)object).ctx);
}
@Override
public int hashCode() {
return ~ctx.hashCode();
} }
@Override @Override
public String toString() { public String toString() {
return "!("+ctx+")"; if ( a == NONE ) return b.toString();
if ( b == NONE ) return a.toString();
return a+"||"+b;
} }
} }
public static SemanticContext and(SemanticContext a, SemanticContext b) { // public static SemanticContext and(AND a, Predicate b) {
//System.out.println("AND: "+a+"&&"+b); // if ( a == NONE ) return new AND(b);
if (a instanceof FalsePredicate || b instanceof FalsePredicate) // if ( b == NONE ) {a.add(b); return a;}
return new FalsePredicate(); // a.add(b);
// AND and = new SemanticContext.AND();
SemanticContext[] terms = factorOr(a, b); // if ( a instanceof AND ) {
SemanticContext commonTerms = terms[0]; // and.add(b);
a = terms[1]; // }
b = terms[2]; // else {
// and.add((Predicate)a);
boolean factored = commonTerms != null && commonTerms != EMPTY_SEMANTIC_CONTEXT && !(commonTerms instanceof TruePredicate); // and.add(b);
if (factored) { // }
return or(commonTerms, and(a, b)); // return and;
} // }
//
//System.Console.Out.WriteLine( "AND: " + a + "&&" + b ); // public static SemanticContext or(SemanticContext a, SemanticContext b) {
if (a instanceof FalsePredicate || b instanceof FalsePredicate) // return null;
return new FalsePredicate(); // }
if ( a==EMPTY_SEMANTIC_CONTEXT || a==null ) {
return b;
}
if ( b==EMPTY_SEMANTIC_CONTEXT || b==null ) {
return a;
}
if (a instanceof TruePredicate)
return b;
if (b instanceof TruePredicate)
return a;
//// Factoring takes care of this case
//if (a.Equals(b))
// return a;
//System.out.println("## have to AND");
return new AND(a,b);
}
public static SemanticContext or(SemanticContext a, SemanticContext b) {
//System.out.println("OR: "+a+"||"+b);
if (a instanceof TruePredicate || b instanceof TruePredicate)
return new TruePredicate();
SemanticContext[] terms = factorAnd(a, b);
SemanticContext commonTerms = terms[0];
a = terms[1];
b = terms[2];
boolean factored = commonTerms != null && commonTerms != EMPTY_SEMANTIC_CONTEXT && !(commonTerms instanceof FalsePredicate);
if (factored) {
return and(commonTerms, or(a, b));
}
if ( a==EMPTY_SEMANTIC_CONTEXT || a==null || a instanceof FalsePredicate ) {
return b;
}
if ( b==EMPTY_SEMANTIC_CONTEXT || b==null || b instanceof FalsePredicate ) {
return a;
}
if ( a instanceof TruePredicate || b instanceof TruePredicate || commonTerms instanceof TruePredicate ) {
return new TruePredicate();
}
//// Factoring takes care of this case
//if (a.equals(b))
// return a;
if ( a instanceof NOT ) {
NOT n = (NOT)a;
// check for !p||p
if ( n.ctx.equals(b) ) {
return new TruePredicate();
}
}
else if ( b instanceof NOT ) {
NOT n = (NOT)b;
// check for p||!p
if ( n.ctx.equals(a) ) {
return new TruePredicate();
}
}
//System.out.println("## have to OR");
OR result = new OR(a,b);
if (result.operands.size() == 1)
return result.operands.iterator().next();
return result;
}
public static SemanticContext not(SemanticContext a) {
if (a instanceof NOT) return ((NOT)a).ctx;
if (a instanceof TruePredicate) return new FalsePredicate();
else if (a instanceof FalsePredicate) return new TruePredicate();
return new NOT(a);
}
// Factor so (a && b) == (result && a && b)
public static SemanticContext[] factorAnd(SemanticContext a, SemanticContext b) {
if (a == EMPTY_SEMANTIC_CONTEXT || a == null || a instanceof FalsePredicate) {
return new SemanticContext[] { EMPTY_SEMANTIC_CONTEXT, a, b };
}
if (b == EMPTY_SEMANTIC_CONTEXT || b == null || b instanceof FalsePredicate) {
return new SemanticContext[] { EMPTY_SEMANTIC_CONTEXT, a, b };
}
if (a instanceof TruePredicate || b instanceof TruePredicate) {
return new SemanticContext[] {
new TruePredicate(), EMPTY_SEMANTIC_CONTEXT, EMPTY_SEMANTIC_CONTEXT
};
}
HashSet<SemanticContext> opsA = new HashSet<SemanticContext>(getAndOperands(a));
HashSet<SemanticContext> opsB = new HashSet<SemanticContext>(getAndOperands(b));
HashSet<SemanticContext> result = new HashSet<SemanticContext>(opsA);
result.retainAll(opsB);
if (result.size() == 0) {
return new SemanticContext[] { EMPTY_SEMANTIC_CONTEXT, a, b };
}
opsA.removeAll(result);
if (opsA.size() == 0) a = new TruePredicate();
else if (opsA.size() == 1) a = opsA.iterator().next();
else a = new AND(opsA);
opsB.removeAll(result);
if (opsB.size() == 0) b = new TruePredicate();
else if (opsB.size() == 1) b = opsB.iterator().next();
else b = new AND(opsB);
if (result.size() == 1) {
return new SemanticContext[] { result.iterator().next(), a, b };
}
return new SemanticContext[] { new AND(result), a, b };
}
// Factor so (a || b) == (result || a || b)
public static SemanticContext[] factorOr(SemanticContext a, SemanticContext b) {
HashSet<SemanticContext> opsA = new HashSet<SemanticContext>(getOrOperands(a));
HashSet<SemanticContext> opsB = new HashSet<SemanticContext>(getOrOperands(b));
HashSet<SemanticContext> result = new HashSet<SemanticContext>(opsA);
result.retainAll(opsB);
if (result.size() == 0) {
return new SemanticContext[] { EMPTY_SEMANTIC_CONTEXT, a, b };
}
opsA.removeAll(result);
if (opsA.size() == 0) a = new FalsePredicate();
else if (opsA.size() == 1) a = opsA.iterator().next();
else a = new OR(opsA);
opsB.removeAll(result);
if (opsB.size() == 0) b = new FalsePredicate();
else if (opsB.size() == 1) b = opsB.iterator().next();
else b = new OR(opsB);
if (result.size() == 1) {
return new SemanticContext[] { result.iterator().next(), a, b };
}
return new SemanticContext[] { new OR(result), a, b };
}
public static Collection<SemanticContext> getAndOperands(SemanticContext context) {
if (context instanceof AND) return ((AND)context).operands;
if (context instanceof NOT) {
Collection<SemanticContext> operands = getOrOperands(((NOT)context).ctx);
List<SemanticContext> result = new ArrayList<SemanticContext>(operands.size());
for (SemanticContext operand : operands) {
result.add(not(operand));
}
return result;
}
ArrayList<SemanticContext> result = new ArrayList<SemanticContext>();
result.add(context);
return result;
}
public static Collection<SemanticContext> getOrOperands(SemanticContext context) {
if (context instanceof OR) return ((OR)context).operands;
if (context instanceof NOT) {
Collection<SemanticContext> operands = getAndOperands(((NOT)context).ctx);
List<SemanticContext> result = new ArrayList<SemanticContext>(operands.size());
for (SemanticContext operand : operands) {
result.add(not(operand));
}
return result;
}
ArrayList<SemanticContext> result = new ArrayList<SemanticContext>();
result.add(context);
return result;
}
} }

7
runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java
View File

@ -32,6 +32,8 @@ package org.antlr.v4.runtime.dfa;
import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable; import org.antlr.v4.runtime.misc.Nullable;
import java.util.Arrays;
/** A DFA walker that knows how to dump them to serialized strings. */ /** A DFA walker that knows how to dump them to serialized strings. */
public class DFASerializer { public class DFASerializer {
@NotNull @NotNull
@ -77,7 +79,12 @@ public class DFASerializer {
int n = s.stateNumber; int n = s.stateNumber;
String stateStr = "s"+n; String stateStr = "s"+n;
if ( s.isAcceptState ) { if ( s.isAcceptState ) {
if ( s.altToPred!=null ) {
stateStr = ":s"+n+"=>"+Arrays.toString(Arrays.copyOfRange(s.altToPred, 1, s.altToPred.length));
}
else {
stateStr = ":s"+n+"=>"+s.prediction; stateStr = ":s"+n+"=>"+s.prediction;
}
} }
if ( s.isCtxSensitive ) { if ( s.isCtxSensitive ) {
stateStr = ":s"+n+"@"+s.ctxToPrediction; stateStr = ":s"+n+"@"+s.ctxToPrediction;

32
runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java
View File

@ -29,12 +29,16 @@
package org.antlr.v4.runtime.dfa; package org.antlr.v4.runtime.dfa;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.atn.ATNConfig; import org.antlr.v4.runtime.atn.ATNConfig;
import org.antlr.v4.runtime.atn.SemanticContext;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.misc.OrderedHashSet; import org.antlr.v4.runtime.misc.OrderedHashSet;
import java.util.*; import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
/** A DFA state represents a set of possible ATN configurations. /** A DFA state represents a set of possible ATN configurations.
* As Aho, Sethi, Ullman p. 117 says "The DFA uses its state * As Aho, Sethi, Ullman p. 117 says "The DFA uses its state
@ -49,16 +53,15 @@ import java.util.*;
* state as well, however. More importantly, we need to maintain * state as well, however. More importantly, we need to maintain
* a stack of states, tracking the closure operations as they * a stack of states, tracking the closure operations as they
* jump from rule to rule, emulating rule invocations (method calls). * jump from rule to rule, emulating rule invocations (method calls).
* Recall that ATNs do not normally have a stack like a pushdown-machine * I have to add a stack to simulate the proper lookahead sequences for
* so I have to add one to simulate the proper lookahead sequences for
* the underlying LL grammar from which the ATN was derived. * the underlying LL grammar from which the ATN was derived.
* *
* I use a list of ATNConfig objects. An ATNConfig * I use a set of ATNConfig objects not simple states. An ATNConfig
* is both a state (ala normal conversion) and a RuleContext describing * is both a state (ala normal conversion) and a RuleContext describing
* the chain of rules (if any) followed to arrive at that state. * the chain of rules (if any) followed to arrive at that state.
* *
* A DFA state may have multiple references to a particular state, * A DFA state may have multiple references to a particular state,
* but with different ATNContexts (with same or different alts) * but with different ATN contexts (with same or different alts)
* meaning that state was reached via a different set of rule invocations. * meaning that state was reached via a different set of rule invocations.
*/ */
public class DFAState { public class DFAState {
@ -84,7 +87,9 @@ public class DFAState {
public boolean isCtxSensitive; public boolean isCtxSensitive;
@Nullable @Nullable
public Map<RuleContext, Integer> ctxToPrediction; public Map<RuleContext, Integer> ctxToPrediction; // used for ctx sensitive parsing
public SemanticContext[] altToPred;
public DFAState() { } public DFAState() { }
@ -150,6 +155,17 @@ public class DFAState {
@Override @Override
public String toString() { public String toString() {
return stateNumber+":"+configs+(isAcceptState?("=>"+prediction):""); StringBuilder buf = new StringBuilder();
buf.append(stateNumber + ":" + configs);
if ( isAcceptState ) {
buf.append("=>");
if ( altToPred!=null ) {
buf.append(Arrays.toString(Arrays.copyOfRange(altToPred, 1, altToPred.length)));
}
else {
buf.append(prediction);
}
}
return buf.toString();
} }
} }

14
tool/test/org/antlr/v4/test/TestSemPredEvalParser.java
View File

@ -3,19 +3,19 @@ package org.antlr.v4.test;
import org.junit.Test; import org.junit.Test;
public class TestSemPredEvalParser extends BaseTest { public class TestSemPredEvalParser extends BaseTest {
@Test public void testToLeft() throws Exception { @Test public void testSimple() throws Exception {
String grammar = String grammar =
"grammar T;\n" + "grammar T;\n" +
"s : a+ ;\n" +
"a : {false}? ID {System.out.println(\"alt 1\");}\n" + "a : {false}? ID {System.out.println(\"alt 1\");}\n" +
" | {true}? ID {System.out.println(\"alt 2\");}\n" + " | {true}? ID {System.out.println(\"alt 2\");}\n" +
" | INT {System.out.println(\"alt 3\");}\n" +
" ;\n" + " ;\n" +
"ID : 'a'..'z'+ ;\n" + "ID : 'a'..'z'+ ;\n" +
"INT : '0'..'9'+;\n" + "INT : '0'..'9'+;\n" +
"WS : (' '|'\\n') {skip();} ;\n"; "WS : (' '|'\\n') {skip();} ;\n";
String found = execParser("T.g", grammar, "TParser", "TLexer", "s", String found = execParser("T.g", grammar, "TParser", "TLexer", "a",
"x x y", false); "x", false);
String expecting = String expecting =
"alt 2\n" + "alt 2\n" +
"alt 2\n" + "alt 2\n" +
@ -23,12 +23,12 @@ public class TestSemPredEvalParser extends BaseTest {
assertEquals(expecting, found); assertEquals(expecting, found);
} }
@Test public void testToRight() throws Exception { @Test public void testToLeft() throws Exception {
String grammar = String grammar =
"grammar T;\n" + "grammar T;\n" +
"s : a+ ;\n" + "s : a+ ;\n" +
"a : ID {false}? {System.out.println(\"alt 1\");}\n" + "a : {false}? ID {System.out.println(\"alt 1\");}\n" +
" | ID {true}? {System.out.println(\"alt 2\");}\n" + " | {true}? ID {System.out.println(\"alt 2\");}\n" +
" ;\n" + " ;\n" +
"ID : 'a'..'z'+ ;\n" + "ID : 'a'..'z'+ ;\n" +
"INT : '0'..'9'+;\n" + "INT : '0'..'9'+;\n" +