forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

interim pred overhaul 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9479]
This commit is contained in:
parrt 2011-11-28 09:37:29 -08:00
parent eb6443b32b
commit e288ff3039
7 changed files with 327 additions and 583 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java
View File

@ -29,9 +29,10 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.*;
/** An ATN state, predicted alt, and syntactic/semantic context.
* The syntactic context is a pointer into the rule invocation
@ -61,16 +62,7 @@ public class ATNConfig {
* otherwise predicates would not get executed again (DFAs don't
* have predicated edges in v4).
*/
public boolean traversedPredicate;
/**
* Indicates that we have reached this ATN configuration after
* traversing a non-forced action transition. We do not execute
* predicates after such actions because the predicates could be
* functions of the side effects. Force actions must be either side
* effect free or automatically undone as the parse continues.
*/
public boolean traversedAction;
public boolean traversedPredicate; // TODO: don't need
/**
* We cannot execute predicates dependent upon local context unless
@ -85,6 +77,18 @@ public class ATNConfig {
*/
public int reachesIntoOuterContext;
@NotNull
public SemanticContext semanticContext = SemanticContext.NONE;
/** This bit is used to indicate a semantic predicate will be
* used to resolve the conflict. Essentially, this is used
* as an "ignore" bit so that upon a set of conflicting configurations,
* such as (s|2|p) and (s|3|q), I can set (s|3) to resolved=true (and any
* other configuration associated with alt 3) to make it look like that set
* uniquely predicts an alt.
*/
protected boolean resolveWithPredicate;
public ATNConfig(@NotNull ATNState state,
int alt,
@Nullable RuleContext context)
@ -94,26 +98,36 @@ public class ATNConfig {
this.context = context;
}
public ATNConfig(@NotNull ATNConfig c) {
this(c, c.state, c.context);
}
// public ATNConfig(@NotNull ATNConfig c) {
// this(c, c.state, c.context, c.semanticContext);
// }
public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state) {
this(c, state, c.context);
}
public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state) {
this(c, state, c.context, c.semanticContext);
}
public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, @Nullable RuleContext context) {
public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, SemanticContext semanticContext) {
this(c, state, c.context, semanticContext);
}
public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, @Nullable RuleContext context) {
this(c, state, context, c.semanticContext);
}
public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, @Nullable RuleContext context,
SemanticContext semanticContext)
{
this.state = state;
this.alt = c.alt;
this.context = context;
this.traversedPredicate = c.traversedPredicate;
this.traversedAction = c.traversedAction;
this.reachesIntoOuterContext = c.reachesIntoOuterContext;
this.semanticContext = semanticContext;
}
public ATNConfig(@NotNull ATNConfig c, @Nullable RuleContext context) {
this(c, c.state, context);
}
// public ATNConfig(@NotNull ATNConfig c, @Nullable RuleContext context) {
// this(c, c.state, context);
// }
/** An ATN configuration is equal to another if both have
* the same state, they predict the same alternative, and
@ -153,16 +167,20 @@ public class ATNConfig {
// }
buf.append(state);
if ( showAlt ) {
buf.append("|");
buf.append(alt);
}
if ( context!=null ) {
buf.append("|");
buf.append(alt);
}
if ( context!=null ) {
buf.append("|");
buf.append(context);
}
if ( reachesIntoOuterContext>0 ) {
buf.append("|up=").append(reachesIntoOuterContext);
}
if ( semanticContext!=null ) {
buf.append("|");
buf.append(semanticContext);
}
if ( reachesIntoOuterContext>0 ) {
buf.append("|up=").append(reachesIntoOuterContext);
}
// if (isAccept) {
// buf.append("|=>"+alt);
// }

202
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -43,8 +43,8 @@ import org.stringtemplate.v4.misc.MultiMap;
import java.util.*;
public class ParserATNSimulator<Symbol> extends ATNSimulator {
public static boolean debug = false;
public static boolean dfa_debug = false;
public static boolean debug = true;
public static boolean dfa_debug = true;
public static int ATN_failover = 0;
public static int predict_calls = 0;
@ -161,7 +161,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
return execATN(input, dfa, input.index(), s0_closure, false);
}
public int execDFA(@NotNull SymbolStream<Symbol> input, @NotNull DFA dfa, @NotNull DFAState s0, @Nullable RuleContext outerContext) {
public int execDFA(@NotNull SymbolStream<Symbol> input, @NotNull DFA dfa, @NotNull DFAState s0,
@Nullable RuleContext outerContext)
{
// dump(dfa);
if ( outerContext==null ) outerContext = RuleContext.EMPTY;
this.outerContext = outerContext;
@ -278,8 +280,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
return prevAccept.alt;
}
DecisionState decState = null;
if ( atn.decisionToState.size()>0 ) decState = atn.decisionToState.get(dfa.decision);
@NotNull DecisionState decState = null;
//if ( atn.decisionToState.size()>0 )
decState = atn.decisionToState.get(dfa.decision);
if ( debug ) System.out.println("decision state = "+decState);
prevAccept = null;
@ -298,7 +301,8 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
ATNState target = getReachableTarget(trans, t);
if ( target!=null ) {
Set<ATNConfig> closureBusy = new HashSet<ATNConfig>();
closure(new ATNConfig(c, target), reach, decState, closureBusy);
closure(new ATNConfig(c, target), reach, decState, closureBusy,
c.semanticContext, false);
}
}
}
@ -316,6 +320,17 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
getInputString(input, startIndex));
System.out.println("REACH="+reach);
}
// can we resolve with predicates?
// TODO: warn if we have uncovered alts?
SemanticContext[] altToPred = getPredsForAmbigAlts(decState, ambigAlts, reach);
if ( altToPred!=null ) {
int uniqueAlt = evalSemanticContext(ambigAlts, altToPred);
DFAState accept = addDFAEdge(dfa, closure, t, reach);
makeAcceptState(accept, altToPred);
return uniqueAlt;
}
dfa.conflict = true; // at least one DFA state is ambiguous
if ( !userWantsCtxSensitive ) reportConflict(startIndex, input.index(), ambigAlts, reach);
@ -348,8 +363,8 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
if ( debug ) System.out.println("PREDICT alt "+uniqueAlt+
" decision "+dfa.decision+
" at index "+input.index());
addDFAEdge(dfa, closure, t, reach);
makeAcceptState(dfa, reach, uniqueAlt);
DFAState accept = addDFAEdge(dfa, closure, t, reach);
makeAcceptState(accept, uniqueAlt);
return uniqueAlt;
}
@ -391,7 +406,32 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
return prevAccept.alt;
}
protected int resolveToMinAlt(@NotNull OrderedHashSet<ATNConfig> reach, @NotNull Set<Integer> ambigAlts) {
// eval preds first in alt order for ambigAlts;
// if all false, then pick first unpredicated alt in ambigAlts
public int evalSemanticContext(Set<Integer> ambigAlts, SemanticContext[] altToPred) {
int uniqueAlt = ATN.INVALID_ALT_NUMBER;
int firstUnpredicatedAmbigAlt = ATN.INVALID_ALT_NUMBER;
for (int i = 1; i < altToPred.length; i++) {
SemanticContext or = altToPred[i];
if ( or==null || or==SemanticContext.NONE ) {
if ( ambigAlts.contains(i) ) firstUnpredicatedAmbigAlt = i;
continue;
}
System.out.println("eval "+or+"="+or.eval(parser, outerContext));
if ( or.eval(parser, outerContext) ) {
System.out.println("PREDICT "+i);
uniqueAlt = i;
break;
}
}
if ( uniqueAlt==ATN.INVALID_ALT_NUMBER ) {
System.out.println("PREDICT firstUnpredicatedAmbigAlt "+firstUnpredicatedAmbigAlt);
uniqueAlt = firstUnpredicatedAmbigAlt;
}
return uniqueAlt;
}
protected int resolveToMinAlt(@NotNull OrderedHashSet<ATNConfig> reach, @NotNull Set<Integer> ambigAlts) {
int min = getMinAlt(ambigAlts);
// create DFA accept state for resolved alt
ambigAlts.remove(min);
@ -474,7 +514,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
}
@NotNull
public OrderedHashSet<ATNConfig> computeStartState(int decision, @NotNull ATNState p, @Nullable RuleContext ctx) {
public OrderedHashSet<ATNConfig> computeStartState(int decision, @NotNull ATNState p,
@Nullable RuleContext ctx)
{
RuleContext initialContext = ctx; // always at least the implicit call to start rule
OrderedHashSet<ATNConfig> configs = new OrderedHashSet<ATNConfig>();
prevAccept = null; // might reach end rule; track
@ -487,7 +529,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
ATNState target = p.transition(i).target;
ATNConfig c = new ATNConfig(target, i+1, initialContext);
Set<ATNConfig> closureBusy = new HashSet<ATNConfig>();
closure(c, configs, decState, closureBusy);
closure(c, configs, decState, closureBusy, SemanticContext.NONE, true);
}
return configs;
@ -528,7 +570,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
protected void closure(@NotNull ATNConfig config,
@NotNull OrderedHashSet<ATNConfig> configs,
@Nullable DecisionState decState,
@NotNull Set<ATNConfig> closureBusy)
@NotNull Set<ATNConfig> closureBusy,
@NotNull SemanticContext semanticContext,
boolean collectPredicates)
{
if ( debug ) System.out.println("closure("+config+")");
@ -547,7 +591,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
// gotten that context AFTER having fallen off a rule.
// Make sure we track that we are now out of context.
c.reachesIntoOuterContext = config.reachesIntoOuterContext;
closure(c, configs, decState, closureBusy);
closure(c, configs, decState, closureBusy, semanticContext, collectPredicates);
return;
}
else {
@ -569,10 +613,16 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
// optimization
if ( !p.onlyHasEpsilonTransitions() ) configs.add(config);
for (int i=0; i<p.getNumberOfTransitions(); i++) {
Transition t = p.transition(i);
boolean ignorePreds = config.traversedAction;
ATNConfig c = getEpsilonTarget(config, t, ignorePreds);
for (int i=0; i<p.getNumberOfTransitions(); i++) {
Transition t = p.transition(i);
boolean continueCollecting = !(t instanceof ActionTransition) && collectPredicates;
if ( debug ) {
if ( t instanceof ActionTransition && collectPredicates ) {
System.out.println("pruning future pred eval derived from s"+
config.state.stateNumber);
}
}
ATNConfig c = getEpsilonTarget(config, t, continueCollecting);
if ( c!=null ) {
if ( config.state instanceof RuleStopState ) {
// fell off end of rule.
@ -581,7 +631,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
// preds if this is > 0.
c.reachesIntoOuterContext++;
}
closure(c, configs, decState, closureBusy);
closure(c, configs, decState, closureBusy, semanticContext, continueCollecting);
}
}
}
@ -593,12 +643,12 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
}
@Nullable
public ATNConfig getEpsilonTarget(@NotNull ATNConfig config, @NotNull Transition t, boolean ignorePreds) {
public ATNConfig getEpsilonTarget(@NotNull ATNConfig config, @NotNull Transition t, boolean collectPredicates) {
if ( t instanceof RuleTransition ) {
return ruleTransition(config, t);
}
else if ( t instanceof PredicateTransition ) {
return predTransition(config, (PredicateTransition)t, ignorePreds);
return predTransition(config, (PredicateTransition)t, collectPredicates);
}
else if ( t instanceof ActionTransition ) {
return actionTransition(config, (ActionTransition)t);
@ -612,23 +662,17 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
@NotNull
public ATNConfig actionTransition(@NotNull ATNConfig config, @NotNull ActionTransition t) {
if ( debug ) System.out.println("ACTION edge "+t.ruleIndex+":"+t.actionIndex);
if ( debug && !config.traversedAction ) {
System.out.println("pruning future pred eval derived from s"+
config.state.stateNumber);
}
ATNConfig c = new ATNConfig(config, t.target);
c.traversedAction = true;
return c;
return new ATNConfig(config, t.target);
}
@Nullable
public ATNConfig predTransition(@NotNull ATNConfig config, @NotNull PredicateTransition pt,
boolean ignorePreds)
boolean collectPredicates)
{
if ( debug ) {
System.out.println("PRED (ignore="+ignorePreds+") "+pt.ruleIndex+":"+pt.predIndex+
", ctx dependent="+pt.isCtxDependent);
System.out.println("PRED (collectPredicates="+collectPredicates+") "+
pt.ruleIndex+":"+pt.predIndex+
", ctx dependent="+pt.isCtxDependent);
if ( parser != null ) {
System.out.println("context surrounding pred is "+
parser.getRuleInvocationStack());
@ -641,27 +685,40 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
// the outer context.
boolean inContext =
config.context==RuleContext.EMPTY && config.reachesIntoOuterContext==0;
RuleContext ctx = null;
if ( inContext ) ctx = outerContext;
// RuleContext ctx = null;
// if ( inContext ) ctx = outerContext;
ATNConfig c = null;
if ( parser==null || !collectPredicates ) {
c = new ATNConfig(config, pt.target);
}
else if ( !pt.isCtxDependent || (pt.isCtxDependent&&inContext) ) {
SemanticContext newSemCtx = new SemanticContext.AND(config.semanticContext, pt.getPredicate());
c = new ATNConfig(config, pt.target, newSemCtx);
}
return c;
// We see through the predicate if:
// 0) we have no parser to eval preds
// 1) we are ignoring them
// 2) we aren't ignoring them and it is not context dependent and
// pred is true
// 3) we aren't ignoring them, it is context dependent, but
// we know the context and pred is true
// 4) we aren't ignoring them, it is context dependent, but we don't know context
ATNConfig c = null;
boolean seeThroughPred =
ignorePreds ||
(!ignorePreds&&!pt.isCtxDependent&&parser.sempred(ctx, pt.ruleIndex, pt.predIndex))||
(!ignorePreds&&pt.isCtxDependent&&inContext&&parser.sempred(ctx, pt.ruleIndex, pt.predIndex)||
(!ignorePreds&&pt.isCtxDependent&&!inContext));
if ( seeThroughPred ) {
c = new ATNConfig(config, pt.target);
c.traversedPredicate = true;
}
return c;
// ATNConfig c = null;
// boolean seeThroughPred =
// parser==null ||
// !collectPredicates ||
// (collectPredicates&&!pt.isCtxDependent&&parser.sempred(ctx, pt.ruleIndex, pt.predIndex))||
// (collectPredicates&&pt.isCtxDependent&&inContext&&parser.sempred(ctx, pt.ruleIndex, pt.predIndex)||
// (collectPredicates&&pt.isCtxDependent&&!inContext));
// if ( seeThroughPred ) {
// c = new ATNConfig(config, pt.target, pt.getSemanticContext());
// c.traversedPredicate = true;
// }
// return c;
}
@NotNull
@ -718,7 +775,6 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
public Set<Integer> getAmbiguousAlts(@NotNull OrderedHashSet<ATNConfig> configs) {
// System.err.println("check ambiguous "+configs);
Set<Integer> ambigAlts = null;
int numConfigs = configs.size();
// First get a list of configurations for each state.
// Most of the time, each state will have one associated configuration.
MultiMap<Integer, ATNConfig> stateToConfigListMap =
@ -759,6 +815,29 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
return ambigAlts;
}
public SemanticContext[] getPredsForAmbigAlts(@NotNull DecisionState decState,
@NotNull Set<Integer> ambigAlts,
@NotNull OrderedHashSet<ATNConfig> configs)
{
// REACH=[1|1|[]|0:0, 1|2|[]|0:1]
if ( debug ) System.out.println("getPredsForAmbigAlts decision "+decState.decision);
int nalts = decState.getNumberOfTransitions();
SemanticContext[] altToPred = new SemanticContext[nalts +1];
for (int alt : ambigAlts) { altToPred[alt] = SemanticContext.NONE; }
boolean atLeastOne = false;
for (ATNConfig c : configs) {
if ( c.semanticContext!=SemanticContext.NONE && ambigAlts.contains(c.alt) ) {
altToPred[c.alt] = new SemanticContext.OR(altToPred[c.alt], c.semanticContext);
c.resolveWithPredicate = true;
atLeastOne = true;
}
}
// nonambig alts are null in altToPred
if ( !atLeastOne ) altToPred = null;
System.out.println(Arrays.toString(altToPred));
return altToPred;
}
public static int getMinAlt(@NotNull Set<Integer> ambigAlts) {
int min = Integer.MAX_VALUE;
for (int alt : ambigAlts) {
@ -783,9 +862,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
int t,
@NotNull OrderedHashSet<ATNConfig> q)
{
// System.out.println("MOVE "+p+" -> "+q+" upon "+getTokenName(t));
DFAState from = addDFAState(dfa, p);
DFAState to = addDFAState(dfa, q);
if ( debug ) System.out.println("EDGE "+from+" -> "+to+" upon "+getTokenName(t));
addDFAEdge(from, t, to);
return to;
}
@ -807,27 +886,32 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
DFAState newState = proposed;
boolean traversedPredicate = false;
for (ATNConfig c : configs) {
if ( c.traversedPredicate ) {traversedPredicate = true; break;}
}
if ( traversedPredicate ) return null; // cannot cache
// boolean traversedPredicate = false;
// for (ATNConfig c : configs) {
// if ( c.traversedPredicate ) {traversedPredicate = true; break;}
// }
//
// if ( traversedPredicate ) return null; // cannot cache
newState.stateNumber = dfa.states.size();
newState.configs = new OrderedHashSet<ATNConfig>();
newState.configs.addAll(configs);
dfa.states.put(newState, newState);
if ( debug ) System.out.println("adding new DFA state: "+newState);
return newState;
}
public void makeAcceptState(@NotNull DFA dfa, @NotNull OrderedHashSet<ATNConfig> reach, int uniqueAlt) {
DFAState accept = dfa.states.get(new DFAState(reach));
if ( accept==null ) return;
accept.isAcceptState = true;
accept.prediction = uniqueAlt;
accept.complete = true;
}
public void makeAcceptState(@NotNull DFAState accept, int uniqueAlt) {
accept.isAcceptState = true;
accept.prediction = uniqueAlt;
accept.complete = true;
}
public void makeAcceptState(@NotNull DFAState accept, @NotNull SemanticContext[] altToPred) {
accept.isAcceptState = true;
accept.complete = true;
accept.altToPred = altToPred;
}
@NotNull
public String getTokenName(int t) {

4
runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java
View File

@ -52,6 +52,10 @@ public class PredicateTransition extends Transition {
@Override
public boolean isEpsilon() { return true; }
public SemanticContext.Predicate getPredicate() {
return new SemanticContext.Predicate(ruleIndex, predIndex, isCtxDependent);
}
@Override
@NotNull
public String toString() {

515
runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java
View File

@ -29,49 +29,26 @@
package org.antlr.v4.runtime.atn;
import java.util.*;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.RuleContext;
/** A binary tree structure used to record the semantic context in which
* an NFA configuration is valid. It's either a single predicate or
* a tree representing an operation tree such as: p1&&p2 or p1||p2.
/** A tree structure used to record the semantic context in which
* an ATN configuration is valid. It's either a single predicate,
* a conjunction p1&&p2, or a sum of products p1||p2.
*
* For ATN o-p1->o-p2->o, create tree AND(p1,p2).
* For ATN (1)-p1->(2)
* | ^
* | |
* (3)-p2----
* we will have to combine p1 and p2 into DFA state as we will be
* adding NFA configurations for state 2 with two predicates p1,p2.
* So, set context for combined NFA config for state 2: OR(p1,p2).
*
* I have scoped the AND, NOT, OR, and Predicate subclasses of
* I have scoped the AND, OR, and Predicate subclasses of
* SemanticContext within the scope of this outer class.
*
* Pulled from v3; lots of fixes by Sam Harwell in v3 I notice.
*/
public abstract class SemanticContext {
/** Create a default value for the semantic context shared among all
* ATNConfigs that do not have an actual semantic context.
* This prevents lots of if!=null type checks all over; it represents
* just an empty set of predicates.
*/
public static final SemanticContext EMPTY_SEMANTIC_CONTEXT = new Predicate();
public static final SemanticContext NONE = new Predicate();
public static class Predicate extends SemanticContext {
public abstract boolean eval(Recognizer<?,?> parser, RuleContext ctx);
public static class Predicate extends SemanticContext {
public final int ruleIndex;
public final int predIndex;
public final boolean isCtxDependent; // e.g., $i ref in pred
// public static final int INVALID_PRED_VALUE = -2;
// public static final int FALSE_PRED = 0;
// public static final int TRUE_PRED = ~0;
/** sometimes predicates are known to be true or false; we need
* a way to represent this without resorting to a target language
* value like true or TRUE.
*/
// protected int constantValue = INVALID_PRED_VALUE;
protected Predicate() {
this.ruleIndex = -1;
this.predIndex = -1;
@ -90,36 +67,16 @@ public abstract class SemanticContext {
this.isCtxDependent = p.isCtxDependent;
}
public boolean equals(Object o) {
if ( !(o instanceof Predicate) ) {
return false;
}
if ( this != o ) return false;
Predicate other = (Predicate)o;
return this.ruleIndex == other.ruleIndex &&
this.predIndex == other.predIndex;
public boolean eval(Recognizer<?,?> parser, RuleContext ctx) {
return parser.sempred(ctx, ruleIndex, predIndex);
}
public int hashCode() {
return ruleIndex+predIndex;
}
public String toString() {
return ruleIndex+":"+predIndex;
}
}
public static class TruePredicate extends Predicate {
@Override
public int hashCode() {
return 1;
}
@Override
public boolean equals(Object o) {
return this == o || o instanceof TruePredicate;
}
@Override
public String toString() {
return "true"; // not used for code gen, just DOT and print outs
@ -127,415 +84,73 @@ public abstract class SemanticContext {
}
public static class FalsePredicate extends Predicate {
@Override
public int hashCode() {
return 0;
}
@Override
public boolean equals(Object o) {
return this == o || o instanceof FalsePredicate;
}
@Override
public String toString() {
return "false"; // not used for code gen, just DOT and print outs
}
}
public static abstract class CommutativePredicate extends SemanticContext {
protected final Set<SemanticContext> operands = new HashSet<SemanticContext>();
protected int hashCode;
public static class AND extends SemanticContext {
public SemanticContext a;
public SemanticContext b;
public AND() { }
public AND(SemanticContext a, SemanticContext b) {
this.a = a;
this.b = b;
}
public CommutativePredicate(SemanticContext a, SemanticContext b) {
if (a.getClass() == this.getClass()){
CommutativePredicate predicate = (CommutativePredicate)a;
operands.addAll(predicate.operands);
}
else {
operands.add(a);
}
public boolean eval(Recognizer<?,?> parser, RuleContext ctx) {
if ( a == NONE ) return b.eval(parser, ctx);
if ( b == NONE ) return a.eval(parser, ctx);
return a.eval(parser, ctx) && b.eval(parser, ctx);
}
if (b.getClass() == this.getClass()){
CommutativePredicate predicate = (CommutativePredicate)b;
operands.addAll(predicate.operands);
}
else {
operands.add(b);
}
hashCode = calculateHashCode();
}
public CommutativePredicate(HashSet<SemanticContext> contexts){
for (SemanticContext context : contexts){
if (context.getClass() == this.getClass()){
CommutativePredicate predicate = (CommutativePredicate)context;
operands.addAll(predicate.operands);
}
else {
operands.add(context);
}
}
hashCode = calculateHashCode();
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj.getClass() == this.getClass()) {
CommutativePredicate commutative = (CommutativePredicate)obj;
Set<SemanticContext> otherOperands = commutative.operands;
if (operands.size() != otherOperands.size())
return false;
return operands.containsAll(otherOperands);
}
if (obj instanceof NOT) {
NOT not = (NOT)obj;
if (not.ctx instanceof CommutativePredicate && not.ctx.getClass() != this.getClass()) {
Set<SemanticContext> otherOperands = ((CommutativePredicate)not.ctx).operands;
if (operands.size() != otherOperands.size())
return false;
ArrayList<SemanticContext> temp = new ArrayList<SemanticContext>(operands.size());
for (SemanticContext context : otherOperands) {
temp.add(not(context));
}
return operands.containsAll(temp);
}
}
return false;
}
@Override
public int hashCode(){
return hashCode;
}
@Override
public String toString() {
StringBuffer buf = new StringBuffer();
buf.append("(");
int i = 0;
for (SemanticContext semctx : operands) {
if ( i>0 ) {
buf.append(getOperandString());
}
buf.append(semctx.toString());
i++;
}
buf.append(")");
return buf.toString();
}
public abstract String getOperandString();
public abstract SemanticContext combinePredicates(SemanticContext left, SemanticContext right);
public int calculateHashCode() {
int hashcode = 0;
for (SemanticContext context : operands) {
hashcode = hashcode ^ context.hashCode();
}
return hashcode;
if ( a == NONE ) return b.toString();
if ( b == NONE ) return a.toString();
return a+"&&"+b;
}
}
public static class AND extends CommutativePredicate {
public AND(SemanticContext a, SemanticContext b) {
super(a,b);
}
public static class OR extends SemanticContext {
public SemanticContext a;
public SemanticContext b;
public OR() { }
public OR(SemanticContext a, SemanticContext b) {
this.a = a;
this.b = b;
}
public AND(HashSet<SemanticContext> contexts) {
super(contexts);
}
public boolean eval(Recognizer<?,?> parser, RuleContext ctx) {
if ( a == NONE ) return b.eval(parser, ctx);
if ( b == NONE ) return a.eval(parser, ctx);
return a.eval(parser, ctx) && b.eval(parser, ctx);
}
@Override
public String getOperandString() {
return "&&";
}
@Override
public String toString() {
if ( a == NONE ) return b.toString();
if ( b == NONE ) return a.toString();
return a+"||"+b;
}
}
@Override
public SemanticContext combinePredicates(SemanticContext left, SemanticContext right) {
return SemanticContext.and(left, right);
}
// @Override
// public int calculateHashCode() {
// int hashcode = 0;
// for (SemanticContext context : operands) {
// hashcode = hashcode ^ context.hashCode();
// }
// public static SemanticContext and(AND a, Predicate b) {
// if ( a == NONE ) return new AND(b);
// if ( b == NONE ) {a.add(b); return a;}
// a.add(b);
// AND and = new SemanticContext.AND();
// if ( a instanceof AND ) {
// and.add(b);
// }
// else {
// and.add((Predicate)a);
// and.add(b);
// }
// return and;
// }
//
// return hashcode;
// }
}
public static class OR extends CommutativePredicate {
public OR(SemanticContext a, SemanticContext b) {
super(a,b);
}
public OR(HashSet<SemanticContext> contexts) {
super(contexts);
}
@Override
public String getOperandString() {
return "||";
}
@Override
public SemanticContext combinePredicates(SemanticContext left, SemanticContext right) {
return SemanticContext.or(left, right);
}
//
// @Override
// public int calculateHashCode() {
// int hashcode = 0;
// for (SemanticContext context : operands) {
// hashcode = ~hashcode ^ context.hashCode();
// }
//
// return hashcode;
// }
}
public static class NOT extends SemanticContext {
protected SemanticContext ctx;
public NOT(SemanticContext ctx) {
this.ctx = ctx;
}
@Override
public boolean equals(Object object) {
if ( !(object instanceof NOT) ) {
return false;
}
return this.ctx.equals(((NOT)object).ctx);
}
@Override
public int hashCode() {
return ~ctx.hashCode();
}
@Override
public String toString() {
return "!("+ctx+")";
}
}
public static SemanticContext and(SemanticContext a, SemanticContext b) {
//System.out.println("AND: "+a+"&&"+b);
if (a instanceof FalsePredicate || b instanceof FalsePredicate)
return new FalsePredicate();
SemanticContext[] terms = factorOr(a, b);
SemanticContext commonTerms = terms[0];
a = terms[1];
b = terms[2];
boolean factored = commonTerms != null && commonTerms != EMPTY_SEMANTIC_CONTEXT && !(commonTerms instanceof TruePredicate);
if (factored) {
return or(commonTerms, and(a, b));
}
//System.Console.Out.WriteLine( "AND: " + a + "&&" + b );
if (a instanceof FalsePredicate || b instanceof FalsePredicate)
return new FalsePredicate();
if ( a==EMPTY_SEMANTIC_CONTEXT || a==null ) {
return b;
}
if ( b==EMPTY_SEMANTIC_CONTEXT || b==null ) {
return a;
}
if (a instanceof TruePredicate)
return b;
if (b instanceof TruePredicate)
return a;
//// Factoring takes care of this case
//if (a.Equals(b))
// return a;
//System.out.println("## have to AND");
return new AND(a,b);
}
public static SemanticContext or(SemanticContext a, SemanticContext b) {
//System.out.println("OR: "+a+"||"+b);
if (a instanceof TruePredicate || b instanceof TruePredicate)
return new TruePredicate();
SemanticContext[] terms = factorAnd(a, b);
SemanticContext commonTerms = terms[0];
a = terms[1];
b = terms[2];
boolean factored = commonTerms != null && commonTerms != EMPTY_SEMANTIC_CONTEXT && !(commonTerms instanceof FalsePredicate);
if (factored) {
return and(commonTerms, or(a, b));
}
if ( a==EMPTY_SEMANTIC_CONTEXT || a==null || a instanceof FalsePredicate ) {
return b;
}
if ( b==EMPTY_SEMANTIC_CONTEXT || b==null || b instanceof FalsePredicate ) {
return a;
}
if ( a instanceof TruePredicate || b instanceof TruePredicate || commonTerms instanceof TruePredicate ) {
return new TruePredicate();
}
//// Factoring takes care of this case
//if (a.equals(b))
// return a;
if ( a instanceof NOT ) {
NOT n = (NOT)a;
// check for !p||p
if ( n.ctx.equals(b) ) {
return new TruePredicate();
}
}
else if ( b instanceof NOT ) {
NOT n = (NOT)b;
// check for p||!p
if ( n.ctx.equals(a) ) {
return new TruePredicate();
}
}
//System.out.println("## have to OR");
OR result = new OR(a,b);
if (result.operands.size() == 1)
return result.operands.iterator().next();
return result;
}
public static SemanticContext not(SemanticContext a) {
if (a instanceof NOT) return ((NOT)a).ctx;
if (a instanceof TruePredicate) return new FalsePredicate();
else if (a instanceof FalsePredicate) return new TruePredicate();
return new NOT(a);
}
// Factor so (a && b) == (result && a && b)
public static SemanticContext[] factorAnd(SemanticContext a, SemanticContext b) {
if (a == EMPTY_SEMANTIC_CONTEXT || a == null || a instanceof FalsePredicate) {
return new SemanticContext[] { EMPTY_SEMANTIC_CONTEXT, a, b };
}
if (b == EMPTY_SEMANTIC_CONTEXT || b == null || b instanceof FalsePredicate) {
return new SemanticContext[] { EMPTY_SEMANTIC_CONTEXT, a, b };
}
if (a instanceof TruePredicate || b instanceof TruePredicate) {
return new SemanticContext[] {
new TruePredicate(), EMPTY_SEMANTIC_CONTEXT, EMPTY_SEMANTIC_CONTEXT
};
}
HashSet<SemanticContext> opsA = new HashSet<SemanticContext>(getAndOperands(a));
HashSet<SemanticContext> opsB = new HashSet<SemanticContext>(getAndOperands(b));
HashSet<SemanticContext> result = new HashSet<SemanticContext>(opsA);
result.retainAll(opsB);
if (result.size() == 0) {
return new SemanticContext[] { EMPTY_SEMANTIC_CONTEXT, a, b };
}
opsA.removeAll(result);
if (opsA.size() == 0) a = new TruePredicate();
else if (opsA.size() == 1) a = opsA.iterator().next();
else a = new AND(opsA);
opsB.removeAll(result);
if (opsB.size() == 0) b = new TruePredicate();
else if (opsB.size() == 1) b = opsB.iterator().next();
else b = new AND(opsB);
if (result.size() == 1) {
return new SemanticContext[] { result.iterator().next(), a, b };
}
return new SemanticContext[] { new AND(result), a, b };
}
// Factor so (a || b) == (result || a || b)
public static SemanticContext[] factorOr(SemanticContext a, SemanticContext b) {
HashSet<SemanticContext> opsA = new HashSet<SemanticContext>(getOrOperands(a));
HashSet<SemanticContext> opsB = new HashSet<SemanticContext>(getOrOperands(b));
HashSet<SemanticContext> result = new HashSet<SemanticContext>(opsA);
result.retainAll(opsB);
if (result.size() == 0) {
return new SemanticContext[] { EMPTY_SEMANTIC_CONTEXT, a, b };
}
opsA.removeAll(result);
if (opsA.size() == 0) a = new FalsePredicate();
else if (opsA.size() == 1) a = opsA.iterator().next();
else a = new OR(opsA);
opsB.removeAll(result);
if (opsB.size() == 0) b = new FalsePredicate();
else if (opsB.size() == 1) b = opsB.iterator().next();
else b = new OR(opsB);
if (result.size() == 1) {
return new SemanticContext[] { result.iterator().next(), a, b };
}
return new SemanticContext[] { new OR(result), a, b };
}
public static Collection<SemanticContext> getAndOperands(SemanticContext context) {
if (context instanceof AND) return ((AND)context).operands;
if (context instanceof NOT) {
Collection<SemanticContext> operands = getOrOperands(((NOT)context).ctx);
List<SemanticContext> result = new ArrayList<SemanticContext>(operands.size());
for (SemanticContext operand : operands) {
result.add(not(operand));
}
return result;
}
ArrayList<SemanticContext> result = new ArrayList<SemanticContext>();
result.add(context);
return result;
}
public static Collection<SemanticContext> getOrOperands(SemanticContext context) {
if (context instanceof OR) return ((OR)context).operands;
if (context instanceof NOT) {
Collection<SemanticContext> operands = getAndOperands(((NOT)context).ctx);
List<SemanticContext> result = new ArrayList<SemanticContext>(operands.size());
for (SemanticContext operand : operands) {
result.add(not(operand));
}
return result;
}
ArrayList<SemanticContext> result = new ArrayList<SemanticContext>();
result.add(context);
return result;
}
// public static SemanticContext or(SemanticContext a, SemanticContext b) {
// return null;
// }
}

9
runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java
View File

@ -32,6 +32,8 @@ package org.antlr.v4.runtime.dfa;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import java.util.Arrays;
/** A DFA walker that knows how to dump them to serialized strings. */
public class DFASerializer {
@NotNull
@ -77,7 +79,12 @@ public class DFASerializer {
int n = s.stateNumber;
String stateStr = "s"+n;
if ( s.isAcceptState ) {
stateStr = ":s"+n+"=>"+s.prediction;
if ( s.altToPred!=null ) {
stateStr = ":s"+n+"=>"+Arrays.toString(Arrays.copyOfRange(s.altToPred, 1, s.altToPred.length));
}
else {
stateStr = ":s"+n+"=>"+s.prediction;
}
}
if ( s.isCtxSensitive ) {
stateStr = ":s"+n+"@"+s.ctxToPrediction;

32
runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java
View File

@ -29,12 +29,16 @@
package org.antlr.v4.runtime.dfa;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.atn.ATNConfig;
import org.antlr.v4.runtime.atn.SemanticContext;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.misc.OrderedHashSet;
import java.util.*;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
/** A DFA state represents a set of possible ATN configurations.
* As Aho, Sethi, Ullman p. 117 says "The DFA uses its state
@ -49,16 +53,15 @@ import java.util.*;
* state as well, however. More importantly, we need to maintain
* a stack of states, tracking the closure operations as they
* jump from rule to rule, emulating rule invocations (method calls).
* Recall that ATNs do not normally have a stack like a pushdown-machine
* so I have to add one to simulate the proper lookahead sequences for
* I have to add a stack to simulate the proper lookahead sequences for
* the underlying LL grammar from which the ATN was derived.
*
* I use a list of ATNConfig objects. An ATNConfig
* I use a set of ATNConfig objects not simple states. An ATNConfig
* is both a state (ala normal conversion) and a RuleContext describing
* the chain of rules (if any) followed to arrive at that state.
*
* A DFA state may have multiple references to a particular state,
* but with different ATNContexts (with same or different alts)
* but with different ATN contexts (with same or different alts)
* meaning that state was reached via a different set of rule invocations.
*/
public class DFAState {
@ -84,7 +87,9 @@ public class DFAState {
public boolean isCtxSensitive;
@Nullable
public Map<RuleContext, Integer> ctxToPrediction;
public Map<RuleContext, Integer> ctxToPrediction; // used for ctx sensitive parsing
public SemanticContext[] altToPred;
public DFAState() { }
@ -150,6 +155,17 @@ public class DFAState {
@Override
public String toString() {
return stateNumber+":"+configs+(isAcceptState?("=>"+prediction):"");
StringBuilder buf = new StringBuilder();
buf.append(stateNumber + ":" + configs);
if ( isAcceptState ) {
buf.append("=>");
if ( altToPred!=null ) {
buf.append(Arrays.toString(Arrays.copyOfRange(altToPred, 1, altToPred.length)));
}
else {
buf.append(prediction);
}
}
return buf.toString();
}
}

72
tool/test/org/antlr/v4/test/TestSemPredEvalParser.java
View File

@ -3,45 +3,45 @@ package org.antlr.v4.test;
import org.junit.Test;
public class TestSemPredEvalParser extends BaseTest {
@Test public void testToLeft() throws Exception {
String grammar =
"grammar T;\n" +
"s : a+ ;\n" +
"a : {false}? ID {System.out.println(\"alt 1\");}\n" +
" | {true}? ID {System.out.println(\"alt 2\");}\n" +
" ;\n" +
"ID : 'a'..'z'+ ;\n" +
"INT : '0'..'9'+;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
@Test public void testSimple() throws Exception {
String grammar =
"grammar T;\n" +
"a : {false}? ID {System.out.println(\"alt 1\");}\n" +
" | {true}? ID {System.out.println(\"alt 2\");}\n" +
" | INT {System.out.println(\"alt 3\");}\n" +
" ;\n" +
"ID : 'a'..'z'+ ;\n" +
"INT : '0'..'9'+;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
String found = execParser("T.g", grammar, "TParser", "TLexer", "s",
"x x y", false);
String expecting =
"alt 2\n" +
"alt 2\n" +
"alt 2\n";
assertEquals(expecting, found);
}
String found = execParser("T.g", grammar, "TParser", "TLexer", "a",
"x", false);
String expecting =
"alt 2\n" +
"alt 2\n" +
"alt 2\n";
assertEquals(expecting, found);
}
@Test public void testToRight() throws Exception {
String grammar =
"grammar T;\n" +
"s : a+ ;\n" +
"a : ID {false}? {System.out.println(\"alt 1\");}\n" +
" | ID {true}? {System.out.println(\"alt 2\");}\n" +
" ;\n" +
"ID : 'a'..'z'+ ;\n" +
"INT : '0'..'9'+;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
@Test public void testToLeft() throws Exception {
String grammar =
"grammar T;\n" +
"s : a+ ;\n" +
"a : {false}? ID {System.out.println(\"alt 1\");}\n" +
" | {true}? ID {System.out.println(\"alt 2\");}\n" +
" ;\n" +
"ID : 'a'..'z'+ ;\n" +
"INT : '0'..'9'+;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
String found = execParser("T.g", grammar, "TParser", "TLexer", "s",
"x x y", false);
String expecting =
"alt 2\n" +
"alt 2\n" +
"alt 2\n";
assertEquals(expecting, found);
}
String found = execParser("T.g", grammar, "TParser", "TLexer", "s",
"x x y", false);
String expecting =
"alt 2\n" +
"alt 2\n" +
"alt 2\n";
assertEquals(expecting, found);
}
@Test public void testActionHidesPreds() throws Exception {
// can't see preds, resolves to first alt found (1 in this case)