diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java index f4fb8dd99..2c60c6647 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java @@ -29,9 +29,10 @@ package org.antlr.v4.runtime.atn; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.Nullable; -import org.antlr.v4.runtime.*; /** An ATN state, predicted alt, and syntactic/semantic context. * The syntactic context is a pointer into the rule invocation @@ -61,16 +62,7 @@ public class ATNConfig { * otherwise predicates would not get executed again (DFAs don't * have predicated edges in v4). */ - public boolean traversedPredicate; - - /** - * Indicates that we have reached this ATN configuration after - * traversing a non-forced action transition. We do not execute - * predicates after such actions because the predicates could be - * functions of the side effects. Force actions must be either side - * effect free or automatically undone as the parse continues. - */ - public boolean traversedAction; + public boolean traversedPredicate; // TODO: don't need /** * We cannot execute predicates dependent upon local context unless @@ -85,6 +77,18 @@ public class ATNConfig { */ public int reachesIntoOuterContext; + @NotNull + public SemanticContext semanticContext = SemanticContext.NONE; + + /** This bit is used to indicate a semantic predicate will be + * used to resolve the conflict. Essentially, this is used + * as an "ignore" bit so that upon a set of conflicting configurations, + * such as (s|2|p) and (s|3|q), I can set (s|3) to resolved=true (and any + * other configuration associated with alt 3) to make it look like that set + * uniquely predicts an alt. + */ + protected boolean resolveWithPredicate; + public ATNConfig(@NotNull ATNState state, int alt, @Nullable RuleContext context) @@ -94,26 +98,36 @@ public class ATNConfig { this.context = context; } - public ATNConfig(@NotNull ATNConfig c) { - this(c, c.state, c.context); - } +// public ATNConfig(@NotNull ATNConfig c) { +// this(c, c.state, c.context, c.semanticContext); +// } - public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state) { - this(c, state, c.context); - } + public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state) { + this(c, state, c.context, c.semanticContext); + } - public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, @Nullable RuleContext context) { + public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, SemanticContext semanticContext) { + this(c, state, c.context, semanticContext); + } + + public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, @Nullable RuleContext context) { + this(c, state, context, c.semanticContext); + } + + public ATNConfig(@NotNull ATNConfig c, @NotNull ATNState state, @Nullable RuleContext context, + SemanticContext semanticContext) + { this.state = state; this.alt = c.alt; this.context = context; this.traversedPredicate = c.traversedPredicate; - this.traversedAction = c.traversedAction; this.reachesIntoOuterContext = c.reachesIntoOuterContext; + this.semanticContext = semanticContext; } - public ATNConfig(@NotNull ATNConfig c, @Nullable RuleContext context) { - this(c, c.state, context); - } +// public ATNConfig(@NotNull ATNConfig c, @Nullable RuleContext context) { +// this(c, c.state, context); +// } /** An ATN configuration is equal to another if both have * the same state, they predict the same alternative, and @@ -153,16 +167,20 @@ public class ATNConfig { // } buf.append(state); if ( showAlt ) { - buf.append("|"); - buf.append(alt); - } - if ( context!=null ) { + buf.append("|"); + buf.append(alt); + } + if ( context!=null ) { buf.append("|"); buf.append(context); } - if ( reachesIntoOuterContext>0 ) { - buf.append("|up=").append(reachesIntoOuterContext); - } + if ( semanticContext!=null ) { + buf.append("|"); + buf.append(semanticContext); + } + if ( reachesIntoOuterContext>0 ) { + buf.append("|up=").append(reachesIntoOuterContext); + } // if (isAccept) { // buf.append("|=>"+alt); // } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 95ed9079e..83ef621b8 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -43,8 +43,8 @@ import org.stringtemplate.v4.misc.MultiMap; import java.util.*; public class ParserATNSimulator extends ATNSimulator { - public static boolean debug = false; - public static boolean dfa_debug = false; + public static boolean debug = true; + public static boolean dfa_debug = true; public static int ATN_failover = 0; public static int predict_calls = 0; @@ -161,7 +161,9 @@ public class ParserATNSimulator extends ATNSimulator { return execATN(input, dfa, input.index(), s0_closure, false); } - public int execDFA(@NotNull SymbolStream input, @NotNull DFA dfa, @NotNull DFAState s0, @Nullable RuleContext outerContext) { + public int execDFA(@NotNull SymbolStream input, @NotNull DFA dfa, @NotNull DFAState s0, + @Nullable RuleContext outerContext) + { // dump(dfa); if ( outerContext==null ) outerContext = RuleContext.EMPTY; this.outerContext = outerContext; @@ -278,8 +280,9 @@ public class ParserATNSimulator extends ATNSimulator { return prevAccept.alt; } - DecisionState decState = null; - if ( atn.decisionToState.size()>0 ) decState = atn.decisionToState.get(dfa.decision); + @NotNull DecisionState decState = null; + //if ( atn.decisionToState.size()>0 ) + decState = atn.decisionToState.get(dfa.decision); if ( debug ) System.out.println("decision state = "+decState); prevAccept = null; @@ -298,7 +301,8 @@ public class ParserATNSimulator extends ATNSimulator { ATNState target = getReachableTarget(trans, t); if ( target!=null ) { Set closureBusy = new HashSet(); - closure(new ATNConfig(c, target), reach, decState, closureBusy); + closure(new ATNConfig(c, target), reach, decState, closureBusy, + c.semanticContext, false); } } } @@ -316,6 +320,17 @@ public class ParserATNSimulator extends ATNSimulator { getInputString(input, startIndex)); System.out.println("REACH="+reach); } + + // can we resolve with predicates? + // TODO: warn if we have uncovered alts? + SemanticContext[] altToPred = getPredsForAmbigAlts(decState, ambigAlts, reach); + if ( altToPred!=null ) { + int uniqueAlt = evalSemanticContext(ambigAlts, altToPred); + DFAState accept = addDFAEdge(dfa, closure, t, reach); + makeAcceptState(accept, altToPred); + return uniqueAlt; + } + dfa.conflict = true; // at least one DFA state is ambiguous if ( !userWantsCtxSensitive ) reportConflict(startIndex, input.index(), ambigAlts, reach); @@ -348,8 +363,8 @@ public class ParserATNSimulator extends ATNSimulator { if ( debug ) System.out.println("PREDICT alt "+uniqueAlt+ " decision "+dfa.decision+ " at index "+input.index()); - addDFAEdge(dfa, closure, t, reach); - makeAcceptState(dfa, reach, uniqueAlt); + DFAState accept = addDFAEdge(dfa, closure, t, reach); + makeAcceptState(accept, uniqueAlt); return uniqueAlt; } @@ -391,7 +406,32 @@ public class ParserATNSimulator extends ATNSimulator { return prevAccept.alt; } - protected int resolveToMinAlt(@NotNull OrderedHashSet reach, @NotNull Set ambigAlts) { + // eval preds first in alt order for ambigAlts; + // if all false, then pick first unpredicated alt in ambigAlts + public int evalSemanticContext(Set ambigAlts, SemanticContext[] altToPred) { + int uniqueAlt = ATN.INVALID_ALT_NUMBER; + int firstUnpredicatedAmbigAlt = ATN.INVALID_ALT_NUMBER; + for (int i = 1; i < altToPred.length; i++) { + SemanticContext or = altToPred[i]; + if ( or==null || or==SemanticContext.NONE ) { + if ( ambigAlts.contains(i) ) firstUnpredicatedAmbigAlt = i; + continue; + } + System.out.println("eval "+or+"="+or.eval(parser, outerContext)); + if ( or.eval(parser, outerContext) ) { + System.out.println("PREDICT "+i); + uniqueAlt = i; + break; + } + } + if ( uniqueAlt==ATN.INVALID_ALT_NUMBER ) { + System.out.println("PREDICT firstUnpredicatedAmbigAlt "+firstUnpredicatedAmbigAlt); + uniqueAlt = firstUnpredicatedAmbigAlt; + } + return uniqueAlt; + } + + protected int resolveToMinAlt(@NotNull OrderedHashSet reach, @NotNull Set ambigAlts) { int min = getMinAlt(ambigAlts); // create DFA accept state for resolved alt ambigAlts.remove(min); @@ -474,7 +514,9 @@ public class ParserATNSimulator extends ATNSimulator { } @NotNull - public OrderedHashSet computeStartState(int decision, @NotNull ATNState p, @Nullable RuleContext ctx) { + public OrderedHashSet computeStartState(int decision, @NotNull ATNState p, + @Nullable RuleContext ctx) + { RuleContext initialContext = ctx; // always at least the implicit call to start rule OrderedHashSet configs = new OrderedHashSet(); prevAccept = null; // might reach end rule; track @@ -487,7 +529,7 @@ public class ParserATNSimulator extends ATNSimulator { ATNState target = p.transition(i).target; ATNConfig c = new ATNConfig(target, i+1, initialContext); Set closureBusy = new HashSet(); - closure(c, configs, decState, closureBusy); + closure(c, configs, decState, closureBusy, SemanticContext.NONE, true); } return configs; @@ -528,7 +570,9 @@ public class ParserATNSimulator extends ATNSimulator { protected void closure(@NotNull ATNConfig config, @NotNull OrderedHashSet configs, @Nullable DecisionState decState, - @NotNull Set closureBusy) + @NotNull Set closureBusy, + @NotNull SemanticContext semanticContext, + boolean collectPredicates) { if ( debug ) System.out.println("closure("+config+")"); @@ -547,7 +591,7 @@ public class ParserATNSimulator extends ATNSimulator { // gotten that context AFTER having fallen off a rule. // Make sure we track that we are now out of context. c.reachesIntoOuterContext = config.reachesIntoOuterContext; - closure(c, configs, decState, closureBusy); + closure(c, configs, decState, closureBusy, semanticContext, collectPredicates); return; } else { @@ -569,10 +613,16 @@ public class ParserATNSimulator extends ATNSimulator { // optimization if ( !p.onlyHasEpsilonTransitions() ) configs.add(config); - for (int i=0; i extends ATNSimulator { // preds if this is > 0. c.reachesIntoOuterContext++; } - closure(c, configs, decState, closureBusy); + closure(c, configs, decState, closureBusy, semanticContext, continueCollecting); } } } @@ -593,12 +643,12 @@ public class ParserATNSimulator extends ATNSimulator { } @Nullable - public ATNConfig getEpsilonTarget(@NotNull ATNConfig config, @NotNull Transition t, boolean ignorePreds) { + public ATNConfig getEpsilonTarget(@NotNull ATNConfig config, @NotNull Transition t, boolean collectPredicates) { if ( t instanceof RuleTransition ) { return ruleTransition(config, t); } else if ( t instanceof PredicateTransition ) { - return predTransition(config, (PredicateTransition)t, ignorePreds); + return predTransition(config, (PredicateTransition)t, collectPredicates); } else if ( t instanceof ActionTransition ) { return actionTransition(config, (ActionTransition)t); @@ -612,23 +662,17 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull public ATNConfig actionTransition(@NotNull ATNConfig config, @NotNull ActionTransition t) { if ( debug ) System.out.println("ACTION edge "+t.ruleIndex+":"+t.actionIndex); - if ( debug && !config.traversedAction ) { - System.out.println("pruning future pred eval derived from s"+ - config.state.stateNumber); - } - - ATNConfig c = new ATNConfig(config, t.target); - c.traversedAction = true; - return c; + return new ATNConfig(config, t.target); } @Nullable public ATNConfig predTransition(@NotNull ATNConfig config, @NotNull PredicateTransition pt, - boolean ignorePreds) + boolean collectPredicates) { if ( debug ) { - System.out.println("PRED (ignore="+ignorePreds+") "+pt.ruleIndex+":"+pt.predIndex+ - ", ctx dependent="+pt.isCtxDependent); + System.out.println("PRED (collectPredicates="+collectPredicates+") "+ + pt.ruleIndex+":"+pt.predIndex+ + ", ctx dependent="+pt.isCtxDependent); if ( parser != null ) { System.out.println("context surrounding pred is "+ parser.getRuleInvocationStack()); @@ -641,27 +685,40 @@ public class ParserATNSimulator extends ATNSimulator { // the outer context. boolean inContext = config.context==RuleContext.EMPTY && config.reachesIntoOuterContext==0; - RuleContext ctx = null; - if ( inContext ) ctx = outerContext; +// RuleContext ctx = null; +// if ( inContext ) ctx = outerContext; + + ATNConfig c = null; + if ( parser==null || !collectPredicates ) { + c = new ATNConfig(config, pt.target); + } + else if ( !pt.isCtxDependent || (pt.isCtxDependent&&inContext) ) { + SemanticContext newSemCtx = new SemanticContext.AND(config.semanticContext, pt.getPredicate()); + c = new ATNConfig(config, pt.target, newSemCtx); + } + + return c; // We see through the predicate if: + // 0) we have no parser to eval preds // 1) we are ignoring them // 2) we aren't ignoring them and it is not context dependent and // pred is true // 3) we aren't ignoring them, it is context dependent, but // we know the context and pred is true // 4) we aren't ignoring them, it is context dependent, but we don't know context - ATNConfig c = null; - boolean seeThroughPred = - ignorePreds || - (!ignorePreds&&!pt.isCtxDependent&&parser.sempred(ctx, pt.ruleIndex, pt.predIndex))|| - (!ignorePreds&&pt.isCtxDependent&&inContext&&parser.sempred(ctx, pt.ruleIndex, pt.predIndex)|| - (!ignorePreds&&pt.isCtxDependent&&!inContext)); - if ( seeThroughPred ) { - c = new ATNConfig(config, pt.target); - c.traversedPredicate = true; - } - return c; +// ATNConfig c = null; +// boolean seeThroughPred = +// parser==null || +// !collectPredicates || +// (collectPredicates&&!pt.isCtxDependent&&parser.sempred(ctx, pt.ruleIndex, pt.predIndex))|| +// (collectPredicates&&pt.isCtxDependent&&inContext&&parser.sempred(ctx, pt.ruleIndex, pt.predIndex)|| +// (collectPredicates&&pt.isCtxDependent&&!inContext)); +// if ( seeThroughPred ) { +// c = new ATNConfig(config, pt.target, pt.getSemanticContext()); +// c.traversedPredicate = true; +// } +// return c; } @NotNull @@ -718,7 +775,6 @@ public class ParserATNSimulator extends ATNSimulator { public Set getAmbiguousAlts(@NotNull OrderedHashSet configs) { // System.err.println("check ambiguous "+configs); Set ambigAlts = null; - int numConfigs = configs.size(); // First get a list of configurations for each state. // Most of the time, each state will have one associated configuration. MultiMap stateToConfigListMap = @@ -759,6 +815,29 @@ public class ParserATNSimulator extends ATNSimulator { return ambigAlts; } + public SemanticContext[] getPredsForAmbigAlts(@NotNull DecisionState decState, + @NotNull Set ambigAlts, + @NotNull OrderedHashSet configs) + { + // REACH=[1|1|[]|0:0, 1|2|[]|0:1] + if ( debug ) System.out.println("getPredsForAmbigAlts decision "+decState.decision); + int nalts = decState.getNumberOfTransitions(); + SemanticContext[] altToPred = new SemanticContext[nalts +1]; + for (int alt : ambigAlts) { altToPred[alt] = SemanticContext.NONE; } + boolean atLeastOne = false; + for (ATNConfig c : configs) { + if ( c.semanticContext!=SemanticContext.NONE && ambigAlts.contains(c.alt) ) { + altToPred[c.alt] = new SemanticContext.OR(altToPred[c.alt], c.semanticContext); + c.resolveWithPredicate = true; + atLeastOne = true; + } + } + // nonambig alts are null in altToPred + if ( !atLeastOne ) altToPred = null; + System.out.println(Arrays.toString(altToPred)); + return altToPred; + } + public static int getMinAlt(@NotNull Set ambigAlts) { int min = Integer.MAX_VALUE; for (int alt : ambigAlts) { @@ -783,9 +862,9 @@ public class ParserATNSimulator extends ATNSimulator { int t, @NotNull OrderedHashSet q) { -// System.out.println("MOVE "+p+" -> "+q+" upon "+getTokenName(t)); DFAState from = addDFAState(dfa, p); DFAState to = addDFAState(dfa, q); + if ( debug ) System.out.println("EDGE "+from+" -> "+to+" upon "+getTokenName(t)); addDFAEdge(from, t, to); return to; } @@ -807,27 +886,32 @@ public class ParserATNSimulator extends ATNSimulator { DFAState newState = proposed; - boolean traversedPredicate = false; - for (ATNConfig c : configs) { - if ( c.traversedPredicate ) {traversedPredicate = true; break;} - } - - if ( traversedPredicate ) return null; // cannot cache +// boolean traversedPredicate = false; +// for (ATNConfig c : configs) { +// if ( c.traversedPredicate ) {traversedPredicate = true; break;} +// } +// +// if ( traversedPredicate ) return null; // cannot cache newState.stateNumber = dfa.states.size(); newState.configs = new OrderedHashSet(); newState.configs.addAll(configs); dfa.states.put(newState, newState); + if ( debug ) System.out.println("adding new DFA state: "+newState); return newState; } - public void makeAcceptState(@NotNull DFA dfa, @NotNull OrderedHashSet reach, int uniqueAlt) { - DFAState accept = dfa.states.get(new DFAState(reach)); - if ( accept==null ) return; - accept.isAcceptState = true; - accept.prediction = uniqueAlt; - accept.complete = true; - } + public void makeAcceptState(@NotNull DFAState accept, int uniqueAlt) { + accept.isAcceptState = true; + accept.prediction = uniqueAlt; + accept.complete = true; + } + + public void makeAcceptState(@NotNull DFAState accept, @NotNull SemanticContext[] altToPred) { + accept.isAcceptState = true; + accept.complete = true; + accept.altToPred = altToPred; + } @NotNull public String getTokenName(int t) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java index ebe7f0095..9cc54d4c7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java @@ -52,6 +52,10 @@ public class PredicateTransition extends Transition { @Override public boolean isEpsilon() { return true; } + public SemanticContext.Predicate getPredicate() { + return new SemanticContext.Predicate(ruleIndex, predIndex, isCtxDependent); + } + @Override @NotNull public String toString() { diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java index 122c8ff20..47c17fd9e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java @@ -29,49 +29,26 @@ package org.antlr.v4.runtime.atn; -import java.util.*; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.RuleContext; -/** A binary tree structure used to record the semantic context in which - * an NFA configuration is valid. It's either a single predicate or - * a tree representing an operation tree such as: p1&&p2 or p1||p2. +/** A tree structure used to record the semantic context in which + * an ATN configuration is valid. It's either a single predicate, + * a conjunction p1&&p2, or a sum of products p1||p2. * - * For ATN o-p1->o-p2->o, create tree AND(p1,p2). - * For ATN (1)-p1->(2) - * | ^ - * | | - * (3)-p2---- - * we will have to combine p1 and p2 into DFA state as we will be - * adding NFA configurations for state 2 with two predicates p1,p2. - * So, set context for combined NFA config for state 2: OR(p1,p2). - * - * I have scoped the AND, NOT, OR, and Predicate subclasses of + * I have scoped the AND, OR, and Predicate subclasses of * SemanticContext within the scope of this outer class. - * - * Pulled from v3; lots of fixes by Sam Harwell in v3 I notice. */ public abstract class SemanticContext { - /** Create a default value for the semantic context shared among all - * ATNConfigs that do not have an actual semantic context. - * This prevents lots of if!=null type checks all over; it represents - * just an empty set of predicates. - */ - public static final SemanticContext EMPTY_SEMANTIC_CONTEXT = new Predicate(); + public static final SemanticContext NONE = new Predicate(); - public static class Predicate extends SemanticContext { + public abstract boolean eval(Recognizer parser, RuleContext ctx); + + public static class Predicate extends SemanticContext { public final int ruleIndex; public final int predIndex; public final boolean isCtxDependent; // e.g., $i ref in pred -// public static final int INVALID_PRED_VALUE = -2; -// public static final int FALSE_PRED = 0; -// public static final int TRUE_PRED = ~0; - - /** sometimes predicates are known to be true or false; we need - * a way to represent this without resorting to a target language - * value like true or TRUE. - */ -// protected int constantValue = INVALID_PRED_VALUE; - protected Predicate() { this.ruleIndex = -1; this.predIndex = -1; @@ -90,36 +67,16 @@ public abstract class SemanticContext { this.isCtxDependent = p.isCtxDependent; } - public boolean equals(Object o) { - if ( !(o instanceof Predicate) ) { - return false; - } - - if ( this != o ) return false; - - Predicate other = (Predicate)o; - return this.ruleIndex == other.ruleIndex && - this.predIndex == other.predIndex; + public boolean eval(Recognizer parser, RuleContext ctx) { + return parser.sempred(ctx, ruleIndex, predIndex); } - public int hashCode() { - return ruleIndex+predIndex; - } public String toString() { return ruleIndex+":"+predIndex; } } public static class TruePredicate extends Predicate { - @Override - public int hashCode() { - return 1; - } - - @Override - public boolean equals(Object o) { - return this == o || o instanceof TruePredicate; - } @Override public String toString() { return "true"; // not used for code gen, just DOT and print outs @@ -127,415 +84,73 @@ public abstract class SemanticContext { } public static class FalsePredicate extends Predicate { - @Override - public int hashCode() { - return 0; - } - - @Override - public boolean equals(Object o) { - return this == o || o instanceof FalsePredicate; - } - @Override public String toString() { return "false"; // not used for code gen, just DOT and print outs } } - public static abstract class CommutativePredicate extends SemanticContext { - protected final Set operands = new HashSet(); - protected int hashCode; + public static class AND extends SemanticContext { + public SemanticContext a; + public SemanticContext b; + public AND() { } + public AND(SemanticContext a, SemanticContext b) { + this.a = a; + this.b = b; + } - public CommutativePredicate(SemanticContext a, SemanticContext b) { - if (a.getClass() == this.getClass()){ - CommutativePredicate predicate = (CommutativePredicate)a; - operands.addAll(predicate.operands); - } - else { - operands.add(a); - } + public boolean eval(Recognizer parser, RuleContext ctx) { + if ( a == NONE ) return b.eval(parser, ctx); + if ( b == NONE ) return a.eval(parser, ctx); + return a.eval(parser, ctx) && b.eval(parser, ctx); + } - if (b.getClass() == this.getClass()){ - CommutativePredicate predicate = (CommutativePredicate)b; - operands.addAll(predicate.operands); - } - else { - operands.add(b); - } - - hashCode = calculateHashCode(); - } - - public CommutativePredicate(HashSet contexts){ - for (SemanticContext context : contexts){ - if (context.getClass() == this.getClass()){ - CommutativePredicate predicate = (CommutativePredicate)context; - operands.addAll(predicate.operands); - } - else { - operands.add(context); - } - } - - hashCode = calculateHashCode(); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - - if (obj.getClass() == this.getClass()) { - CommutativePredicate commutative = (CommutativePredicate)obj; - Set otherOperands = commutative.operands; - if (operands.size() != otherOperands.size()) - return false; - - return operands.containsAll(otherOperands); - } - - if (obj instanceof NOT) { - NOT not = (NOT)obj; - if (not.ctx instanceof CommutativePredicate && not.ctx.getClass() != this.getClass()) { - Set otherOperands = ((CommutativePredicate)not.ctx).operands; - if (operands.size() != otherOperands.size()) - return false; - - ArrayList temp = new ArrayList(operands.size()); - for (SemanticContext context : otherOperands) { - temp.add(not(context)); - } - - return operands.containsAll(temp); - } - } - - return false; - } - - @Override - public int hashCode(){ - return hashCode; - } - - @Override public String toString() { - StringBuffer buf = new StringBuffer(); - buf.append("("); - int i = 0; - for (SemanticContext semctx : operands) { - if ( i>0 ) { - buf.append(getOperandString()); - } - buf.append(semctx.toString()); - i++; - } - buf.append(")"); - return buf.toString(); - } - - public abstract String getOperandString(); - - public abstract SemanticContext combinePredicates(SemanticContext left, SemanticContext right); - - public int calculateHashCode() { - int hashcode = 0; - for (SemanticContext context : operands) { - hashcode = hashcode ^ context.hashCode(); - } - - return hashcode; + if ( a == NONE ) return b.toString(); + if ( b == NONE ) return a.toString(); + return a+"&&"+b; } } - public static class AND extends CommutativePredicate { - public AND(SemanticContext a, SemanticContext b) { - super(a,b); - } + public static class OR extends SemanticContext { + public SemanticContext a; + public SemanticContext b; + public OR() { } + public OR(SemanticContext a, SemanticContext b) { + this.a = a; + this.b = b; + } - public AND(HashSet contexts) { - super(contexts); - } + public boolean eval(Recognizer parser, RuleContext ctx) { + if ( a == NONE ) return b.eval(parser, ctx); + if ( b == NONE ) return a.eval(parser, ctx); + return a.eval(parser, ctx) && b.eval(parser, ctx); + } - @Override - public String getOperandString() { - return "&&"; - } + @Override + public String toString() { + if ( a == NONE ) return b.toString(); + if ( b == NONE ) return a.toString(); + return a+"||"+b; + } + } - @Override - public SemanticContext combinePredicates(SemanticContext left, SemanticContext right) { - return SemanticContext.and(left, right); - } - -// @Override -// public int calculateHashCode() { -// int hashcode = 0; -// for (SemanticContext context : operands) { -// hashcode = hashcode ^ context.hashCode(); -// } +// public static SemanticContext and(AND a, Predicate b) { +// if ( a == NONE ) return new AND(b); +// if ( b == NONE ) {a.add(b); return a;} +// a.add(b); +// AND and = new SemanticContext.AND(); +// if ( a instanceof AND ) { +// and.add(b); +// } +// else { +// and.add((Predicate)a); +// and.add(b); +// } +// return and; +// } // -// return hashcode; -// } - } - - public static class OR extends CommutativePredicate { - public OR(SemanticContext a, SemanticContext b) { - super(a,b); - } - - public OR(HashSet contexts) { - super(contexts); - } - - @Override - public String getOperandString() { - return "||"; - } - - @Override - public SemanticContext combinePredicates(SemanticContext left, SemanticContext right) { - return SemanticContext.or(left, right); - } -// -// @Override -// public int calculateHashCode() { -// int hashcode = 0; -// for (SemanticContext context : operands) { -// hashcode = ~hashcode ^ context.hashCode(); -// } -// -// return hashcode; -// } - } - - public static class NOT extends SemanticContext { - protected SemanticContext ctx; - public NOT(SemanticContext ctx) { - this.ctx = ctx; - } - - @Override - public boolean equals(Object object) { - if ( !(object instanceof NOT) ) { - return false; - } - return this.ctx.equals(((NOT)object).ctx); - } - - @Override - public int hashCode() { - return ~ctx.hashCode(); - } - - @Override - public String toString() { - return "!("+ctx+")"; - } - } - - public static SemanticContext and(SemanticContext a, SemanticContext b) { - //System.out.println("AND: "+a+"&&"+b); - if (a instanceof FalsePredicate || b instanceof FalsePredicate) - return new FalsePredicate(); - - SemanticContext[] terms = factorOr(a, b); - SemanticContext commonTerms = terms[0]; - a = terms[1]; - b = terms[2]; - - boolean factored = commonTerms != null && commonTerms != EMPTY_SEMANTIC_CONTEXT && !(commonTerms instanceof TruePredicate); - if (factored) { - return or(commonTerms, and(a, b)); - } - - //System.Console.Out.WriteLine( "AND: " + a + "&&" + b ); - if (a instanceof FalsePredicate || b instanceof FalsePredicate) - return new FalsePredicate(); - - if ( a==EMPTY_SEMANTIC_CONTEXT || a==null ) { - return b; - } - if ( b==EMPTY_SEMANTIC_CONTEXT || b==null ) { - return a; - } - - if (a instanceof TruePredicate) - return b; - - if (b instanceof TruePredicate) - return a; - - //// Factoring takes care of this case - //if (a.Equals(b)) - // return a; - - //System.out.println("## have to AND"); - return new AND(a,b); - } - - public static SemanticContext or(SemanticContext a, SemanticContext b) { - //System.out.println("OR: "+a+"||"+b); - if (a instanceof TruePredicate || b instanceof TruePredicate) - return new TruePredicate(); - - SemanticContext[] terms = factorAnd(a, b); - SemanticContext commonTerms = terms[0]; - a = terms[1]; - b = terms[2]; - boolean factored = commonTerms != null && commonTerms != EMPTY_SEMANTIC_CONTEXT && !(commonTerms instanceof FalsePredicate); - if (factored) { - return and(commonTerms, or(a, b)); - } - - if ( a==EMPTY_SEMANTIC_CONTEXT || a==null || a instanceof FalsePredicate ) { - return b; - } - - if ( b==EMPTY_SEMANTIC_CONTEXT || b==null || b instanceof FalsePredicate ) { - return a; - } - - if ( a instanceof TruePredicate || b instanceof TruePredicate || commonTerms instanceof TruePredicate ) { - return new TruePredicate(); - } - - //// Factoring takes care of this case - //if (a.equals(b)) - // return a; - - if ( a instanceof NOT ) { - NOT n = (NOT)a; - // check for !p||p - if ( n.ctx.equals(b) ) { - return new TruePredicate(); - } - } - else if ( b instanceof NOT ) { - NOT n = (NOT)b; - // check for p||!p - if ( n.ctx.equals(a) ) { - return new TruePredicate(); - } - } - - //System.out.println("## have to OR"); - OR result = new OR(a,b); - if (result.operands.size() == 1) - return result.operands.iterator().next(); - - return result; - } - - public static SemanticContext not(SemanticContext a) { - if (a instanceof NOT) return ((NOT)a).ctx; - - if (a instanceof TruePredicate) return new FalsePredicate(); - else if (a instanceof FalsePredicate) return new TruePredicate(); - - return new NOT(a); - } - - // Factor so (a && b) == (result && a && b) - public static SemanticContext[] factorAnd(SemanticContext a, SemanticContext b) { - if (a == EMPTY_SEMANTIC_CONTEXT || a == null || a instanceof FalsePredicate) { - return new SemanticContext[] { EMPTY_SEMANTIC_CONTEXT, a, b }; - } - if (b == EMPTY_SEMANTIC_CONTEXT || b == null || b instanceof FalsePredicate) { - return new SemanticContext[] { EMPTY_SEMANTIC_CONTEXT, a, b }; - } - - if (a instanceof TruePredicate || b instanceof TruePredicate) { - return new SemanticContext[] { - new TruePredicate(), EMPTY_SEMANTIC_CONTEXT, EMPTY_SEMANTIC_CONTEXT - }; - } - - HashSet opsA = new HashSet(getAndOperands(a)); - HashSet opsB = new HashSet(getAndOperands(b)); - - HashSet result = new HashSet(opsA); - result.retainAll(opsB); - if (result.size() == 0) { - return new SemanticContext[] { EMPTY_SEMANTIC_CONTEXT, a, b }; - } - - opsA.removeAll(result); - if (opsA.size() == 0) a = new TruePredicate(); - else if (opsA.size() == 1) a = opsA.iterator().next(); - else a = new AND(opsA); - - opsB.removeAll(result); - if (opsB.size() == 0) b = new TruePredicate(); - else if (opsB.size() == 1) b = opsB.iterator().next(); - else b = new AND(opsB); - - if (result.size() == 1) { - return new SemanticContext[] { result.iterator().next(), a, b }; - } - - return new SemanticContext[] { new AND(result), a, b }; - } - - // Factor so (a || b) == (result || a || b) - public static SemanticContext[] factorOr(SemanticContext a, SemanticContext b) { - HashSet opsA = new HashSet(getOrOperands(a)); - HashSet opsB = new HashSet(getOrOperands(b)); - - HashSet result = new HashSet(opsA); - result.retainAll(opsB); - if (result.size() == 0) { - return new SemanticContext[] { EMPTY_SEMANTIC_CONTEXT, a, b }; - } - - opsA.removeAll(result); - if (opsA.size() == 0) a = new FalsePredicate(); - else if (opsA.size() == 1) a = opsA.iterator().next(); - else a = new OR(opsA); - - opsB.removeAll(result); - if (opsB.size() == 0) b = new FalsePredicate(); - else if (opsB.size() == 1) b = opsB.iterator().next(); - else b = new OR(opsB); - - if (result.size() == 1) { - return new SemanticContext[] { result.iterator().next(), a, b }; - } - - return new SemanticContext[] { new OR(result), a, b }; - } - - public static Collection getAndOperands(SemanticContext context) { - if (context instanceof AND) return ((AND)context).operands; - - if (context instanceof NOT) { - Collection operands = getOrOperands(((NOT)context).ctx); - List result = new ArrayList(operands.size()); - for (SemanticContext operand : operands) { - result.add(not(operand)); - } - return result; - } - - ArrayList result = new ArrayList(); - result.add(context); - return result; - } - - public static Collection getOrOperands(SemanticContext context) { - if (context instanceof OR) return ((OR)context).operands; - - if (context instanceof NOT) { - Collection operands = getAndOperands(((NOT)context).ctx); - List result = new ArrayList(operands.size()); - for (SemanticContext operand : operands) { - result.add(not(operand)); - } - return result; - } - - ArrayList result = new ArrayList(); - result.add(context); - return result; - } +// public static SemanticContext or(SemanticContext a, SemanticContext b) { +// return null; +// } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java index 7da56699a..652d17d41 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java @@ -32,6 +32,8 @@ package org.antlr.v4.runtime.dfa; import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.Nullable; +import java.util.Arrays; + /** A DFA walker that knows how to dump them to serialized strings. */ public class DFASerializer { @NotNull @@ -77,7 +79,12 @@ public class DFASerializer { int n = s.stateNumber; String stateStr = "s"+n; if ( s.isAcceptState ) { - stateStr = ":s"+n+"=>"+s.prediction; + if ( s.altToPred!=null ) { + stateStr = ":s"+n+"=>"+Arrays.toString(Arrays.copyOfRange(s.altToPred, 1, s.altToPred.length)); + } + else { + stateStr = ":s"+n+"=>"+s.prediction; + } } if ( s.isCtxSensitive ) { stateStr = ":s"+n+"@"+s.ctxToPrediction; diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java index 7659b182f..16960de45 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java @@ -29,12 +29,16 @@ package org.antlr.v4.runtime.dfa; -import org.antlr.v4.runtime.misc.Nullable; import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.atn.ATNConfig; +import org.antlr.v4.runtime.atn.SemanticContext; +import org.antlr.v4.runtime.misc.Nullable; import org.antlr.v4.runtime.misc.OrderedHashSet; -import java.util.*; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; /** A DFA state represents a set of possible ATN configurations. * As Aho, Sethi, Ullman p. 117 says "The DFA uses its state @@ -49,16 +53,15 @@ import java.util.*; * state as well, however. More importantly, we need to maintain * a stack of states, tracking the closure operations as they * jump from rule to rule, emulating rule invocations (method calls). - * Recall that ATNs do not normally have a stack like a pushdown-machine - * so I have to add one to simulate the proper lookahead sequences for + * I have to add a stack to simulate the proper lookahead sequences for * the underlying LL grammar from which the ATN was derived. * - * I use a list of ATNConfig objects. An ATNConfig + * I use a set of ATNConfig objects not simple states. An ATNConfig * is both a state (ala normal conversion) and a RuleContext describing * the chain of rules (if any) followed to arrive at that state. * * A DFA state may have multiple references to a particular state, - * but with different ATNContexts (with same or different alts) + * but with different ATN contexts (with same or different alts) * meaning that state was reached via a different set of rule invocations. */ public class DFAState { @@ -84,7 +87,9 @@ public class DFAState { public boolean isCtxSensitive; @Nullable - public Map ctxToPrediction; + public Map ctxToPrediction; // used for ctx sensitive parsing + + public SemanticContext[] altToPred; public DFAState() { } @@ -150,6 +155,17 @@ public class DFAState { @Override public String toString() { - return stateNumber+":"+configs+(isAcceptState?("=>"+prediction):""); + StringBuilder buf = new StringBuilder(); + buf.append(stateNumber + ":" + configs); + if ( isAcceptState ) { + buf.append("=>"); + if ( altToPred!=null ) { + buf.append(Arrays.toString(Arrays.copyOfRange(altToPred, 1, altToPred.length))); + } + else { + buf.append(prediction); + } + } + return buf.toString(); } } diff --git a/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java b/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java index 8b18054d1..2973f22be 100644 --- a/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java +++ b/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java @@ -3,45 +3,45 @@ package org.antlr.v4.test; import org.junit.Test; public class TestSemPredEvalParser extends BaseTest { - @Test public void testToLeft() throws Exception { - String grammar = - "grammar T;\n" + - "s : a+ ;\n" + - "a : {false}? ID {System.out.println(\"alt 1\");}\n" + - " | {true}? ID {System.out.println(\"alt 2\");}\n" + - " ;\n" + - "ID : 'a'..'z'+ ;\n" + - "INT : '0'..'9'+;\n" + - "WS : (' '|'\\n') {skip();} ;\n"; + @Test public void testSimple() throws Exception { + String grammar = + "grammar T;\n" + + "a : {false}? ID {System.out.println(\"alt 1\");}\n" + + " | {true}? ID {System.out.println(\"alt 2\");}\n" + + " | INT {System.out.println(\"alt 3\");}\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; - String found = execParser("T.g", grammar, "TParser", "TLexer", "s", - "x x y", false); - String expecting = - "alt 2\n" + - "alt 2\n" + - "alt 2\n"; - assertEquals(expecting, found); - } + String found = execParser("T.g", grammar, "TParser", "TLexer", "a", + "x", false); + String expecting = + "alt 2\n" + + "alt 2\n" + + "alt 2\n"; + assertEquals(expecting, found); + } - @Test public void testToRight() throws Exception { - String grammar = - "grammar T;\n" + - "s : a+ ;\n" + - "a : ID {false}? {System.out.println(\"alt 1\");}\n" + - " | ID {true}? {System.out.println(\"alt 2\");}\n" + - " ;\n" + - "ID : 'a'..'z'+ ;\n" + - "INT : '0'..'9'+;\n" + - "WS : (' '|'\\n') {skip();} ;\n"; + @Test public void testToLeft() throws Exception { + String grammar = + "grammar T;\n" + + "s : a+ ;\n" + + "a : {false}? ID {System.out.println(\"alt 1\");}\n" + + " | {true}? ID {System.out.println(\"alt 2\");}\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; - String found = execParser("T.g", grammar, "TParser", "TLexer", "s", - "x x y", false); - String expecting = - "alt 2\n" + - "alt 2\n" + - "alt 2\n"; - assertEquals(expecting, found); - } + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x x y", false); + String expecting = + "alt 2\n" + + "alt 2\n" + + "alt 2\n"; + assertEquals(expecting, found); + } @Test public void testActionHidesPreds() throws Exception { // can't see preds, resolves to first alt found (1 in this case)