diff --git a/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java b/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java index d4b259c10..b2670a638 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java @@ -1,35 +1,36 @@ /* [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. + Copyright (c) 2011 Terence Parr + All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.runtime; import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.misc.IntervalSet; +import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.Nullable; import org.antlr.v4.runtime.misc.OrderedHashSet; import org.antlr.v4.runtime.tree.ParseTreeListener; @@ -358,6 +359,17 @@ public abstract class BaseRecognizer extends Recognizer configs) {} /** If context sensitive parsing, we know it's ambiguity not conflict */ - public void reportAmbiguity(int startIndex, int stopIndex, Set alts, - OrderedHashSet configs) {} + public void reportAmbiguity(int startIndex, int stopIndex, + @NotNull Set ambigAlts, + @NotNull OrderedHashSet configs) { + + } + + public void reportInsufficientPredicates(int startIndex, int stopIndex, + @NotNull Set ambigAlts, + @NotNull SemanticContext[] altToPred, + @NotNull OrderedHashSet configs) + { + } + } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java index 205d5b05d..0bbdbba31 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java @@ -1,30 +1,30 @@ /* [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. + Copyright (c) 2011 Terence Parr + All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.runtime.atn; @@ -41,7 +41,6 @@ import java.util.Map; /** */ public class ATN { public static final int INVALID_ALT_NUMBER = 0; -// public static final int INVALID_DECISION_NUMBER = -1; public static final int PARSER = 1; public static final int LEXER = 2; diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 83ef621b8..103b13b3b 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -1,30 +1,30 @@ /* [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. + Copyright (c) 2011 Terence Parr + All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.runtime.atn; @@ -35,6 +35,7 @@ import org.antlr.v4.runtime.dfa.DFAState; import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.Nullable; import org.antlr.v4.runtime.misc.OrderedHashSet; +import org.antlr.v4.runtime.misc.Utils; import org.antlr.v4.runtime.tree.ASTNodeStream; import org.antlr.v4.runtime.tree.BufferedASTNodeStream; import org.antlr.v4.runtime.tree.TreeParser; @@ -43,8 +44,8 @@ import org.stringtemplate.v4.misc.MultiMap; import java.util.*; public class ParserATNSimulator extends ATNSimulator { - public static boolean debug = true; - public static boolean dfa_debug = true; + public static boolean debug = false; + public static boolean dfa_debug = false; public static int ATN_failover = 0; public static int predict_calls = 0; @@ -106,7 +107,7 @@ public class ParserATNSimulator extends ATNSimulator { int m = input.mark(); int index = input.index(); try { - int alt = execDFA(input, dfa, dfa.s0, outerContext); + int alt = execDFA(input, dfa, index, dfa.s0, outerContext); return alt; } finally { @@ -139,7 +140,7 @@ public class ParserATNSimulator extends ATNSimulator { int m = input.mark(); int index = input.index(); try { - alt = execATN(input, dfa, m, s0_closure, useContext); + alt = execATN(input, dfa, index, s0_closure, useContext); } catch (NoViableAltException nvae) { if ( debug ) dumpDeadEndConfigs(nvae); @@ -154,14 +155,19 @@ public class ParserATNSimulator extends ATNSimulator { } // doesn't create DFA when matching - public int matchATN(@NotNull SymbolStream input, @NotNull ATNState startState) { + public int matchATN(@NotNull SymbolStream input, + @NotNull ATNState startState) + { DFA dfa = new DFA(startState); RuleContext ctx = RuleContext.EMPTY; - OrderedHashSet s0_closure = computeStartState(dfa.decision, startState, ctx); + OrderedHashSet s0_closure = + computeStartState(dfa.decision, startState, ctx); return execATN(input, dfa, input.index(), s0_closure, false); } - public int execDFA(@NotNull SymbolStream input, @NotNull DFA dfa, @NotNull DFAState s0, + public int execDFA(@NotNull SymbolStream input, @NotNull DFA dfa, + int startIndex, + @NotNull DFAState s0, @Nullable RuleContext outerContext) { // dump(dfa); @@ -173,7 +179,6 @@ public class ParserATNSimulator extends ATNSimulator { DFAState prevAcceptState = null; DFAState s = s0; int t = input.LA(1); - int startIndex = input.index(); loop: while ( true ) { if ( dfa_debug ) System.out.println("DFA state "+s.stateNumber+" LA(1)=="+t); @@ -192,10 +197,17 @@ public class ParserATNSimulator extends ATNSimulator { return alt; } if ( s.isAcceptState ) { - if ( dfa_debug ) System.out.println("accept; predict "+s.prediction +" in state "+s.stateNumber); + if ( s.altToPred!=null ) { + if ( dfa_debug ) System.out.println("accept "+s); + } + else { + if ( dfa_debug ) System.out.println("accept; predict "+s.prediction +" in state "+s.stateNumber); + } prevAcceptState = s; // keep going unless we're at EOF or state only has one alt number // mentioned in configs; check if something else could match + // TODO: don't we always stop? only lexer would keep going + // TODO: v3 dfa don't do this. if ( s.complete || t==CharStream.EOF ) break; } // if no edge, pop over to ATN interpreter, update DFA and return @@ -247,6 +259,23 @@ public class ParserATNSimulator extends ATNSimulator { } if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+ " predicts "+prevAcceptState.prediction); + + // TODO: Factor this code that is very similar to ATN version + if ( s.altToPred!=null ) { + // rewind input so pred's LT(i) calls make sense + input.seek(startIndex); + int predictPredicatedAlt = evalSemanticContext(s.altToPred); + if ( predictPredicatedAlt!=ATN.INVALID_ALT_NUMBER ) { + return predictPredicatedAlt; + } + // no predicate evaluated to true + if ( prevAcceptState.prediction==ATN.INVALID_ALT_NUMBER ) { + // and now we find out that we had no uncovered alt + // to fall back to. must announce parsing error. + throw noViableAlt(input, outerContext, s.configs, startIndex); + } + } + return prevAcceptState.prediction; } @@ -322,12 +351,33 @@ public class ParserATNSimulator extends ATNSimulator { } // can we resolve with predicates? - // TODO: warn if we have uncovered alts? - SemanticContext[] altToPred = getPredsForAmbigAlts(decState, ambigAlts, reach); + SemanticContext[] altToPred = + getPredsForAmbigAlts(decState, ambigAlts, reach); if ( altToPred!=null ) { - int uniqueAlt = evalSemanticContext(ambigAlts, altToPred); + // We need at least n-1 predicates for n ambiguous alts + // [1, 2]:[null, null, 1:0] + int nPredAlts = Utils.numNonnull(altToPred); + if ( nPredAlts < ambigAlts.size()-1 ) { + reportInsufficientPredicates(startIndex, input.index(), + ambigAlts, altToPred, reach); + } + // rewind input so pred's LT(i) calls make sense + input.seek(startIndex); + int uniqueAlt = evalSemanticContext(altToPred); + if ( uniqueAlt==ATN.INVALID_ALT_NUMBER ) { + // no predicate evaluated to true + uniqueAlt = getFirstUnpredicatedAmbigAlt(ambigAlts, altToPred); + if ( debug ) System.out.println("PREDICT firstUnpredicatedAmbigAlt "+ + uniqueAlt); + if ( uniqueAlt==ATN.INVALID_ALT_NUMBER ) { + // and now we find out that we had no uncovered alt + // to fall back to. must announce parsing error. + throw noViableAlt(input, outerContext, closure, startIndex); + } + } DFAState accept = addDFAEdge(dfa, closure, t, reach); - makeAcceptState(accept, altToPred); + // TODO: split into preds to test then firstUnpredicatedAlt + makeAcceptState(accept, ambigAlts, altToPred); return uniqueAlt; } @@ -406,31 +456,37 @@ public class ParserATNSimulator extends ATNSimulator { return prevAccept.alt; } - // eval preds first in alt order for ambigAlts; - // if all false, then pick first unpredicated alt in ambigAlts - public int evalSemanticContext(Set ambigAlts, SemanticContext[] altToPred) { + // eval preds first in alt order for ambigAlts; only ambig alts can + // can have preds, though some ambig alts will be unpredicated. + // if all preds false, then pick first unpredicated alt in ambigAlts + public int evalSemanticContext(SemanticContext[] altToPred) { int uniqueAlt = ATN.INVALID_ALT_NUMBER; - int firstUnpredicatedAmbigAlt = ATN.INVALID_ALT_NUMBER; for (int i = 1; i < altToPred.length; i++) { SemanticContext or = altToPred[i]; - if ( or==null || or==SemanticContext.NONE ) { - if ( ambigAlts.contains(i) ) firstUnpredicatedAmbigAlt = i; - continue; - } - System.out.println("eval "+or+"="+or.eval(parser, outerContext)); + if ( or==null ) { + if ( debug || dfa_debug ) System.out.println("eval alt "+i+" unpredicated"); + continue; + } + if ( debug || dfa_debug ) System.out.println("eval alt "+i+" pred "+or+"="+or.eval(parser, outerContext)); if ( or.eval(parser, outerContext) ) { - System.out.println("PREDICT "+i); + if ( debug || dfa_debug ) System.out.println("PREDICT "+i); uniqueAlt = i; break; } } - if ( uniqueAlt==ATN.INVALID_ALT_NUMBER ) { - System.out.println("PREDICT firstUnpredicatedAmbigAlt "+firstUnpredicatedAmbigAlt); - uniqueAlt = firstUnpredicatedAmbigAlt; - } return uniqueAlt; } + public int getFirstUnpredicatedAmbigAlt(Set ambigAlts, SemanticContext[] altToPred) { + for (int i = 1; i < altToPred.length; i++) { + SemanticContext or = altToPred[i]; + if ( or==null || or==SemanticContext.NONE ) { + if ( ambigAlts.contains(i) ) return i; + } + } + return 0; + } + protected int resolveToMinAlt(@NotNull OrderedHashSet reach, @NotNull Set ambigAlts) { int min = getMinAlt(ambigAlts); // create DFA accept state for resolved alt @@ -734,16 +790,44 @@ public class ParserATNSimulator extends ATNSimulator { } public void reportConflict(int startIndex, int stopIndex, @NotNull Set alts, @NotNull OrderedHashSet configs) { + if ( debug ) { + System.out.println("reportConflict "+alts+":"+configs); + } if ( parser!=null ) parser.reportConflict(startIndex, stopIndex, alts, configs); } - public void reportContextSensitivity(int startIndex, int stopIndex, @NotNull Set alts, @NotNull OrderedHashSet configs) { + public void reportContextSensitivity(int startIndex, int stopIndex, + @NotNull Set alts, + @NotNull OrderedHashSet configs) + { if ( parser!=null ) parser.reportContextSensitivity(startIndex, stopIndex, alts, configs); } /** If context sensitive parsing, we know it's ambiguity not conflict */ - public void reportAmbiguity(int startIndex, int stopIndex, @NotNull Set alts, @NotNull OrderedHashSet configs) { - if ( parser!=null ) parser.reportAmbiguity(startIndex, stopIndex, alts, configs); + public void reportAmbiguity(int startIndex, int stopIndex, + @NotNull Set ambigAlts, + @NotNull OrderedHashSet configs) + { + if ( debug ) { + System.out.println("reportAmbiguity "+ + ambigAlts+":"+configs); + } + if ( parser!=null ) parser.reportAmbiguity(startIndex, stopIndex, ambigAlts, configs); + } + + public void reportInsufficientPredicates(int startIndex, int stopIndex, + @NotNull Set ambigAlts, + @NotNull SemanticContext[] altToPred, + @NotNull OrderedHashSet configs) + { + if ( debug ) { + System.out.println("reportInsufficientPredicates "+ + ambigAlts+":"+Arrays.toString(altToPred)); + } + if ( parser!=null ) { + parser.reportInsufficientPredicates(startIndex, stopIndex, ambigAlts, + altToPred, configs); + } } public static int getUniqueAlt(@NotNull Collection configs) { @@ -824,17 +908,23 @@ public class ParserATNSimulator extends ATNSimulator { int nalts = decState.getNumberOfTransitions(); SemanticContext[] altToPred = new SemanticContext[nalts +1]; for (int alt : ambigAlts) { altToPred[alt] = SemanticContext.NONE; } - boolean atLeastOne = false; + int nPredAlts = 0; for (ATNConfig c : configs) { if ( c.semanticContext!=SemanticContext.NONE && ambigAlts.contains(c.alt) ) { altToPred[c.alt] = new SemanticContext.OR(altToPred[c.alt], c.semanticContext); c.resolveWithPredicate = true; - atLeastOne = true; + nPredAlts++; } } - // nonambig alts are null in altToPred - if ( !atLeastOne ) altToPred = null; - System.out.println(Arrays.toString(altToPred)); + + // Optimize away p||p and p&&p and also make any NONE = null + for (int i = 0; i < altToPred.length; i++) { + if ( altToPred[i]!=null ) altToPred[i] = altToPred[i].optimize(); + if ( altToPred[i] == SemanticContext.NONE ) altToPred[i] = null; + } + + // nonambig alts are null in altToPred + if ( nPredAlts==0 ) altToPred = null; return altToPred; } @@ -886,13 +976,6 @@ public class ParserATNSimulator extends ATNSimulator { DFAState newState = proposed; -// boolean traversedPredicate = false; -// for (ATNConfig c : configs) { -// if ( c.traversedPredicate ) {traversedPredicate = true; break;} -// } -// -// if ( traversedPredicate ) return null; // cannot cache - newState.stateNumber = dfa.states.size(); newState.configs = new OrderedHashSet(); newState.configs.addAll(configs); @@ -907,11 +990,16 @@ public class ParserATNSimulator extends ATNSimulator { accept.complete = true; } - public void makeAcceptState(@NotNull DFAState accept, @NotNull SemanticContext[] altToPred) { + public void makeAcceptState(@NotNull DFAState accept, + @NotNull Set ambigAlts, + @NotNull SemanticContext[] altToPred) { accept.isAcceptState = true; accept.complete = true; - accept.altToPred = altToPred; - } + accept.altToPred = altToPred; + // find min unpredicated ambig alt and default to it if no + // preds match (during dfa matching) + accept.prediction = getFirstUnpredicatedAmbigAlt(ambigAlts, altToPred); + } @NotNull public String getTokenName(int t) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java index 47c17fd9e..9104bfda5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java @@ -1,36 +1,37 @@ /* [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. + Copyright (c) 2011 Terence Parr + All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.runtime.atn; import org.antlr.v4.runtime.Recognizer; import org.antlr.v4.runtime.RuleContext; +import org.antlr.v4.runtime.misc.NotNull; /** A tree structure used to record the semantic context in which * an ATN configuration is valid. It's either a single predicate, @@ -44,6 +45,9 @@ public abstract class SemanticContext { public abstract boolean eval(Recognizer parser, RuleContext ctx); + /** Optimize tree and return altered one */ + public abstract SemanticContext optimize(); + public static class Predicate extends SemanticContext { public final int ruleIndex; public final int predIndex; @@ -71,30 +75,47 @@ public abstract class SemanticContext { return parser.sempred(ctx, ruleIndex, predIndex); } - public String toString() { - return ruleIndex+":"+predIndex; + public SemanticContext optimize() { return this; } + + @Override + public int hashCode() { + return ruleIndex + predIndex; + } + + @Override + public boolean equals(Object obj) { + if ( !(obj instanceof Predicate) ) return false; + if ( this == obj ) return true; + Predicate p = (Predicate)obj; + return this.ruleIndex == p.ruleIndex && + this.predIndex == p.predIndex && + this.isCtxDependent == p.isCtxDependent; + } + + public String toString() { + return "{"+ruleIndex+":"+predIndex+"}?"; } } - public static class TruePredicate extends Predicate { - @Override - public String toString() { - return "true"; // not used for code gen, just DOT and print outs - } - } - - public static class FalsePredicate extends Predicate { - @Override - public String toString() { - return "false"; // not used for code gen, just DOT and print outs - } - } +// public static class TruePredicate extends Predicate { +// @Override +// public String toString() { +// return "true"; // not used for code gen, just DOT and print outs +// } +// } +// +// public static class FalsePredicate extends Predicate { +// @Override +// public String toString() { +// return "false"; // not used for code gen, just DOT and print outs +// } +// } public static class AND extends SemanticContext { public SemanticContext a; public SemanticContext b; public AND() { } - public AND(SemanticContext a, SemanticContext b) { + public AND(@NotNull SemanticContext a, @NotNull SemanticContext b) { this.a = a; this.b = b; } @@ -105,6 +126,15 @@ public abstract class SemanticContext { return a.eval(parser, ctx) && b.eval(parser, ctx); } + public SemanticContext optimize() { + SemanticContext a_ = a.optimize(); + SemanticContext b_ = b.optimize(); + if ( a_ == NONE ) return b_; + if ( b_ == NONE ) return a_; + if ( a_.equals(b_) ) return a_; + return new AND(a_, b_); + } + public String toString() { if ( a == NONE ) return b.toString(); if ( b == NONE ) return a.toString(); @@ -116,7 +146,7 @@ public abstract class SemanticContext { public SemanticContext a; public SemanticContext b; public OR() { } - public OR(SemanticContext a, SemanticContext b) { + public OR(@NotNull SemanticContext a, @NotNull SemanticContext b) { this.a = a; this.b = b; } @@ -127,6 +157,15 @@ public abstract class SemanticContext { return a.eval(parser, ctx) && b.eval(parser, ctx); } + public SemanticContext optimize() { + SemanticContext a_ = a.optimize(); + SemanticContext b_ = b.optimize(); + if ( a_ == NONE ) return b_; + if ( b_ == NONE ) return a_; + if ( a_.equals(b_) ) return a_; + return new OR(a_, b_); + } + @Override public String toString() { if ( a == NONE ) return b.toString(); @@ -134,23 +173,4 @@ public abstract class SemanticContext { return a+"||"+b; } } - -// public static SemanticContext and(AND a, Predicate b) { -// if ( a == NONE ) return new AND(b); -// if ( b == NONE ) {a.add(b); return a;} -// a.add(b); -// AND and = new SemanticContext.AND(); -// if ( a instanceof AND ) { -// and.add(b); -// } -// else { -// and.add((Predicate)a); -// and.add(b); -// } -// return and; -// } -// -// public static SemanticContext or(SemanticContext a, SemanticContext b) { -// return null; -// } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java index 16960de45..600ed8f1a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java @@ -1,30 +1,30 @@ /* [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. + Copyright (c) 2011 Terence Parr + All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.runtime.dfa; @@ -79,10 +79,11 @@ public class DFAState { public boolean isAcceptState = false; - public int prediction; // if accept state, what ttype do we match? + public int prediction; // if accept state, what ttype do we match? is "else" clause if predicated public int ruleIndex; // if accept, exec what action? + // todo: rename as unique? public boolean complete; // all alts predict "prediction" public boolean isCtxSensitive; diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java index a4bec0285..2d0dd1584 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java @@ -1,30 +1,30 @@ /* [The "BSD license"] - Copyright (c) 2011 Terence Parr - All rights reserved. + Copyright (c) 2011 Terence Parr + All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.antlr.v4.runtime.misc; @@ -43,4 +43,13 @@ public class Utils { } return buf.toString(); } + + public static int numNonnull(Object[] data) { + int n = 0; + if ( data == null ) return n; + for (Object o : data) { + if ( o!=null ) n++; + } + return n; + } } diff --git a/tool/playground/HTMLParser.g b/tool/playground/HTMLParser.g index 4f055b493..a01bca072 100644 --- a/tool/playground/HTMLParser.g +++ b/tool/playground/HTMLParser.g @@ -1,6 +1,6 @@ parser grammar HTMLParser; -options { tokenVocab=HTMLLexer; } +options { tokenVocab=HTMLParser; } file : ( TAG_START (starttag | endtag) | TEXT {System.out.println("TEXT "+$TEXT);} )+ EOF ; diff --git a/tool/playground/T.g b/tool/playground/T.g index a67e4c623..567d60cae 100644 --- a/tool/playground/T.g +++ b/tool/playground/T.g @@ -1,13 +1,8 @@ grammar T; -options {output=AST;} - -s : i=ifstat {System.out.println(_input.toString(0,_input.index()-1));} ; - -ifstat : 'if' '(' expr ')' assign ; -assign : ID '=' expr ';' ; -expr : INT | ID ; - -EQ : '=' ; -INT : '0'..'9'+ ; +s : a a; +a : {_input.LT(1).equals("x")}? ID INT {System.out.println("alt 1");} + | {_input.LT(1).equals("y")}? ID INT {System.out.println("alt 2");} + ; ID : 'a'..'z'+ ; -WS : (' '|'\n')+ {skip();} ; +INT : '0'..'9'+; +WS : (' '|'\n') {skip();} ; diff --git a/tool/playground/TestT.java b/tool/playground/TestT.java index eacc8d83b..e4f3a84ad 100644 --- a/tool/playground/TestT.java +++ b/tool/playground/TestT.java @@ -27,11 +27,8 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import org.antlr.v4.runtime.*; -import org.antlr.v4.runtime.tree.*; -import org.antlr.v4.runtime.tree.gui.TreeViewer; - -import java.util.ArrayList; +import org.antlr.v4.runtime.ANTLRFileStream; +import org.antlr.v4.runtime.CommonTokenStream; public class TestT { public static void main(String[] args) throws Exception { @@ -45,14 +42,14 @@ public class TestT { p.setBuildParseTree(true); final TParser.sContext tree = p.s(); System.out.println(tree.toStringTree(p)); - TreeViewer v = new TreeViewer(p, tree); - v.setHighlightedBoxColor(TreeViewer.LIGHT_RED); - v.addHighlightedNodes(new ArrayList() {{ - ParseTree c0 = tree.getChild(0); - add(c0); - add(c0.getChild(0)); - }}); - v.open(); +// TreeViewer v = new TreeViewer(p, tree); +// v.setHighlightedBoxColor(TreeViewer.LIGHT_RED); +// v.addHighlightedNodes(new ArrayList() {{ +// ParseTree c0 = tree.getChild(0); +// add(c0); +// add(c0.getChild(0)); +// }}); +// v.open(); // tree.inspect(p); // // ParseTreeWalker walker = new ParseTreeWalker(); diff --git a/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java b/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java index 2973f22be..7f368b953 100644 --- a/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java +++ b/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java @@ -1,32 +1,206 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + package org.antlr.v4.test; import org.junit.Test; public class TestSemPredEvalParser extends BaseTest { - @Test public void testSimple() throws Exception { - String grammar = - "grammar T;\n" + - "a : {false}? ID {System.out.println(\"alt 1\");}\n" + - " | {true}? ID {System.out.println(\"alt 2\");}\n" + - " | INT {System.out.println(\"alt 3\");}\n" + - " ;\n" + - "ID : 'a'..'z'+ ;\n" + - "INT : '0'..'9'+;\n" + - "WS : (' '|'\\n') {skip();} ;\n"; + @Test public void testSimple() throws Exception { + String grammar = + "grammar T;\n" + + "s : a a a;\n" + // do 3x: once in ATN, next in DFA then INT in ATN + "a : {false}? ID {System.out.println(\"alt 1\");}\n" + + " | {true}? ID {System.out.println(\"alt 2\");}\n" + + " | INT {System.out.println(\"alt 3\");}\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; - String found = execParser("T.g", grammar, "TParser", "TLexer", "a", - "x", false); - String expecting = - "alt 2\n" + - "alt 2\n" + - "alt 2\n"; - assertEquals(expecting, found); - } + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x y 3", false); + String expecting = + "alt 2\n" + + "alt 2\n" + + "alt 3\n"; + assertEquals(expecting, found); + } - @Test public void testToLeft() throws Exception { - String grammar = - "grammar T;\n" + - "s : a+ ;\n" + + @Test public void testOrder() throws Exception { + // Predicates disambiguate and so we don't arbitrarily choose the first alt + // Here, there are n-1 predicates for n=2 alts and so we simulate + // the nth predicate as !(others). We do that by testing the + // predicates first and then try the on predicated alternatives. + // Since the 2nd alternative has a true predicate, we always choose that one + String grammar = + "grammar T;\n" + + "s : a a;\n" + // do 2x: once in ATN, next in DFA + "a : ID {System.out.println(\"alt 1\");}\n" + + " | {true}? ID {System.out.println(\"alt 2\");}\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x y", false); + String expecting = + "alt 2\n" + + "alt 2\n"; + assertEquals(expecting, found); + } + + @Test public void test2UnpredicatedAlts() throws Exception { + // We have n-2 predicates for n alternatives. We have no choice + // but to pick the first on predicated alternative if the n-2 + // predicates fail. + // this should call reportInsufficientPredicates() + String grammar = + "grammar T;\n" + + "@header {" + + "import java.util.*;" + + "}" + + "@parser::members {" + + "public void reportInsufficientPredicates(int startIndex, int stopIndex,\n" + + "\t\t\t\t\t\t\t\t\t\t\t @NotNull Set ambigAlts,\n" + + "\t\t\t\t\t\t\t\t\t\t\t @NotNull SemanticContext[] altToPred,\n" + + "\t\t\t\t\t\t\t\t\t\t\t @NotNull OrderedHashSet configs)\n" + + "{\n" + + "System.out.println(\"reportInsufficientPredicates\");\n" + + "}\n" + + "}\n"+ + "s : a a;\n" + // do 2x: once in ATN, next in DFA + "a : ID {System.out.println(\"alt 1\");}\n" + + " | ID {System.out.println(\"alt 2\");}\n" + + " | {false}? ID {System.out.println(\"alt 3\");}\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "x y", false); + String expecting = + "reportInsufficientPredicates\n" + + "alt 1\n" + + "alt 1\n"; + assertEquals(expecting, found); + } + + @Test public void test2UnpredicatedAltsAndOneOrthogonalAlt() throws Exception { + // We have n-2 predicates for n alternatives. We have no choice + // but to pick the first on predicated alternative if the n-2 + // predicates fail. + // this should call reportInsufficientPredicates() + String grammar = + "grammar T;\n" + + "@header {" + + "import java.util.*;" + + "}" + + "@parser::members {" + + "public void reportInsufficientPredicates(int startIndex, int stopIndex,\n" + + "\t\t\t\t\t\t\t\t\t\t\t @NotNull Set ambigAlts,\n" + + "\t\t\t\t\t\t\t\t\t\t\t @NotNull SemanticContext[] altToPred,\n" + + "\t\t\t\t\t\t\t\t\t\t\t @NotNull OrderedHashSet configs)\n" + + "{\n" + + "System.out.println(\"reportInsufficientPredicates\");\n" + + "}\n" + + "}\n"+ + "s : a a a;\n" + + "a : INT {System.out.println(\"alt 1\");}\n" + + " | ID {System.out.println(\"alt 2\");}\n" + + " | ID {System.out.println(\"alt 3\");}\n" + + " | {false}? ID {System.out.println(\"alt 4\");}\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "34 x y", false); + String expecting = + "alt 1\n" + + "reportInsufficientPredicates\n" + + "alt 2\n" + + "alt 2\n"; + assertEquals(expecting, found); + } + + @Test public void testRewindBeforePredEval() throws Exception { + // The parser consumes ID and moves to the 2nd token INT. + // To properly evaluate the predicates after matching ID INT, + // we must correctly see come back to starting index so LT(1) works + String grammar = + "grammar T;\n" + + "s : a a;\n" + + "a : {_input.LT(1).getText().equals(\"x\")}? ID INT {System.out.println(\"alt 1\");}\n" + + " | {_input.LT(1).getText().equals(\"y\")}? ID INT {System.out.println(\"alt 2\");}\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + String found = execParser("T.g", grammar, "TParser", "TLexer", "s", + "y 3 x 4", false); + String expecting = + "alt 2\n" + + "alt 1\n"; + assertEquals(expecting, found); + } + + @Test public void testNoTruePredsThrowsNoViableAlt() throws Exception { + // checks that we throw exception if all alts + // are covered with a predicate and none succeeds + String grammar = + "grammar T;\n" + + "s : a a;\n" + + "a : {false}? ID INT {System.out.println(\"alt 1\");}\n" + + " | {false}? ID INT {System.out.println(\"alt 2\");}\n" + + " ;\n" + + "ID : 'a'..'z'+ ;\n" + + "INT : '0'..'9'+;\n" + + "WS : (' '|'\\n') {skip();} ;\n"; + + execParser("T.g", grammar, "TParser", "TLexer", "s", + "y 3 x 4", false); + String expecting = "line 1:0 no viable alternative at input 'y'\n"; + String result = stderrDuringParse; + assertEquals(expecting, result); + } + + // TODO: test predicates that are validating predicates; not ambiguous decisions + + @Test public void testToLeft() throws Exception { + String grammar = + "grammar T;\n" + + "s : a+ ;\n" + "a : {false}? ID {System.out.println(\"alt 1\");}\n" + " | {true}? ID {System.out.println(\"alt 2\");}\n" + " ;\n" + @@ -151,31 +325,6 @@ public class TestSemPredEvalParser extends BaseTest { assertEquals(expecting, found); } - @Test public void testToRightWithVaryingPredicate() throws Exception { - // alternate predicted alt to ensure DFA doesn't cache - String grammar = - "grammar T;\n" + - "@members {int i=0;}\n" + - "s : ({i++; System.out.println(\"i=\"+i);} a)+ ;\n" + - "a : ID {i \\% 2 == 0}? {System.out.println(\"alt 1\");}\n" + - " | ID {i \\% 2 != 0}? {System.out.println(\"alt 2\");}\n" + - " ;\n" + - "ID : 'a'..'z'+ ;\n" + - "INT : '0'..'9'+;\n" + - "WS : (' '|'\\n') {skip();} ;\n"; - - String found = execParser("T.g", grammar, "TParser", "TLexer", "s", - "x x y", false); - String expecting = - "i=1\n" + - "alt 2\n" + - "i=2\n" + - "alt 1\n" + - "i=3\n" + - "alt 2\n"; - assertEquals(expecting, found); - } - /** During a global follow operation, we still execute semantic * predicates as long as they are not dependent on local context */ @@ -196,10 +345,6 @@ public class TestSemPredEvalParser extends BaseTest { String found = execParser("T.g", grammar, "TParser", "TLexer", "s", "a!", false); String expecting = - "eval=true\n" + // do p(true), p(false) once during s0 computation from epsilon edge in e - "eval=false\n" + - "eval=true\n" + // do them again during closure after passing ID in e - "eval=false\n" + "eval=true\n" + // now we are parsing "parse\n"; assertEquals(expecting, found);