forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

got ambig pred thing working i think; unit tests. missing validating pred thing. 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9480]
This commit is contained in:
parrt 2011-11-28 19:16:45 -08:00
parent e288ff3039
commit 9a1a8428b5
10 changed files with 559 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java
View File

@ -1,35 +1,36 @@
/* /*
[The "BSD license"] [The "BSD license"]
Copyright (c) 2011 Terence Parr Copyright (c) 2011 Terence Parr
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions modification, are permitted provided that the following conditions
are met: are met:
1. Redistributions of source code must retain the above copyright 1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright 2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution. documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products 3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission. derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package org.antlr.v4.runtime; package org.antlr.v4.runtime;
import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.misc.IntervalSet; import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable; import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.misc.OrderedHashSet; import org.antlr.v4.runtime.misc.OrderedHashSet;
import org.antlr.v4.runtime.tree.ParseTreeListener; import org.antlr.v4.runtime.tree.ParseTreeListener;
@ -358,6 +359,17 @@ public abstract class BaseRecognizer<Symbol> extends Recognizer<Symbol, ParserAT
OrderedHashSet<ATNConfig> configs) {} OrderedHashSet<ATNConfig> configs) {}
/** If context sensitive parsing, we know it's ambiguity not conflict */ /** If context sensitive parsing, we know it's ambiguity not conflict */
public void reportAmbiguity(int startIndex, int stopIndex, Set<Integer> alts, public void reportAmbiguity(int startIndex, int stopIndex,
OrderedHashSet<ATNConfig> configs) {} @NotNull Set<Integer> ambigAlts,
@NotNull OrderedHashSet<ATNConfig> configs) {
}
public void reportInsufficientPredicates(int startIndex, int stopIndex,
@NotNull Set<Integer> ambigAlts,
@NotNull SemanticContext[] altToPred,
@NotNull OrderedHashSet<ATNConfig> configs)
{
}
} }

45
runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java
View File

@ -1,30 +1,30 @@
/* /*
[The "BSD license"] [The "BSD license"]
Copyright (c) 2011 Terence Parr Copyright (c) 2011 Terence Parr
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions modification, are permitted provided that the following conditions
are met: are met:
1. Redistributions of source code must retain the above copyright 1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright 2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution. documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products 3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission. derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package org.antlr.v4.runtime.atn; package org.antlr.v4.runtime.atn;
@ -41,7 +41,6 @@ import java.util.Map;
/** */ /** */
public class ATN { public class ATN {
public static final int INVALID_ALT_NUMBER = 0; public static final int INVALID_ALT_NUMBER = 0;
// public static final int INVALID_DECISION_NUMBER = -1;
public static final int PARSER = 1; public static final int PARSER = 1;
public static final int LEXER = 2; public static final int LEXER = 2;

222
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -1,30 +1,30 @@
/* /*
[The "BSD license"] [The "BSD license"]
Copyright (c) 2011 Terence Parr Copyright (c) 2011 Terence Parr
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions modification, are permitted provided that the following conditions
are met: are met:
1. Redistributions of source code must retain the above copyright 1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright 2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution. documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products 3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission. derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package org.antlr.v4.runtime.atn; package org.antlr.v4.runtime.atn;
@ -35,6 +35,7 @@ import org.antlr.v4.runtime.dfa.DFAState;
import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable; import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.misc.OrderedHashSet; import org.antlr.v4.runtime.misc.OrderedHashSet;
import org.antlr.v4.runtime.misc.Utils;
import org.antlr.v4.runtime.tree.ASTNodeStream; import org.antlr.v4.runtime.tree.ASTNodeStream;
import org.antlr.v4.runtime.tree.BufferedASTNodeStream; import org.antlr.v4.runtime.tree.BufferedASTNodeStream;
import org.antlr.v4.runtime.tree.TreeParser; import org.antlr.v4.runtime.tree.TreeParser;
@ -43,8 +44,8 @@ import org.stringtemplate.v4.misc.MultiMap;
import java.util.*; import java.util.*;
public class ParserATNSimulator<Symbol> extends ATNSimulator { public class ParserATNSimulator<Symbol> extends ATNSimulator {
public static boolean debug = true; public static boolean debug = false;
public static boolean dfa_debug = true; public static boolean dfa_debug = false;
public static int ATN_failover = 0; public static int ATN_failover = 0;
public static int predict_calls = 0; public static int predict_calls = 0;
@ -106,7 +107,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
int m = input.mark(); int m = input.mark();
int index = input.index(); int index = input.index();
try { try {
int alt = execDFA(input, dfa, dfa.s0, outerContext); int alt = execDFA(input, dfa, index, dfa.s0, outerContext);
return alt; return alt;
} }
finally { finally {
@ -139,7 +140,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
int m = input.mark(); int m = input.mark();
int index = input.index(); int index = input.index();
try { try {
alt = execATN(input, dfa, m, s0_closure, useContext); alt = execATN(input, dfa, index, s0_closure, useContext);
} }
catch (NoViableAltException nvae) { catch (NoViableAltException nvae) {
if ( debug ) dumpDeadEndConfigs(nvae); if ( debug ) dumpDeadEndConfigs(nvae);
@ -154,14 +155,19 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
} }
// doesn't create DFA when matching // doesn't create DFA when matching
public int matchATN(@NotNull SymbolStream<Symbol> input, @NotNull ATNState startState) { public int matchATN(@NotNull SymbolStream<Symbol> input,
@NotNull ATNState startState)
{
DFA dfa = new DFA(startState); DFA dfa = new DFA(startState);
RuleContext ctx = RuleContext.EMPTY; RuleContext ctx = RuleContext.EMPTY;
OrderedHashSet<ATNConfig> s0_closure = computeStartState(dfa.decision, startState, ctx); OrderedHashSet<ATNConfig> s0_closure =
computeStartState(dfa.decision, startState, ctx);
return execATN(input, dfa, input.index(), s0_closure, false); return execATN(input, dfa, input.index(), s0_closure, false);
} }
public int execDFA(@NotNull SymbolStream<Symbol> input, @NotNull DFA dfa, @NotNull DFAState s0, public int execDFA(@NotNull SymbolStream<Symbol> input, @NotNull DFA dfa,
int startIndex,
@NotNull DFAState s0,
@Nullable RuleContext outerContext) @Nullable RuleContext outerContext)
{ {
// dump(dfa); // dump(dfa);
@ -173,7 +179,6 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
DFAState prevAcceptState = null; DFAState prevAcceptState = null;
DFAState s = s0; DFAState s = s0;
int t = input.LA(1); int t = input.LA(1);
int startIndex = input.index();
loop: loop:
while ( true ) { while ( true ) {
if ( dfa_debug ) System.out.println("DFA state "+s.stateNumber+" LA(1)=="+t); if ( dfa_debug ) System.out.println("DFA state "+s.stateNumber+" LA(1)=="+t);
@ -192,10 +197,17 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
return alt; return alt;
} }
if ( s.isAcceptState ) { if ( s.isAcceptState ) {
if ( dfa_debug ) System.out.println("accept; predict "+s.prediction +" in state "+s.stateNumber); if ( s.altToPred!=null ) {
if ( dfa_debug ) System.out.println("accept "+s);
}
else {
if ( dfa_debug ) System.out.println("accept; predict "+s.prediction +" in state "+s.stateNumber);
}
prevAcceptState = s; prevAcceptState = s;
// keep going unless we're at EOF or state only has one alt number // keep going unless we're at EOF or state only has one alt number
// mentioned in configs; check if something else could match // mentioned in configs; check if something else could match
// TODO: don't we always stop? only lexer would keep going
// TODO: v3 dfa don't do this.
if ( s.complete || t==CharStream.EOF ) break; if ( s.complete || t==CharStream.EOF ) break;
} }
// if no edge, pop over to ATN interpreter, update DFA and return // if no edge, pop over to ATN interpreter, update DFA and return
@ -247,6 +259,23 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
} }
if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+ if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+
" predicts "+prevAcceptState.prediction); " predicts "+prevAcceptState.prediction);
// TODO: Factor this code that is very similar to ATN version
if ( s.altToPred!=null ) {
// rewind input so pred's LT(i) calls make sense
input.seek(startIndex);
int predictPredicatedAlt = evalSemanticContext(s.altToPred);
if ( predictPredicatedAlt!=ATN.INVALID_ALT_NUMBER ) {
return predictPredicatedAlt;
}
// no predicate evaluated to true
if ( prevAcceptState.prediction==ATN.INVALID_ALT_NUMBER ) {
// and now we find out that we had no uncovered alt
// to fall back to. must announce parsing error.
throw noViableAlt(input, outerContext, s.configs, startIndex);
}
}
return prevAcceptState.prediction; return prevAcceptState.prediction;
} }
@ -322,12 +351,33 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
} }
// can we resolve with predicates? // can we resolve with predicates?
// TODO: warn if we have uncovered alts? SemanticContext[] altToPred =
SemanticContext[] altToPred = getPredsForAmbigAlts(decState, ambigAlts, reach); getPredsForAmbigAlts(decState, ambigAlts, reach);
if ( altToPred!=null ) { if ( altToPred!=null ) {
int uniqueAlt = evalSemanticContext(ambigAlts, altToPred); // We need at least n-1 predicates for n ambiguous alts
// [1, 2]:[null, null, 1:0]
int nPredAlts = Utils.numNonnull(altToPred);
if ( nPredAlts < ambigAlts.size()-1 ) {
reportInsufficientPredicates(startIndex, input.index(),
ambigAlts, altToPred, reach);
}
// rewind input so pred's LT(i) calls make sense
input.seek(startIndex);
int uniqueAlt = evalSemanticContext(altToPred);
if ( uniqueAlt==ATN.INVALID_ALT_NUMBER ) {
// no predicate evaluated to true
uniqueAlt = getFirstUnpredicatedAmbigAlt(ambigAlts, altToPred);
if ( debug ) System.out.println("PREDICT firstUnpredicatedAmbigAlt "+
uniqueAlt);
if ( uniqueAlt==ATN.INVALID_ALT_NUMBER ) {
// and now we find out that we had no uncovered alt
// to fall back to. must announce parsing error.
throw noViableAlt(input, outerContext, closure, startIndex);
}
}
DFAState accept = addDFAEdge(dfa, closure, t, reach); DFAState accept = addDFAEdge(dfa, closure, t, reach);
makeAcceptState(accept, altToPred); // TODO: split into preds to test then firstUnpredicatedAlt
makeAcceptState(accept, ambigAlts, altToPred);
return uniqueAlt; return uniqueAlt;
} }
@ -406,31 +456,37 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
return prevAccept.alt; return prevAccept.alt;
} }
// eval preds first in alt order for ambigAlts; // eval preds first in alt order for ambigAlts; only ambig alts can
// if all false, then pick first unpredicated alt in ambigAlts // can have preds, though some ambig alts will be unpredicated.
public int evalSemanticContext(Set<Integer> ambigAlts, SemanticContext[] altToPred) { // if all preds false, then pick first unpredicated alt in ambigAlts
public int evalSemanticContext(SemanticContext[] altToPred) {
int uniqueAlt = ATN.INVALID_ALT_NUMBER; int uniqueAlt = ATN.INVALID_ALT_NUMBER;
int firstUnpredicatedAmbigAlt = ATN.INVALID_ALT_NUMBER;
for (int i = 1; i < altToPred.length; i++) { for (int i = 1; i < altToPred.length; i++) {
SemanticContext or = altToPred[i]; SemanticContext or = altToPred[i];
if ( or==null || or==SemanticContext.NONE ) { if ( or==null ) {
if ( ambigAlts.contains(i) ) firstUnpredicatedAmbigAlt = i; if ( debug || dfa_debug ) System.out.println("eval alt "+i+" unpredicated");
continue; continue;
} }
System.out.println("eval "+or+"="+or.eval(parser, outerContext)); if ( debug || dfa_debug ) System.out.println("eval alt "+i+" pred "+or+"="+or.eval(parser, outerContext));
if ( or.eval(parser, outerContext) ) { if ( or.eval(parser, outerContext) ) {
System.out.println("PREDICT "+i); if ( debug || dfa_debug ) System.out.println("PREDICT "+i);
uniqueAlt = i; uniqueAlt = i;
break; break;
} }
} }
if ( uniqueAlt==ATN.INVALID_ALT_NUMBER ) {
System.out.println("PREDICT firstUnpredicatedAmbigAlt "+firstUnpredicatedAmbigAlt);
uniqueAlt = firstUnpredicatedAmbigAlt;
}
return uniqueAlt; return uniqueAlt;
} }
public int getFirstUnpredicatedAmbigAlt(Set<Integer> ambigAlts, SemanticContext[] altToPred) {
for (int i = 1; i < altToPred.length; i++) {
SemanticContext or = altToPred[i];
if ( or==null || or==SemanticContext.NONE ) {
if ( ambigAlts.contains(i) ) return i;
}
}
return 0;
}
protected int resolveToMinAlt(@NotNull OrderedHashSet<ATNConfig> reach, @NotNull Set<Integer> ambigAlts) { protected int resolveToMinAlt(@NotNull OrderedHashSet<ATNConfig> reach, @NotNull Set<Integer> ambigAlts) {
int min = getMinAlt(ambigAlts); int min = getMinAlt(ambigAlts);
// create DFA accept state for resolved alt // create DFA accept state for resolved alt
@ -734,16 +790,44 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
} }
public void reportConflict(int startIndex, int stopIndex, @NotNull Set<Integer> alts, @NotNull OrderedHashSet<ATNConfig> configs) { public void reportConflict(int startIndex, int stopIndex, @NotNull Set<Integer> alts, @NotNull OrderedHashSet<ATNConfig> configs) {
if ( debug ) {
System.out.println("reportConflict "+alts+":"+configs);
}
if ( parser!=null ) parser.reportConflict(startIndex, stopIndex, alts, configs); if ( parser!=null ) parser.reportConflict(startIndex, stopIndex, alts, configs);
} }
public void reportContextSensitivity(int startIndex, int stopIndex, @NotNull Set<Integer> alts, @NotNull OrderedHashSet<ATNConfig> configs) { public void reportContextSensitivity(int startIndex, int stopIndex,
@NotNull Set<Integer> alts,
@NotNull OrderedHashSet<ATNConfig> configs)
{
if ( parser!=null ) parser.reportContextSensitivity(startIndex, stopIndex, alts, configs); if ( parser!=null ) parser.reportContextSensitivity(startIndex, stopIndex, alts, configs);
} }
/** If context sensitive parsing, we know it's ambiguity not conflict */ /** If context sensitive parsing, we know it's ambiguity not conflict */
public void reportAmbiguity(int startIndex, int stopIndex, @NotNull Set<Integer> alts, @NotNull OrderedHashSet<ATNConfig> configs) { public void reportAmbiguity(int startIndex, int stopIndex,
if ( parser!=null ) parser.reportAmbiguity(startIndex, stopIndex, alts, configs); @NotNull Set<Integer> ambigAlts,
@NotNull OrderedHashSet<ATNConfig> configs)
{
if ( debug ) {
System.out.println("reportAmbiguity "+
ambigAlts+":"+configs);
}
if ( parser!=null ) parser.reportAmbiguity(startIndex, stopIndex, ambigAlts, configs);
}
public void reportInsufficientPredicates(int startIndex, int stopIndex,
@NotNull Set<Integer> ambigAlts,
@NotNull SemanticContext[] altToPred,
@NotNull OrderedHashSet<ATNConfig> configs)
{
if ( debug ) {
System.out.println("reportInsufficientPredicates "+
ambigAlts+":"+Arrays.toString(altToPred));
}
if ( parser!=null ) {
parser.reportInsufficientPredicates(startIndex, stopIndex, ambigAlts,
altToPred, configs);
}
} }
public static int getUniqueAlt(@NotNull Collection<ATNConfig> configs) { public static int getUniqueAlt(@NotNull Collection<ATNConfig> configs) {
@ -824,17 +908,23 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
int nalts = decState.getNumberOfTransitions(); int nalts = decState.getNumberOfTransitions();
SemanticContext[] altToPred = new SemanticContext[nalts +1]; SemanticContext[] altToPred = new SemanticContext[nalts +1];
for (int alt : ambigAlts) { altToPred[alt] = SemanticContext.NONE; } for (int alt : ambigAlts) { altToPred[alt] = SemanticContext.NONE; }
boolean atLeastOne = false; int nPredAlts = 0;
for (ATNConfig c : configs) { for (ATNConfig c : configs) {
if ( c.semanticContext!=SemanticContext.NONE && ambigAlts.contains(c.alt) ) { if ( c.semanticContext!=SemanticContext.NONE && ambigAlts.contains(c.alt) ) {
altToPred[c.alt] = new SemanticContext.OR(altToPred[c.alt], c.semanticContext); altToPred[c.alt] = new SemanticContext.OR(altToPred[c.alt], c.semanticContext);
c.resolveWithPredicate = true; c.resolveWithPredicate = true;
atLeastOne = true; nPredAlts++;
} }
} }
// nonambig alts are null in altToPred
if ( !atLeastOne ) altToPred = null; // Optimize away p||p and p&&p and also make any NONE = null
System.out.println(Arrays.toString(altToPred)); for (int i = 0; i < altToPred.length; i++) {
if ( altToPred[i]!=null ) altToPred[i] = altToPred[i].optimize();
if ( altToPred[i] == SemanticContext.NONE ) altToPred[i] = null;
}
// nonambig alts are null in altToPred
if ( nPredAlts==0 ) altToPred = null;
return altToPred; return altToPred;
} }
@ -886,13 +976,6 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
DFAState newState = proposed; DFAState newState = proposed;
// boolean traversedPredicate = false;
// for (ATNConfig c : configs) {
// if ( c.traversedPredicate ) {traversedPredicate = true; break;}
// }
//
// if ( traversedPredicate ) return null; // cannot cache
newState.stateNumber = dfa.states.size(); newState.stateNumber = dfa.states.size();
newState.configs = new OrderedHashSet<ATNConfig>(); newState.configs = new OrderedHashSet<ATNConfig>();
newState.configs.addAll(configs); newState.configs.addAll(configs);
@ -907,11 +990,16 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
accept.complete = true; accept.complete = true;
} }
public void makeAcceptState(@NotNull DFAState accept, @NotNull SemanticContext[] altToPred) { public void makeAcceptState(@NotNull DFAState accept,
@NotNull Set<Integer> ambigAlts,
@NotNull SemanticContext[] altToPred) {
accept.isAcceptState = true; accept.isAcceptState = true;
accept.complete = true; accept.complete = true;
accept.altToPred = altToPred; accept.altToPred = altToPred;
} // find min unpredicated ambig alt and default to it if no
// preds match (during dfa matching)
accept.prediction = getFirstUnpredicatedAmbigAlt(ambigAlts, altToPred);
}
@NotNull @NotNull
public String getTokenName(int t) { public String getTokenName(int t) {

136
runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java
View File

@ -1,36 +1,37 @@
/* /*
[The "BSD license"] [The "BSD license"]
Copyright (c) 2011 Terence Parr Copyright (c) 2011 Terence Parr
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions modification, are permitted provided that the following conditions
are met: are met:
1. Redistributions of source code must retain the above copyright 1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright 2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution. documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products 3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission. derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package org.antlr.v4.runtime.atn; package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.Recognizer; import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.misc.NotNull;
/** A tree structure used to record the semantic context in which /** A tree structure used to record the semantic context in which
* an ATN configuration is valid. It's either a single predicate, * an ATN configuration is valid. It's either a single predicate,
@ -44,6 +45,9 @@ public abstract class SemanticContext {
public abstract boolean eval(Recognizer<?,?> parser, RuleContext ctx); public abstract boolean eval(Recognizer<?,?> parser, RuleContext ctx);
/** Optimize tree and return altered one */
public abstract SemanticContext optimize();
public static class Predicate extends SemanticContext { public static class Predicate extends SemanticContext {
public final int ruleIndex; public final int ruleIndex;
public final int predIndex; public final int predIndex;
@ -71,30 +75,47 @@ public abstract class SemanticContext {
return parser.sempred(ctx, ruleIndex, predIndex); return parser.sempred(ctx, ruleIndex, predIndex);
} }
public String toString() { public SemanticContext optimize() { return this; }
return ruleIndex+":"+predIndex;
@Override
public int hashCode() {
return ruleIndex + predIndex;
}
@Override
public boolean equals(Object obj) {
if ( !(obj instanceof Predicate) ) return false;
if ( this == obj ) return true;
Predicate p = (Predicate)obj;
return this.ruleIndex == p.ruleIndex &&
this.predIndex == p.predIndex &&
this.isCtxDependent == p.isCtxDependent;
}
public String toString() {
return "{"+ruleIndex+":"+predIndex+"}?";
} }
} }
public static class TruePredicate extends Predicate { // public static class TruePredicate extends Predicate {
@Override // @Override
public String toString() { // public String toString() {
return "true"; // not used for code gen, just DOT and print outs // return "true"; // not used for code gen, just DOT and print outs
} // }
} // }
//
public static class FalsePredicate extends Predicate { // public static class FalsePredicate extends Predicate {
@Override // @Override
public String toString() { // public String toString() {
return "false"; // not used for code gen, just DOT and print outs // return "false"; // not used for code gen, just DOT and print outs
} // }
} // }
public static class AND extends SemanticContext { public static class AND extends SemanticContext {
public SemanticContext a; public SemanticContext a;
public SemanticContext b; public SemanticContext b;
public AND() { } public AND() { }
public AND(SemanticContext a, SemanticContext b) { public AND(@NotNull SemanticContext a, @NotNull SemanticContext b) {
this.a = a; this.a = a;
this.b = b; this.b = b;
} }
@ -105,6 +126,15 @@ public abstract class SemanticContext {
return a.eval(parser, ctx) && b.eval(parser, ctx); return a.eval(parser, ctx) && b.eval(parser, ctx);
} }
public SemanticContext optimize() {
SemanticContext a_ = a.optimize();
SemanticContext b_ = b.optimize();
if ( a_ == NONE ) return b_;
if ( b_ == NONE ) return a_;
if ( a_.equals(b_) ) return a_;
return new AND(a_, b_);
}
public String toString() { public String toString() {
if ( a == NONE ) return b.toString(); if ( a == NONE ) return b.toString();
if ( b == NONE ) return a.toString(); if ( b == NONE ) return a.toString();
@ -116,7 +146,7 @@ public abstract class SemanticContext {
public SemanticContext a; public SemanticContext a;
public SemanticContext b; public SemanticContext b;
public OR() { } public OR() { }
public OR(SemanticContext a, SemanticContext b) { public OR(@NotNull SemanticContext a, @NotNull SemanticContext b) {
this.a = a; this.a = a;
this.b = b; this.b = b;
} }
@ -127,6 +157,15 @@ public abstract class SemanticContext {
return a.eval(parser, ctx) && b.eval(parser, ctx); return a.eval(parser, ctx) && b.eval(parser, ctx);
} }
public SemanticContext optimize() {
SemanticContext a_ = a.optimize();
SemanticContext b_ = b.optimize();
if ( a_ == NONE ) return b_;
if ( b_ == NONE ) return a_;
if ( a_.equals(b_) ) return a_;
return new OR(a_, b_);
}
@Override @Override
public String toString() { public String toString() {
if ( a == NONE ) return b.toString(); if ( a == NONE ) return b.toString();
@ -134,23 +173,4 @@ public abstract class SemanticContext {
return a+"||"+b; return a+"||"+b;
} }
} }
// public static SemanticContext and(AND a, Predicate b) {
// if ( a == NONE ) return new AND(b);
// if ( b == NONE ) {a.add(b); return a;}
// a.add(b);
// AND and = new SemanticContext.AND();
// if ( a instanceof AND ) {
// and.add(b);
// }
// else {
// and.add((Predicate)a);
// and.add(b);
// }
// return and;
// }
//
// public static SemanticContext or(SemanticContext a, SemanticContext b) {
// return null;
// }
} }

47
runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java
View File

@ -1,30 +1,30 @@
/* /*
[The "BSD license"] [The "BSD license"]
Copyright (c) 2011 Terence Parr Copyright (c) 2011 Terence Parr
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions modification, are permitted provided that the following conditions
are met: are met:
1. Redistributions of source code must retain the above copyright 1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright 2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution. documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products 3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission. derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package org.antlr.v4.runtime.dfa; package org.antlr.v4.runtime.dfa;
@ -79,10 +79,11 @@ public class DFAState {
public boolean isAcceptState = false; public boolean isAcceptState = false;
public int prediction; // if accept state, what ttype do we match? public int prediction; // if accept state, what ttype do we match? is "else" clause if predicated
public int ruleIndex; // if accept, exec what action? public int ruleIndex; // if accept, exec what action?
// todo: rename as unique?
public boolean complete; // all alts predict "prediction" public boolean complete; // all alts predict "prediction"
public boolean isCtxSensitive; public boolean isCtxSensitive;

53
runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java
View File

@ -1,30 +1,30 @@
/* /*
[The "BSD license"] [The "BSD license"]
Copyright (c) 2011 Terence Parr Copyright (c) 2011 Terence Parr
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions modification, are permitted provided that the following conditions
are met: are met:
1. Redistributions of source code must retain the above copyright 1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright 2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution. documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products 3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission. derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package org.antlr.v4.runtime.misc; package org.antlr.v4.runtime.misc;
@ -43,4 +43,13 @@ public class Utils {
} }
return buf.toString(); return buf.toString();
} }
public static int numNonnull(Object[] data) {
int n = 0;
if ( data == null ) return n;
for (Object o : data) {
if ( o!=null ) n++;
}
return n;
}
} }

2
tool/playground/HTMLParser.g
View File

@ -1,6 +1,6 @@
parser grammar HTMLParser; parser grammar HTMLParser;
options { tokenVocab=HTMLLexer; } options { tokenVocab=HTMLParser; }
file : ( TAG_START (starttag | endtag) | TEXT file : ( TAG_START (starttag | endtag) | TEXT
{System.out.println("TEXT "+$TEXT);} )+ EOF ; {System.out.println("TEXT "+$TEXT);} )+ EOF ;

17
tool/playground/T.g
View File

@ -1,13 +1,8 @@
grammar T; grammar T;
options {output=AST;} s : a a;
a : {_input.LT(1).equals("x")}? ID INT {System.out.println("alt 1");}
s : i=ifstat {System.out.println(_input.toString(0,_input.index()-1));} ; | {_input.LT(1).equals("y")}? ID INT {System.out.println("alt 2");}
;
ifstat : 'if' '(' expr ')' assign ;
assign : ID '=' expr ';' ;
expr : INT | ID ;
EQ : '=' ;
INT : '0'..'9'+ ;
ID : 'a'..'z'+ ; ID : 'a'..'z'+ ;
WS : (' '|'\n')+ {skip();} ; INT : '0'..'9'+;
WS : (' '|'\n') {skip();} ;

23
tool/playground/TestT.java
View File

@ -27,11 +27,8 @@
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
import org.antlr.v4.runtime.*; import org.antlr.v4.runtime.ANTLRFileStream;
import org.antlr.v4.runtime.tree.*; import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.gui.TreeViewer;
import java.util.ArrayList;
public class TestT { public class TestT {
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
@ -45,14 +42,14 @@ public class TestT {
p.setBuildParseTree(true); p.setBuildParseTree(true);
final TParser.sContext tree = p.s(); final TParser.sContext tree = p.s();
System.out.println(tree.toStringTree(p)); System.out.println(tree.toStringTree(p));
TreeViewer v = new TreeViewer(p, tree); // TreeViewer v = new TreeViewer(p, tree);
v.setHighlightedBoxColor(TreeViewer.LIGHT_RED); // v.setHighlightedBoxColor(TreeViewer.LIGHT_RED);
v.addHighlightedNodes(new ArrayList<Tree>() {{ // v.addHighlightedNodes(new ArrayList<Tree>() {{
ParseTree c0 = tree.getChild(0); // ParseTree c0 = tree.getChild(0);
add(c0); // add(c0);
add(c0.getChild(0)); // add(c0.getChild(0));
}}); // }});
v.open(); // v.open();
// tree.inspect(p); // tree.inspect(p);
// //
// ParseTreeWalker walker = new ParseTreeWalker(); // ParseTreeWalker walker = new ParseTreeWalker();

247
tool/test/org/antlr/v4/test/TestSemPredEvalParser.java
View File

@ -1,32 +1,206 @@
/*
[The "BSD license"]
Copyright (c) 2011 Terence Parr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. The name of the author may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.test; package org.antlr.v4.test;
import org.junit.Test; import org.junit.Test;
public class TestSemPredEvalParser extends BaseTest { public class TestSemPredEvalParser extends BaseTest {
@Test public void testSimple() throws Exception { @Test public void testSimple() throws Exception {
String grammar = String grammar =
"grammar T;\n" + "grammar T;\n" +
"a : {false}? ID {System.out.println(\"alt 1\");}\n" + "s : a a a;\n" + // do 3x: once in ATN, next in DFA then INT in ATN
" | {true}? ID {System.out.println(\"alt 2\");}\n" + "a : {false}? ID {System.out.println(\"alt 1\");}\n" +
" | INT {System.out.println(\"alt 3\");}\n" + " | {true}? ID {System.out.println(\"alt 2\");}\n" +
" ;\n" + " | INT {System.out.println(\"alt 3\");}\n" +
"ID : 'a'..'z'+ ;\n" + " ;\n" +
"INT : '0'..'9'+;\n" + "ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n') {skip();} ;\n"; "INT : '0'..'9'+;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
String found = execParser("T.g", grammar, "TParser", "TLexer", "a", String found = execParser("T.g", grammar, "TParser", "TLexer", "s",
"x", false); "x y 3", false);
String expecting = String expecting =
"alt 2\n" + "alt 2\n" +
"alt 2\n" + "alt 2\n" +
"alt 2\n"; "alt 3\n";
assertEquals(expecting, found); assertEquals(expecting, found);
} }
@Test public void testToLeft() throws Exception { @Test public void testOrder() throws Exception {
String grammar = // Predicates disambiguate and so we don't arbitrarily choose the first alt
"grammar T;\n" + // Here, there are n-1 predicates for n=2 alts and so we simulate
"s : a+ ;\n" + // the nth predicate as !(others). We do that by testing the
// predicates first and then try the on predicated alternatives.
// Since the 2nd alternative has a true predicate, we always choose that one
String grammar =
"grammar T;\n" +
"s : a a;\n" + // do 2x: once in ATN, next in DFA
"a : ID {System.out.println(\"alt 1\");}\n" +
" | {true}? ID {System.out.println(\"alt 2\");}\n" +
" ;\n" +
"ID : 'a'..'z'+ ;\n" +
"INT : '0'..'9'+;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
String found = execParser("T.g", grammar, "TParser", "TLexer", "s",
"x y", false);
String expecting =
"alt 2\n" +
"alt 2\n";
assertEquals(expecting, found);
}
@Test public void test2UnpredicatedAlts() throws Exception {
// We have n-2 predicates for n alternatives. We have no choice
// but to pick the first on predicated alternative if the n-2
// predicates fail.
// this should call reportInsufficientPredicates()
String grammar =
"grammar T;\n" +
"@header {" +
"import java.util.*;" +
"}" +
"@parser::members {" +
"public void reportInsufficientPredicates(int startIndex, int stopIndex,\n" +
"\t\t\t\t\t\t\t\t\t\t\t @NotNull Set<Integer> ambigAlts,\n" +
"\t\t\t\t\t\t\t\t\t\t\t @NotNull SemanticContext[] altToPred,\n" +
"\t\t\t\t\t\t\t\t\t\t\t @NotNull OrderedHashSet<ATNConfig> configs)\n" +
"{\n" +
"System.out.println(\"reportInsufficientPredicates\");\n" +
"}\n" +
"}\n"+
"s : a a;\n" + // do 2x: once in ATN, next in DFA
"a : ID {System.out.println(\"alt 1\");}\n" +
" | ID {System.out.println(\"alt 2\");}\n" +
" | {false}? ID {System.out.println(\"alt 3\");}\n" +
" ;\n" +
"ID : 'a'..'z'+ ;\n" +
"INT : '0'..'9'+;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
String found = execParser("T.g", grammar, "TParser", "TLexer", "s",
"x y", false);
String expecting =
"reportInsufficientPredicates\n" +
"alt 1\n" +
"alt 1\n";
assertEquals(expecting, found);
}
@Test public void test2UnpredicatedAltsAndOneOrthogonalAlt() throws Exception {
// We have n-2 predicates for n alternatives. We have no choice
// but to pick the first on predicated alternative if the n-2
// predicates fail.
// this should call reportInsufficientPredicates()
String grammar =
"grammar T;\n" +
"@header {" +
"import java.util.*;" +
"}" +
"@parser::members {" +
"public void reportInsufficientPredicates(int startIndex, int stopIndex,\n" +
"\t\t\t\t\t\t\t\t\t\t\t @NotNull Set<Integer> ambigAlts,\n" +
"\t\t\t\t\t\t\t\t\t\t\t @NotNull SemanticContext[] altToPred,\n" +
"\t\t\t\t\t\t\t\t\t\t\t @NotNull OrderedHashSet<ATNConfig> configs)\n" +
"{\n" +
"System.out.println(\"reportInsufficientPredicates\");\n" +
"}\n" +
"}\n"+
"s : a a a;\n" +
"a : INT {System.out.println(\"alt 1\");}\n" +
" | ID {System.out.println(\"alt 2\");}\n" +
" | ID {System.out.println(\"alt 3\");}\n" +
" | {false}? ID {System.out.println(\"alt 4\");}\n" +
" ;\n" +
"ID : 'a'..'z'+ ;\n" +
"INT : '0'..'9'+;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
String found = execParser("T.g", grammar, "TParser", "TLexer", "s",
"34 x y", false);
String expecting =
"alt 1\n" +
"reportInsufficientPredicates\n" +
"alt 2\n" +
"alt 2\n";
assertEquals(expecting, found);
}
@Test public void testRewindBeforePredEval() throws Exception {
// The parser consumes ID and moves to the 2nd token INT.
// To properly evaluate the predicates after matching ID INT,
// we must correctly see come back to starting index so LT(1) works
String grammar =
"grammar T;\n" +
"s : a a;\n" +
"a : {_input.LT(1).getText().equals(\"x\")}? ID INT {System.out.println(\"alt 1\");}\n" +
" | {_input.LT(1).getText().equals(\"y\")}? ID INT {System.out.println(\"alt 2\");}\n" +
" ;\n" +
"ID : 'a'..'z'+ ;\n" +
"INT : '0'..'9'+;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
String found = execParser("T.g", grammar, "TParser", "TLexer", "s",
"y 3 x 4", false);
String expecting =
"alt 2\n" +
"alt 1\n";
assertEquals(expecting, found);
}
@Test public void testNoTruePredsThrowsNoViableAlt() throws Exception {
// checks that we throw exception if all alts
// are covered with a predicate and none succeeds
String grammar =
"grammar T;\n" +
"s : a a;\n" +
"a : {false}? ID INT {System.out.println(\"alt 1\");}\n" +
" | {false}? ID INT {System.out.println(\"alt 2\");}\n" +
" ;\n" +
"ID : 'a'..'z'+ ;\n" +
"INT : '0'..'9'+;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
execParser("T.g", grammar, "TParser", "TLexer", "s",
"y 3 x 4", false);
String expecting = "line 1:0 no viable alternative at input 'y'\n";
String result = stderrDuringParse;
assertEquals(expecting, result);
}
// TODO: test predicates that are validating predicates; not ambiguous decisions
@Test public void testToLeft() throws Exception {
String grammar =
"grammar T;\n" +
"s : a+ ;\n" +
"a : {false}? ID {System.out.println(\"alt 1\");}\n" + "a : {false}? ID {System.out.println(\"alt 1\");}\n" +
" | {true}? ID {System.out.println(\"alt 2\");}\n" + " | {true}? ID {System.out.println(\"alt 2\");}\n" +
" ;\n" + " ;\n" +
@ -151,31 +325,6 @@ public class TestSemPredEvalParser extends BaseTest {
assertEquals(expecting, found); assertEquals(expecting, found);
} }
@Test public void testToRightWithVaryingPredicate() throws Exception {
// alternate predicted alt to ensure DFA doesn't cache
String grammar =
"grammar T;\n" +
"@members {int i=0;}\n" +
"s : ({i++; System.out.println(\"i=\"+i);} a)+ ;\n" +
"a : ID {i \\% 2 == 0}? {System.out.println(\"alt 1\");}\n" +
" | ID {i \\% 2 != 0}? {System.out.println(\"alt 2\");}\n" +
" ;\n" +
"ID : 'a'..'z'+ ;\n" +
"INT : '0'..'9'+;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
String found = execParser("T.g", grammar, "TParser", "TLexer", "s",
"x x y", false);
String expecting =
"i=1\n" +
"alt 2\n" +
"i=2\n" +
"alt 1\n" +
"i=3\n" +
"alt 2\n";
assertEquals(expecting, found);
}
/** During a global follow operation, we still execute semantic /** During a global follow operation, we still execute semantic
* predicates as long as they are not dependent on local context * predicates as long as they are not dependent on local context
*/ */
@ -196,10 +345,6 @@ public class TestSemPredEvalParser extends BaseTest {
String found = execParser("T.g", grammar, "TParser", "TLexer", "s", String found = execParser("T.g", grammar, "TParser", "TLexer", "s",
"a!", false); "a!", false);
String expecting = String expecting =
"eval=true\n" + // do p(true), p(false) once during s0 computation from epsilon edge in e
"eval=false\n" +
"eval=true\n" + // do them again during closure after passing ID in e
"eval=false\n" +
"eval=true\n" + // now we are parsing "eval=true\n" + // now we are parsing
"parse\n"; "parse\n";
assertEquals(expecting, found); assertEquals(expecting, found);