forked from jasder/antlr
add tests, refactor get-all-parse-tree stuff. add Trees support routines.
This commit is contained in:
parrt
parent
e0c6210d22
commit
a9ca2efae5
|
|
@ -7,7 +7,7 @@
|
|||
<component name="ProjectKey">
|
||||
<option name="state" value="project://e2804f05-5315-4fc6-a121-c522a6c26470" />
|
||||
</component>
|
||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_6" default="false" assert-keyword="true" jdk-15="true">
|
||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_6" default="false" assert-keyword="true" jdk-15="true" project-jdk-name="1.7" project-jdk-type="JavaSDK">
|
||||
<output url="file://$PROJECT_DIR$/out" />
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -30,9 +30,12 @@
|
|||
|
||||
package org.antlr.v4.runtime.tree;
|
||||
|
||||
import org.antlr.v4.runtime.CommonToken;
|
||||
import org.antlr.v4.runtime.Parser;
|
||||
import org.antlr.v4.runtime.ParserRuleContext;
|
||||
import org.antlr.v4.runtime.Token;
|
||||
import org.antlr.v4.runtime.misc.Interval;
|
||||
import org.antlr.v4.runtime.misc.Predicate;
|
||||
import org.antlr.v4.runtime.misc.Utils;
|
||||
import org.antlr.v4.runtime.tree.gui.TreePostScriptGenerator;
|
||||
import org.antlr.v4.runtime.tree.gui.TreeTextProvider;
|
||||
|
|
@ -178,6 +181,8 @@ public class Trees {
|
|||
|
||||
/** Return a list of all ancestors of this node. The first node of
|
||||
* list is the root and the last is the parent of this node.
|
||||
*
|
||||
* @since 4.5.1
|
||||
*/
|
||||
public static List<? extends Tree> getAncestors(Tree t) {
|
||||
if ( t.getParent()==null ) return Collections.emptyList();
|
||||
|
|
@ -237,17 +242,26 @@ public class Trees {
|
|||
}
|
||||
}
|
||||
|
||||
public static List<ParseTree> descendants(ParseTree t){
|
||||
/** Get all descendents; includes t itself.
|
||||
*
|
||||
* @since 4.5.1
|
||||
*/
|
||||
public static List<ParseTree> getDescendants(ParseTree t) {
|
||||
List<ParseTree> nodes = new ArrayList<ParseTree>();
|
||||
nodes.add(t);
|
||||
|
||||
int n = t.getChildCount();
|
||||
for (int i = 0 ; i < n ; i++){
|
||||
nodes.addAll(descendants(t.getChild(i)));
|
||||
nodes.addAll(getDescendants(t.getChild(i)));
|
||||
}
|
||||
return nodes;
|
||||
}
|
||||
|
||||
/** @deprecated */
|
||||
public static List<ParseTree> descendants(ParseTree t) {
|
||||
return getDescendants(t);
|
||||
}
|
||||
|
||||
/** Find smallest subtree of t enclosing range startTokenIndex..stopTokenIndex
|
||||
* inclusively using postorder traversal. Recursive depth-first-search.
|
||||
*
|
||||
|
|
@ -275,6 +289,47 @@ public class Trees {
|
|||
return null;
|
||||
}
|
||||
|
||||
/** Replace any subtree siblings of root that are completely to left
|
||||
* or right of lookahead range with a CommonToken(Token.INVALID_TYPE,"...")
|
||||
* node. The source interval for t is not altered to suit smaller range!
|
||||
*
|
||||
* WARNING: destructive to t.
|
||||
*
|
||||
* @since 4.5.1
|
||||
*/
|
||||
public static void stripChildrenOutOfRange(ParserRuleContext t,
|
||||
ParserRuleContext root,
|
||||
int startIndex,
|
||||
int stopIndex)
|
||||
{
|
||||
if ( t==null ) return;
|
||||
for (int i = 0; i < t.getChildCount(); i++) {
|
||||
ParseTree child = t.getChild(i);
|
||||
Interval range = child.getSourceInterval();
|
||||
if ( child instanceof ParserRuleContext && (range.b < startIndex || range.a > stopIndex) ) {
|
||||
if ( isAncestorOf(child, root) ) { // replace only if subtree doesn't have displayed root
|
||||
CommonToken abbrev = new CommonToken(Token.INVALID_TYPE, "...");
|
||||
t.children.set(i, new TerminalNodeImpl(abbrev));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Return first node satisfying the pred
|
||||
*
|
||||
* @since 4.5.1
|
||||
*/
|
||||
public static Tree findNodeSuchThat(Tree t, Predicate<Tree> pred) {
|
||||
if ( pred.apply(t) ) return t;
|
||||
|
||||
int n = t.getChildCount();
|
||||
for (int i = 0 ; i < n ; i++){
|
||||
Tree u = findNodeSuchThat(t.getChild(i), pred);
|
||||
if ( u!=null ) return u;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private Trees() {
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,6 @@ public class XPathWildcardAnywhereElement extends XPathElement {
|
|||
@Override
|
||||
public Collection<ParseTree> evaluate(ParseTree t) {
|
||||
if ( invert ) return new ArrayList<ParseTree>(); // !* is weird but valid (empty)
|
||||
return Trees.descendants(t);
|
||||
return Trees.getDescendants(t);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
package org.antlr.v4.tool;
|
||||
|
||||
import org.antlr.v4.runtime.BailErrorStrategy;
|
||||
import org.antlr.v4.runtime.DefaultErrorStrategy;
|
||||
import org.antlr.v4.runtime.InputMismatchException;
|
||||
import org.antlr.v4.runtime.InterpreterRuleContext;
|
||||
import org.antlr.v4.runtime.Parser;
|
||||
import org.antlr.v4.runtime.ParserInterpreter;
|
||||
import org.antlr.v4.runtime.ParserRuleContext;
|
||||
import org.antlr.v4.runtime.RecognitionException;
|
||||
import org.antlr.v4.runtime.Token;
|
||||
import org.antlr.v4.runtime.TokenStream;
|
||||
import org.antlr.v4.runtime.Vocabulary;
|
||||
import org.antlr.v4.runtime.atn.ATN;
|
||||
|
|
@ -64,7 +67,7 @@ public class GrammarParserInterpreter extends ParserInterpreter {
|
|||
input);
|
||||
this.g = g;
|
||||
decisionStatesThatSetOuterAltNumInContext = findOuterMostDecisionStates();
|
||||
stateToAltsMap = new int[g.atn.getNumberOfDecisions()][];
|
||||
stateToAltsMap = new int[g.atn.states.size()][];
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -270,27 +273,7 @@ public class GrammarParserInterpreter extends ParserInterpreter {
|
|||
{
|
||||
List<ParserRuleContext> trees = new ArrayList<ParserRuleContext>();
|
||||
// Create a new parser interpreter to parse the ambiguous subphrase
|
||||
ParserInterpreter parser;
|
||||
if (originalParser instanceof ParserInterpreter) {
|
||||
parser = new GrammarParserInterpreter(g, originalParser.getATN(), originalParser.getTokenStream());
|
||||
}
|
||||
else { // must've been a generated parser
|
||||
char[] serializedAtn = ATNSerializer.getSerializedAsChars(originalParser.getATN());
|
||||
ATN deserialized = new ATNDeserializer().deserialize(serializedAtn);
|
||||
parser = new ParserInterpreter(originalParser.getGrammarFileName(),
|
||||
originalParser.getVocabulary(),
|
||||
Arrays.asList(originalParser.getRuleNames()),
|
||||
deserialized,
|
||||
tokens);
|
||||
}
|
||||
|
||||
parser.setInputStream(tokens);
|
||||
|
||||
// Make sure that we don't get any error messages from using this temporary parser
|
||||
parser.setErrorHandler(new BailErrorStrategy());
|
||||
parser.removeErrorListeners();
|
||||
parser.removeParseListeners();
|
||||
parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);
|
||||
ParserInterpreter parser = getAmbuityParserInterpreter(g, originalParser, tokens);
|
||||
|
||||
// get ambig trees
|
||||
int alt = alts.nextSetBit(0);
|
||||
|
|
@ -315,4 +298,126 @@ public class GrammarParserInterpreter extends ParserInterpreter {
|
|||
return trees;
|
||||
}
|
||||
|
||||
// we must parse the entire input now with decision overrides
|
||||
// we cannot parse a subset because it could be that a decision
|
||||
// above our decision of interest needs to read way past
|
||||
// lookaheadInfo.stopIndex. It seems like there is no escaping
|
||||
// the use of a full and complete token stream if we are
|
||||
// resetting to token index 0 and re-parsing from the start symbol.
|
||||
// It's not easy to restart parsing somewhere in the middle like a
|
||||
// continuation because our call stack does not match the
|
||||
// tree stack because of left recursive rule rewriting. grrrr!
|
||||
public static List<ParserRuleContext> getLookaheadParseTrees(Grammar g,
|
||||
ParserInterpreter originalParser,
|
||||
TokenStream tokens,
|
||||
int startRuleIndex,
|
||||
int decision,
|
||||
int startIndex,
|
||||
int stopIndex)
|
||||
{
|
||||
List<ParserRuleContext> trees = new ArrayList<ParserRuleContext>();
|
||||
// Create a new parser interpreter to parse the ambiguous subphrase
|
||||
ParserInterpreter parser = getAmbuityParserInterpreter(g, originalParser, tokens);
|
||||
BailButConsumeErrorStrategy errorHandler = new BailButConsumeErrorStrategy();
|
||||
parser.setErrorHandler(errorHandler);
|
||||
|
||||
DecisionState decisionState = originalParser.getATN().decisionToState.get(decision);
|
||||
|
||||
for (int alt=1; alt<=decisionState.getTransitions().length; alt++) {
|
||||
// re-parse entire input for all ambiguous alternatives
|
||||
// (don't have to do first as it's been parsed, but do again for simplicity
|
||||
// using this temp parser.)
|
||||
parser.reset();
|
||||
parser.addDecisionOverride(decision, startIndex, alt);
|
||||
ParserRuleContext tt = parser.parse(startRuleIndex);
|
||||
int stopTreeAt = stopIndex;
|
||||
if ( errorHandler.firstErrorTokenIndex>=0 ) {
|
||||
stopTreeAt = errorHandler.firstErrorTokenIndex; // cut off rest at first error
|
||||
}
|
||||
ParserRuleContext subtree =
|
||||
Trees.getRootOfSubtreeEnclosingRegion(tt,
|
||||
startIndex,
|
||||
stopTreeAt);
|
||||
// Use higher of overridden decision tree or tree enclosing all tokens
|
||||
if ( Trees.isAncestorOf(parser.getOverrideDecisionRoot(), subtree) ) {
|
||||
subtree = parser.getOverrideDecisionRoot();
|
||||
}
|
||||
Trees.stripChildrenOutOfRange(subtree, parser.getOverrideDecisionRoot(), startIndex, stopTreeAt);
|
||||
trees.add(subtree);
|
||||
}
|
||||
|
||||
return trees;
|
||||
}
|
||||
|
||||
/** Derive a new parser from an old one that has knowledge of the grammar.
|
||||
* The Grammar object is used to correctly compute outer alternative
|
||||
* numbers for parse tree nodes.
|
||||
* @param g
|
||||
* @param originalParser
|
||||
* @param tokens
|
||||
* @return
|
||||
*/
|
||||
public static ParserInterpreter getAmbuityParserInterpreter(Grammar g, Parser originalParser, TokenStream tokens) {
|
||||
ParserInterpreter parser;
|
||||
if (originalParser instanceof ParserInterpreter) {
|
||||
parser = new GrammarParserInterpreter(g, originalParser.getATN(), originalParser.getTokenStream());
|
||||
}
|
||||
else { // must've been a generated parser
|
||||
char[] serializedAtn = ATNSerializer.getSerializedAsChars(originalParser.getATN());
|
||||
ATN deserialized = new ATNDeserializer().deserialize(serializedAtn);
|
||||
parser = new ParserInterpreter(originalParser.getGrammarFileName(),
|
||||
originalParser.getVocabulary(),
|
||||
Arrays.asList(originalParser.getRuleNames()),
|
||||
deserialized,
|
||||
tokens);
|
||||
}
|
||||
|
||||
parser.setInputStream(tokens);
|
||||
|
||||
// Make sure that we don't get any error messages from using this temporary parser
|
||||
parser.setErrorHandler(new BailErrorStrategy());
|
||||
parser.removeErrorListeners();
|
||||
parser.removeParseListeners();
|
||||
parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);
|
||||
return parser;
|
||||
}
|
||||
|
||||
/** We want to stop and track the first air but we cannot bail out like
|
||||
* {@link BailErrorStrategy} as consume() constructs trees. We make sure
|
||||
* to create an error node during recovery with this strategy. We
|
||||
* consume() 1 token during the "bail out of rule" mechanism in recover()
|
||||
* and let it fall out of the rule to finish constructing trees. For
|
||||
* recovery in line, we throw InputMismatchException to engage recover().
|
||||
*/
|
||||
public static class BailButConsumeErrorStrategy extends DefaultErrorStrategy {
|
||||
public int firstErrorTokenIndex = -1;
|
||||
@Override
|
||||
public void recover(Parser recognizer, RecognitionException e) {
|
||||
int errIndex = recognizer.getInputStream().index();
|
||||
if ( firstErrorTokenIndex == -1 ) {
|
||||
firstErrorTokenIndex = errIndex; // latch
|
||||
}
|
||||
// System.err.println("recover: error at " + errIndex);
|
||||
TokenStream input = recognizer.getInputStream();
|
||||
if ( input.index()<input.size()-1 ) { // don't consume() eof
|
||||
recognizer.consume(); // just kill this bad token and let it continue.
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Token recoverInline(Parser recognizer) throws RecognitionException {
|
||||
int errIndex = recognizer.getInputStream().index();
|
||||
if ( firstErrorTokenIndex == -1 ) {
|
||||
firstErrorTokenIndex = errIndex; // latch
|
||||
}
|
||||
// System.err.println("recoverInline: error at " + errIndex);
|
||||
InputMismatchException e = new InputMismatchException(recognizer);
|
||||
// TokenStream input = recognizer.getInputStream(); // seek EOF
|
||||
// input.seek(input.size() - 1);
|
||||
throw e;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void sync(Parser recognizer) { } // don't consume anything; let it fail later
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
package org.antlr.v4.test.tool;
|
||||
|
||||
import org.antlr.v4.runtime.tree.ErrorNode;
|
||||
import org.antlr.v4.runtime.tree.Tree;
|
||||
import org.antlr.v4.runtime.tree.Trees;
|
||||
import org.antlr.v4.runtime.tree.gui.TreeTextProvider;
|
||||
import org.antlr.v4.tool.GrammarInterpreterRuleContext;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
public class InterpreterTreeTextProvider implements TreeTextProvider {
|
||||
public List<String> ruleNames;
|
||||
public InterpreterTreeTextProvider(String[] ruleNames) {this.ruleNames = Arrays.asList(ruleNames);}
|
||||
|
||||
@Override
|
||||
public String getText(Tree node) {
|
||||
if ( node==null ) return "null";
|
||||
String nodeText = Trees.getNodeText(node, ruleNames);
|
||||
if ( node instanceof GrammarInterpreterRuleContext) {
|
||||
GrammarInterpreterRuleContext ctx = (GrammarInterpreterRuleContext) node;
|
||||
return nodeText+":"+ctx.getOuterAltNum();
|
||||
}
|
||||
if ( node instanceof ErrorNode) {
|
||||
return "<error "+nodeText+">";
|
||||
}
|
||||
return nodeText;
|
||||
}
|
||||
}
|
||||
|
|
@ -14,6 +14,7 @@ import org.antlr.v4.runtime.atn.PredictionMode;
|
|||
import org.antlr.v4.runtime.atn.RuleStartState;
|
||||
import org.antlr.v4.runtime.atn.Transition;
|
||||
import org.antlr.v4.runtime.tree.ParseTree;
|
||||
import org.antlr.v4.runtime.tree.Trees;
|
||||
import org.antlr.v4.tool.Grammar;
|
||||
import org.antlr.v4.tool.GrammarParserInterpreter;
|
||||
import org.antlr.v4.tool.LexerGrammar;
|
||||
|
|
@ -38,8 +39,8 @@ public class TestAmbigParseTrees {
|
|||
"x : B ; \n",
|
||||
lg);
|
||||
|
||||
testInterpAtSpecificAlt(lg, g, "s", 1, "abc", "(s a (x b) c)");
|
||||
testInterpAtSpecificAlt(lg, g, "s", 2, "abc", "(s a b c)");
|
||||
testInterpAtSpecificAlt(lg, g, "s", 1, "abc", "(s:1 a (x:1 b) c)");
|
||||
testInterpAtSpecificAlt(lg, g, "s", 2, "abc", "(s:2 a b c)");
|
||||
}
|
||||
|
||||
@Test public void testAmbigAltsAtRoot() throws Exception {
|
||||
|
|
@ -60,12 +61,13 @@ public class TestAmbigParseTrees {
|
|||
String input = "abc";
|
||||
String expectedAmbigAlts = "{1, 2}";
|
||||
int decision = 0;
|
||||
String expectedOverallTree = "(s a (x b) c)";
|
||||
String[] expectedParseTrees = {"(s a (x b) c)","(s a b c)"};
|
||||
String expectedOverallTree = "(s:1 a (x:1 b) c)";
|
||||
String[] expectedParseTrees = {"(s:1 a (x:1 b) c)",
|
||||
"(s:2 a b c)"};
|
||||
|
||||
testInterp(lg, g, startRule, input, decision,
|
||||
expectedAmbigAlts,
|
||||
expectedOverallTree, expectedParseTrees);
|
||||
testAmbiguousTrees(lg, g, startRule, input, decision,
|
||||
expectedAmbigAlts,
|
||||
expectedOverallTree, expectedParseTrees);
|
||||
}
|
||||
|
||||
@Test public void testAmbigAltsNotAtRoot() throws Exception {
|
||||
|
|
@ -76,24 +78,25 @@ public class TestAmbigParseTrees {
|
|||
"C : 'c' ;\n");
|
||||
Grammar g = new Grammar(
|
||||
"parser grammar T;\n" +
|
||||
"s : a ;" +
|
||||
"a : b ;" +
|
||||
"b : A x C" +
|
||||
"s : x ;" +
|
||||
"x : y ;" +
|
||||
"y : A z C" +
|
||||
" | A B C" +
|
||||
" ;" +
|
||||
"x : B ; \n",
|
||||
"z : B ; \n",
|
||||
lg);
|
||||
|
||||
String startRule = "s";
|
||||
String input = "abc";
|
||||
String expectedAmbigAlts = "{1, 2}";
|
||||
int decision = 0;
|
||||
String expectedOverallTree = "(s (a (b a (x b) c)))";
|
||||
String[] expectedParseTrees = {"(b a (x b) c)","(b a b c)"};
|
||||
String expectedOverallTree = "(s:1 (x:1 (y:1 a (z:1 b) c)))";
|
||||
String[] expectedParseTrees = {"(y:1 a (z:1 b) c)",
|
||||
"(y:2 a b c)"};
|
||||
|
||||
testInterp(lg, g, startRule, input, decision,
|
||||
expectedAmbigAlts,
|
||||
expectedOverallTree, expectedParseTrees);
|
||||
testAmbiguousTrees(lg, g, startRule, input, decision,
|
||||
expectedAmbigAlts,
|
||||
expectedOverallTree, expectedParseTrees);
|
||||
}
|
||||
|
||||
@Test public void testAmbigAltDipsIntoOuterContextToRoot() throws Exception {
|
||||
|
|
@ -104,7 +107,6 @@ public class TestAmbigParseTrees {
|
|||
"DOT : '.' ;\n");
|
||||
Grammar g = new Grammar(
|
||||
"parser grammar T;\n" +
|
||||
// "s : e ;\n"+
|
||||
"e : p (DOT ID)* ;\n"+
|
||||
"p : SELF" +
|
||||
" | SELF DOT ID" +
|
||||
|
|
@ -114,13 +116,14 @@ public class TestAmbigParseTrees {
|
|||
String startRule = "e";
|
||||
String input = "self.x";
|
||||
String expectedAmbigAlts = "{1, 2}";
|
||||
int decision = 1; // decision in s
|
||||
String expectedOverallTree = "(e (p self) . x)";
|
||||
String[] expectedParseTrees = {"(e (p self) . x)","(p self . x)"};
|
||||
int decision = 1; // decision in p
|
||||
String expectedOverallTree = "(e:1 (p:1 self) . x)";
|
||||
String[] expectedParseTrees = {"(e:1 (p:1 self) . x)",
|
||||
"(p:2 self . x)"};
|
||||
|
||||
testInterp(lg, g, startRule, input, decision,
|
||||
expectedAmbigAlts,
|
||||
expectedOverallTree, expectedParseTrees);
|
||||
testAmbiguousTrees(lg, g, startRule, input, decision,
|
||||
expectedAmbigAlts,
|
||||
expectedOverallTree, expectedParseTrees);
|
||||
}
|
||||
|
||||
@Test public void testAmbigAltDipsIntoOuterContextBelowRoot() throws Exception {
|
||||
|
|
@ -141,13 +144,14 @@ public class TestAmbigParseTrees {
|
|||
String startRule = "s";
|
||||
String input = "self.x";
|
||||
String expectedAmbigAlts = "{1, 2}";
|
||||
int decision = 1; // decision in s
|
||||
String expectedOverallTree = "(s (e (p self) . x))";
|
||||
String[] expectedParseTrees = {"(e (p self) . x)","(p self . x)"};
|
||||
int decision = 1; // decision in p
|
||||
String expectedOverallTree = "(s:1 (e:1 (p:1 self) . x))";
|
||||
String[] expectedParseTrees = {"(e:1 (p:1 self) . x)", // shouldn't include s
|
||||
"(p:2 self . x)"}; // shouldn't include e
|
||||
|
||||
testInterp(lg, g, startRule, input, decision,
|
||||
expectedAmbigAlts,
|
||||
expectedOverallTree, expectedParseTrees);
|
||||
testAmbiguousTrees(lg, g, startRule, input, decision,
|
||||
expectedAmbigAlts,
|
||||
expectedOverallTree, expectedParseTrees);
|
||||
}
|
||||
|
||||
@Test public void testAmbigAltInLeftRecursiveBelowStartRule() throws Exception {
|
||||
|
|
@ -168,13 +172,14 @@ public class TestAmbigParseTrees {
|
|||
String startRule = "s";
|
||||
String input = "self.x";
|
||||
String expectedAmbigAlts = "{1, 2}";
|
||||
int decision = 1; // decision in s
|
||||
String expectedOverallTree = "(s (e (e (p self)) . x))";
|
||||
String[] expectedParseTrees = {"(e (e (p self)) . x)","(p self . x)"};
|
||||
int decision = 1; // decision in p
|
||||
String expectedOverallTree = "(s:1 (e:2 (e:1 (p:1 self)) . x))";
|
||||
String[] expectedParseTrees = {"(e:2 (e:1 (p:1 self)) . x)",
|
||||
"(p:2 self . x)"};
|
||||
|
||||
testInterp(lg, g, startRule, input, decision,
|
||||
expectedAmbigAlts,
|
||||
expectedOverallTree, expectedParseTrees);
|
||||
testAmbiguousTrees(lg, g, startRule, input, decision,
|
||||
expectedAmbigAlts,
|
||||
expectedOverallTree, expectedParseTrees);
|
||||
}
|
||||
|
||||
@Test public void testAmbigAltInLeftRecursiveStartRule() throws Exception {
|
||||
|
|
@ -194,21 +199,24 @@ public class TestAmbigParseTrees {
|
|||
String startRule = "e";
|
||||
String input = "self.x";
|
||||
String expectedAmbigAlts = "{1, 2}";
|
||||
int decision = 1; // decision in s
|
||||
String expectedOverallTree = "(e (e (p self)) . x)";
|
||||
String[] expectedParseTrees = {"(e (e (p self)) . x)","(p self . x)"};
|
||||
int decision = 1; // decision in p
|
||||
String expectedOverallTree = "(e:2 (e:1 (p:1 self)) . x)";
|
||||
String[] expectedParseTrees = {"(e:2 (e:1 (p:1 self)) . x)",
|
||||
"(p:2 self . x)"}; // shows just enough for self.x
|
||||
|
||||
testInterp(lg, g, startRule, input, decision,
|
||||
expectedAmbigAlts,
|
||||
expectedOverallTree, expectedParseTrees);
|
||||
testAmbiguousTrees(lg, g, startRule, input, decision,
|
||||
expectedAmbigAlts,
|
||||
expectedOverallTree, expectedParseTrees);
|
||||
}
|
||||
|
||||
public void testInterp(LexerGrammar lg, Grammar g,
|
||||
String startRule, String input, int decision,
|
||||
String expectedAmbigAlts,
|
||||
String overallTree,
|
||||
String[] expectedParseTrees)
|
||||
public void testAmbiguousTrees(LexerGrammar lg, Grammar g,
|
||||
String startRule, String input, int decision,
|
||||
String expectedAmbigAlts,
|
||||
String overallTree,
|
||||
String[] expectedParseTrees)
|
||||
{
|
||||
InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(g.getRuleNames());
|
||||
|
||||
LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
|
||||
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
|
||||
final GrammarParserInterpreter parser = g.createGrammarParserInterpreter(tokens);
|
||||
|
|
@ -218,7 +226,7 @@ public class TestAmbigParseTrees {
|
|||
// PARSE
|
||||
int ruleIndex = g.rules.get(startRule).index;
|
||||
ParserRuleContext parseTree = parser.parse(ruleIndex);
|
||||
assertEquals(overallTree, parseTree.toStringTree(parser));
|
||||
assertEquals(overallTree, Trees.toStringTree(parseTree, nodeTextProvider));
|
||||
System.out.println();
|
||||
|
||||
DecisionInfo[] decisionInfo = parser.getParseInfo().getDecisionInfo();
|
||||
|
|
@ -230,17 +238,17 @@ public class TestAmbigParseTrees {
|
|||
GrammarParserInterpreter.getAllPossibleParseTrees(g,
|
||||
parser,
|
||||
tokens,
|
||||
ambiguityInfo.decision,
|
||||
decision,
|
||||
ambiguityInfo.ambigAlts,
|
||||
ambiguityInfo.startIndex,
|
||||
ambiguityInfo.stopIndex,
|
||||
ruleIndex);
|
||||
assertEquals(expectedAmbigAlts, ambiguityInfo.ambigAlts.toString());
|
||||
|
||||
assertEquals(ambiguityInfo.ambigAlts.cardinality(), ambiguousParseTrees.size());
|
||||
|
||||
for (int i = 0; i<ambiguousParseTrees.size(); i++) {
|
||||
ParserRuleContext t = ambiguousParseTrees.get(i);
|
||||
assertEquals(expectedParseTrees[i], t.toStringTree(parser));
|
||||
assertEquals(expectedParseTrees[i], Trees.toStringTree(t, nodeTextProvider));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -251,7 +259,7 @@ public class TestAmbigParseTrees {
|
|||
{
|
||||
LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
|
||||
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
|
||||
ParserInterpreter parser = g.createParserInterpreter(tokens);
|
||||
ParserInterpreter parser = g.createGrammarParserInterpreter(tokens);
|
||||
RuleStartState ruleStartState = g.atn.ruleToStartState[g.getRule(startRule).index];
|
||||
Transition tr = ruleStartState.transition(0);
|
||||
ATNState t2 = tr.target;
|
||||
|
|
@ -260,6 +268,7 @@ public class TestAmbigParseTrees {
|
|||
}
|
||||
parser.addDecisionOverride(((DecisionState)t2).decision, 0, startAlt);
|
||||
ParseTree t = parser.parse(g.rules.get(startRule).index);
|
||||
assertEquals(expectedParseTree, t.toStringTree(parser));
|
||||
InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(g.getRuleNames());
|
||||
assertEquals(expectedParseTree, Trees.toStringTree(t, nodeTextProvider));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,113 @@
|
|||
package org.antlr.v4.test.tool;
|
||||
|
||||
import org.antlr.v4.runtime.ANTLRInputStream;
|
||||
import org.antlr.v4.runtime.CommonTokenStream;
|
||||
import org.antlr.v4.runtime.InterpreterRuleContext;
|
||||
import org.antlr.v4.runtime.LexerInterpreter;
|
||||
import org.antlr.v4.runtime.tree.ParseTree;
|
||||
import org.antlr.v4.runtime.tree.Trees;
|
||||
import org.antlr.v4.tool.Grammar;
|
||||
import org.antlr.v4.tool.GrammarParserInterpreter;
|
||||
import org.antlr.v4.tool.LexerGrammar;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
/** Tests to ensure GrammarParserInterpreter subclass of ParserInterpreter
|
||||
* hasn't messed anything up.
|
||||
*/
|
||||
public class TestGrammarParserInterpreter {
|
||||
public static final String lexerText = "lexer grammar L;\n" +
|
||||
"PLUS : '+' ;\n" +
|
||||
"MULT : '*' ;\n" +
|
||||
"ID : [a-z]+ ;\n" +
|
||||
"INT : [0-9]+ ;\n" +
|
||||
"WS : [ \\r\\t\\n]+ ;\n";
|
||||
|
||||
@Test
|
||||
public void testAlts() throws Exception {
|
||||
LexerGrammar lg = new LexerGrammar(lexerText);
|
||||
Grammar g = new Grammar(
|
||||
"parser grammar T;\n" +
|
||||
"s : ID\n"+
|
||||
" | INT{;}\n"+
|
||||
" ;\n",
|
||||
lg);
|
||||
testInterp(lg, g, "s", "a", "(s:1 a)");
|
||||
testInterp(lg, g, "s", "3", "(s:2 3)");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAltsAsSet() throws Exception {
|
||||
LexerGrammar lg = new LexerGrammar(lexerText);
|
||||
Grammar g = new Grammar(
|
||||
"parser grammar T;\n" +
|
||||
"s : ID\n"+
|
||||
" | INT\n"+
|
||||
" ;\n",
|
||||
lg);
|
||||
testInterp(lg, g, "s", "a", "(s:1 a)");
|
||||
testInterp(lg, g, "s", "3", "(s:1 3)");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAltsWithLabels() throws Exception {
|
||||
LexerGrammar lg = new LexerGrammar(lexerText);
|
||||
Grammar g = new Grammar(
|
||||
"parser grammar T;\n" +
|
||||
"s : ID # foo\n" +
|
||||
" | INT # bar\n" +
|
||||
" ;\n",
|
||||
lg);
|
||||
// it won't show the labels here because my simple node text provider above just shows the alternative
|
||||
testInterp(lg, g, "s", "a", "(s:1 a)");
|
||||
testInterp(lg, g, "s", "3", "(s:2 3)");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOneAlt() throws Exception {
|
||||
LexerGrammar lg = new LexerGrammar(lexerText);
|
||||
Grammar g = new Grammar(
|
||||
"parser grammar T;\n" +
|
||||
"s : ID\n"+
|
||||
" ;\n",
|
||||
lg);
|
||||
testInterp(lg, g, "s", "a", "(s:1 a)");
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testLeftRecursionWithMultiplePrimaryAndRecursiveOps() throws Exception {
|
||||
LexerGrammar lg = new LexerGrammar(lexerText);
|
||||
Grammar g = new Grammar(
|
||||
"parser grammar T;\n" +
|
||||
"s : e EOF ;\n" +
|
||||
"e : e MULT e\n" +
|
||||
" | e PLUS e\n" +
|
||||
" | INT\n" +
|
||||
" | ID\n" +
|
||||
" ;\n",
|
||||
lg);
|
||||
|
||||
testInterp(lg, g, "s", "a", "(s:1 (e:4 a) <EOF>)");
|
||||
testInterp(lg, g, "e", "a", "(e:4 a)");
|
||||
testInterp(lg, g, "e", "34", "(e:3 34)");
|
||||
testInterp(lg, g, "e", "a+1", "(e:2 (e:4 a) + (e:3 1))");
|
||||
testInterp(lg, g, "e", "1+2*a", "(e:2 (e:3 1) + (e:1 (e:3 2) * (e:4 a)))");
|
||||
}
|
||||
|
||||
InterpreterRuleContext testInterp(LexerGrammar lg, Grammar g,
|
||||
String startRule, String input,
|
||||
String expectedParseTree)
|
||||
{
|
||||
LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
|
||||
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
|
||||
GrammarParserInterpreter parser = g.createGrammarParserInterpreter(tokens);
|
||||
ParseTree t = parser.parse(g.rules.get(startRule).index);
|
||||
InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(g.getRuleNames());
|
||||
String treeStr = Trees.toStringTree(t, nodeTextProvider);
|
||||
System.out.println("parse tree: "+treeStr);
|
||||
assertEquals(expectedParseTree, treeStr);
|
||||
return (InterpreterRuleContext)t;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,144 @@
|
|||
package org.antlr.v4.test.tool;
|
||||
|
||||
import org.antlr.v4.runtime.ANTLRInputStream;
|
||||
import org.antlr.v4.runtime.CommonTokenStream;
|
||||
import org.antlr.v4.runtime.LexerInterpreter;
|
||||
import org.antlr.v4.runtime.ParserRuleContext;
|
||||
import org.antlr.v4.runtime.atn.DecisionInfo;
|
||||
import org.antlr.v4.runtime.atn.LookaheadEventInfo;
|
||||
import org.antlr.v4.runtime.tree.ParseTree;
|
||||
import org.antlr.v4.runtime.tree.Trees;
|
||||
import org.antlr.v4.tool.Grammar;
|
||||
import org.antlr.v4.tool.GrammarParserInterpreter;
|
||||
import org.antlr.v4.tool.LexerGrammar;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class TestLookaheadTrees {
|
||||
public static final String lexerText =
|
||||
"lexer grammar L;\n" +
|
||||
"DOT : '.' ;\n" +
|
||||
"SEMI : ';' ;\n" +
|
||||
"BANG : '!' ;\n" +
|
||||
"PLUS : '+' ;\n" +
|
||||
"LPAREN : '(' ;\n" +
|
||||
"RPAREN : ')' ;\n" +
|
||||
"MULT : '*' ;\n" +
|
||||
"ID : [a-z]+ ;\n" +
|
||||
"INT : [0-9]+ ;\n" +
|
||||
"WS : [ \\r\\t\\n]+ ;\n";
|
||||
|
||||
@Test
|
||||
public void testAlts() throws Exception {
|
||||
LexerGrammar lg = new LexerGrammar(lexerText);
|
||||
Grammar g = new Grammar(
|
||||
"parser grammar T;\n" +
|
||||
"s : e SEMI EOF ;\n" +
|
||||
"e : ID DOT ID\n"+
|
||||
" | ID LPAREN RPAREN\n"+
|
||||
" ;\n",
|
||||
lg);
|
||||
|
||||
String startRuleName = "s";
|
||||
int decision = 0;
|
||||
|
||||
testLookaheadTrees(lg, g, "a.b;", startRuleName, decision,
|
||||
new String[] {"(e:1 a . b)", "(e:2 a <error .>)"});
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAlts2() throws Exception {
|
||||
LexerGrammar lg = new LexerGrammar(lexerText);
|
||||
Grammar g = new Grammar(
|
||||
"parser grammar T;\n" +
|
||||
"s : e? SEMI EOF ;\n" +
|
||||
"e : ID\n" +
|
||||
" | e BANG" +
|
||||
" ;\n",
|
||||
lg);
|
||||
|
||||
String startRuleName = "s";
|
||||
int decision = 1; // (...)* in e.
|
||||
|
||||
testLookaheadTrees(lg, g, "a;", startRuleName, decision,
|
||||
new String[] {"(e:2 (e:1 a) <error ;>)", // Decision for alt 1 is error as no ! char, but alt 2 (exit) is good.
|
||||
"(s:1 (e:1 a) ; <EOF>)"}); // root s:1 is included to show ';' node
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIncludeEOF() throws Exception {
|
||||
LexerGrammar lg = new LexerGrammar(lexerText);
|
||||
Grammar g = new Grammar(
|
||||
"parser grammar T;\n" +
|
||||
"s : e ;\n" +
|
||||
"e : ID DOT ID EOF\n"+
|
||||
" | ID DOT ID EOF\n"+
|
||||
" ;\n",
|
||||
lg);
|
||||
|
||||
int decision = 0;
|
||||
testLookaheadTrees(lg, g, "a.b", "s", decision,
|
||||
new String[] {"(e:1 a . b <EOF>)", "(e:2 a . b <EOF>)"});
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCallLeftRecursiveRule() throws Exception {
|
||||
LexerGrammar lg = new LexerGrammar(lexerText);
|
||||
Grammar g = new Grammar(
|
||||
"parser grammar T;\n" +
|
||||
"s : a BANG EOF;\n" +
|
||||
"a : e SEMI \n" +
|
||||
" | ID SEMI \n" +
|
||||
" ;" +
|
||||
"e : e MULT e\n" +
|
||||
" | e PLUS e\n" +
|
||||
" | e DOT e\n" +
|
||||
" | ID\n" +
|
||||
" | INT\n" +
|
||||
" ;\n",
|
||||
lg);
|
||||
|
||||
int decision = 0;
|
||||
testLookaheadTrees(lg, g, "x;!", "s", decision,
|
||||
new String[] {"(a:1 (e:4 x) ;)",
|
||||
"(a:2 x ;)"}); // shouldn't include BANG, EOF
|
||||
decision = 2; // (...)* in e
|
||||
testLookaheadTrees(lg, g, "x+1;!", "s", decision,
|
||||
new String[] {"(e:1 (e:4 x) <error +>)",
|
||||
"(e:2 (e:4 x) + (e:5 1))",
|
||||
"(e:3 (e:4 x) <error +>)"});
|
||||
}
|
||||
|
||||
public void testLookaheadTrees(LexerGrammar lg, Grammar g,
|
||||
String input,
|
||||
String startRuleName,
|
||||
int decision,
|
||||
String[] expectedTrees)
|
||||
{
|
||||
int startRuleIndex = g.getRule(startRuleName).index;
|
||||
InterpreterTreeTextProvider nodeTextProvider =
|
||||
new InterpreterTreeTextProvider(g.getRuleNames());
|
||||
|
||||
LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
|
||||
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
|
||||
GrammarParserInterpreter parser = g.createGrammarParserInterpreter(tokens);
|
||||
parser.setProfile(true);
|
||||
ParseTree t = parser.parse(startRuleIndex);
|
||||
|
||||
DecisionInfo decisionInfo = parser.getParseInfo().getDecisionInfo()[decision];
|
||||
LookaheadEventInfo lookaheadEventInfo = decisionInfo.SLL_MaxLookEvent;
|
||||
|
||||
List<ParserRuleContext> lookaheadParseTrees =
|
||||
GrammarParserInterpreter.getLookaheadParseTrees(g, parser, tokens, startRuleIndex, lookaheadEventInfo.decision,
|
||||
lookaheadEventInfo.startIndex, lookaheadEventInfo.stopIndex);
|
||||
|
||||
assertEquals(expectedTrees.length, lookaheadParseTrees.size());
|
||||
for (int i = 0; i < lookaheadParseTrees.size(); i++) {
|
||||
ParserRuleContext lt = lookaheadParseTrees.get(i);
|
||||
assertEquals(expectedTrees[i], Trees.toStringTree(lt, nodeTextProvider));
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue