From a9ca2efae56815dc464189b055ffe9da23766f7f Mon Sep 17 00:00:00 2001 From: parrt Date: Thu, 18 Jun 2015 17:25:51 -0700 Subject: [PATCH] add tests, refactor get-all-parse-tree stuff. add Trees support routines. --- .idea/misc.xml | 2 +- .../src/org/antlr/v4/runtime/tree/Trees.java | 59 ++++++- .../xpath/XPathWildcardAnywhereElement.java | 2 +- .../v4/tool/GrammarParserInterpreter.java | 149 +++++++++++++++--- .../tool/InterpreterTreeTextProvider.java | 29 ++++ .../v4/test/tool/TestAmbigParseTrees.java | 113 +++++++------ .../tool/TestGrammarParserInterpreter.java | 113 +++++++++++++ .../v4/test/tool/TestLookaheadTrees.java | 144 +++++++++++++++++ 8 files changed, 533 insertions(+), 78 deletions(-) create mode 100644 tool/test/org/antlr/v4/test/tool/InterpreterTreeTextProvider.java create mode 100644 tool/test/org/antlr/v4/test/tool/TestGrammarParserInterpreter.java create mode 100644 tool/test/org/antlr/v4/test/tool/TestLookaheadTrees.java diff --git a/.idea/misc.xml b/.idea/misc.xml index 800f8ad99..0412a13af 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -7,7 +7,7 @@ - + \ No newline at end of file diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java index aee54eea7..081a11247 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java @@ -30,9 +30,12 @@ package org.antlr.v4.runtime.tree; +import org.antlr.v4.runtime.CommonToken; import org.antlr.v4.runtime.Parser; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.misc.Interval; +import org.antlr.v4.runtime.misc.Predicate; import org.antlr.v4.runtime.misc.Utils; import org.antlr.v4.runtime.tree.gui.TreePostScriptGenerator; import org.antlr.v4.runtime.tree.gui.TreeTextProvider; @@ -178,6 +181,8 @@ public class Trees { /** Return a list of all ancestors of this node. The first node of * list is the root and the last is the parent of this node. + * + * @since 4.5.1 */ public static List getAncestors(Tree t) { if ( t.getParent()==null ) return Collections.emptyList(); @@ -237,17 +242,26 @@ public class Trees { } } - public static List descendants(ParseTree t){ + /** Get all descendents; includes t itself. + * + * @since 4.5.1 + */ + public static List getDescendants(ParseTree t) { List nodes = new ArrayList(); nodes.add(t); int n = t.getChildCount(); for (int i = 0 ; i < n ; i++){ - nodes.addAll(descendants(t.getChild(i))); + nodes.addAll(getDescendants(t.getChild(i))); } return nodes; } + /** @deprecated */ + public static List descendants(ParseTree t) { + return getDescendants(t); + } + /** Find smallest subtree of t enclosing range startTokenIndex..stopTokenIndex * inclusively using postorder traversal. Recursive depth-first-search. * @@ -275,6 +289,47 @@ public class Trees { return null; } + /** Replace any subtree siblings of root that are completely to left + * or right of lookahead range with a CommonToken(Token.INVALID_TYPE,"...") + * node. The source interval for t is not altered to suit smaller range! + * + * WARNING: destructive to t. + * + * @since 4.5.1 + */ + public static void stripChildrenOutOfRange(ParserRuleContext t, + ParserRuleContext root, + int startIndex, + int stopIndex) + { + if ( t==null ) return; + for (int i = 0; i < t.getChildCount(); i++) { + ParseTree child = t.getChild(i); + Interval range = child.getSourceInterval(); + if ( child instanceof ParserRuleContext && (range.b < startIndex || range.a > stopIndex) ) { + if ( isAncestorOf(child, root) ) { // replace only if subtree doesn't have displayed root + CommonToken abbrev = new CommonToken(Token.INVALID_TYPE, "..."); + t.children.set(i, new TerminalNodeImpl(abbrev)); + } + } + } + } + + /** Return first node satisfying the pred + * + * @since 4.5.1 + */ + public static Tree findNodeSuchThat(Tree t, Predicate pred) { + if ( pred.apply(t) ) return t; + + int n = t.getChildCount(); + for (int i = 0 ; i < n ; i++){ + Tree u = findNodeSuchThat(t.getChild(i), pred); + if ( u!=null ) return u; + } + return null; + } + private Trees() { } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardAnywhereElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardAnywhereElement.java index 4a2af465a..b4f7c8a0d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardAnywhereElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardAnywhereElement.java @@ -14,6 +14,6 @@ public class XPathWildcardAnywhereElement extends XPathElement { @Override public Collection evaluate(ParseTree t) { if ( invert ) return new ArrayList(); // !* is weird but valid (empty) - return Trees.descendants(t); + return Trees.getDescendants(t); } } diff --git a/tool/src/org/antlr/v4/tool/GrammarParserInterpreter.java b/tool/src/org/antlr/v4/tool/GrammarParserInterpreter.java index 881127c38..4910eaadd 100644 --- a/tool/src/org/antlr/v4/tool/GrammarParserInterpreter.java +++ b/tool/src/org/antlr/v4/tool/GrammarParserInterpreter.java @@ -1,11 +1,14 @@ package org.antlr.v4.tool; import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.InputMismatchException; import org.antlr.v4.runtime.InterpreterRuleContext; import org.antlr.v4.runtime.Parser; import org.antlr.v4.runtime.ParserInterpreter; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.TokenStream; import org.antlr.v4.runtime.Vocabulary; import org.antlr.v4.runtime.atn.ATN; @@ -64,7 +67,7 @@ public class GrammarParserInterpreter extends ParserInterpreter { input); this.g = g; decisionStatesThatSetOuterAltNumInContext = findOuterMostDecisionStates(); - stateToAltsMap = new int[g.atn.getNumberOfDecisions()][]; + stateToAltsMap = new int[g.atn.states.size()][]; } @Override @@ -270,27 +273,7 @@ public class GrammarParserInterpreter extends ParserInterpreter { { List trees = new ArrayList(); // Create a new parser interpreter to parse the ambiguous subphrase - ParserInterpreter parser; - if (originalParser instanceof ParserInterpreter) { - parser = new GrammarParserInterpreter(g, originalParser.getATN(), originalParser.getTokenStream()); - } - else { // must've been a generated parser - char[] serializedAtn = ATNSerializer.getSerializedAsChars(originalParser.getATN()); - ATN deserialized = new ATNDeserializer().deserialize(serializedAtn); - parser = new ParserInterpreter(originalParser.getGrammarFileName(), - originalParser.getVocabulary(), - Arrays.asList(originalParser.getRuleNames()), - deserialized, - tokens); - } - - parser.setInputStream(tokens); - - // Make sure that we don't get any error messages from using this temporary parser - parser.setErrorHandler(new BailErrorStrategy()); - parser.removeErrorListeners(); - parser.removeParseListeners(); - parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION); + ParserInterpreter parser = getAmbuityParserInterpreter(g, originalParser, tokens); // get ambig trees int alt = alts.nextSetBit(0); @@ -315,4 +298,126 @@ public class GrammarParserInterpreter extends ParserInterpreter { return trees; } + // we must parse the entire input now with decision overrides + // we cannot parse a subset because it could be that a decision + // above our decision of interest needs to read way past + // lookaheadInfo.stopIndex. It seems like there is no escaping + // the use of a full and complete token stream if we are + // resetting to token index 0 and re-parsing from the start symbol. + // It's not easy to restart parsing somewhere in the middle like a + // continuation because our call stack does not match the + // tree stack because of left recursive rule rewriting. grrrr! + public static List getLookaheadParseTrees(Grammar g, + ParserInterpreter originalParser, + TokenStream tokens, + int startRuleIndex, + int decision, + int startIndex, + int stopIndex) + { + List trees = new ArrayList(); + // Create a new parser interpreter to parse the ambiguous subphrase + ParserInterpreter parser = getAmbuityParserInterpreter(g, originalParser, tokens); + BailButConsumeErrorStrategy errorHandler = new BailButConsumeErrorStrategy(); + parser.setErrorHandler(errorHandler); + + DecisionState decisionState = originalParser.getATN().decisionToState.get(decision); + + for (int alt=1; alt<=decisionState.getTransitions().length; alt++) { + // re-parse entire input for all ambiguous alternatives + // (don't have to do first as it's been parsed, but do again for simplicity + // using this temp parser.) + parser.reset(); + parser.addDecisionOverride(decision, startIndex, alt); + ParserRuleContext tt = parser.parse(startRuleIndex); + int stopTreeAt = stopIndex; + if ( errorHandler.firstErrorTokenIndex>=0 ) { + stopTreeAt = errorHandler.firstErrorTokenIndex; // cut off rest at first error + } + ParserRuleContext subtree = + Trees.getRootOfSubtreeEnclosingRegion(tt, + startIndex, + stopTreeAt); + // Use higher of overridden decision tree or tree enclosing all tokens + if ( Trees.isAncestorOf(parser.getOverrideDecisionRoot(), subtree) ) { + subtree = parser.getOverrideDecisionRoot(); + } + Trees.stripChildrenOutOfRange(subtree, parser.getOverrideDecisionRoot(), startIndex, stopTreeAt); + trees.add(subtree); + } + + return trees; + } + + /** Derive a new parser from an old one that has knowledge of the grammar. + * The Grammar object is used to correctly compute outer alternative + * numbers for parse tree nodes. + * @param g + * @param originalParser + * @param tokens + * @return + */ + public static ParserInterpreter getAmbuityParserInterpreter(Grammar g, Parser originalParser, TokenStream tokens) { + ParserInterpreter parser; + if (originalParser instanceof ParserInterpreter) { + parser = new GrammarParserInterpreter(g, originalParser.getATN(), originalParser.getTokenStream()); + } + else { // must've been a generated parser + char[] serializedAtn = ATNSerializer.getSerializedAsChars(originalParser.getATN()); + ATN deserialized = new ATNDeserializer().deserialize(serializedAtn); + parser = new ParserInterpreter(originalParser.getGrammarFileName(), + originalParser.getVocabulary(), + Arrays.asList(originalParser.getRuleNames()), + deserialized, + tokens); + } + + parser.setInputStream(tokens); + + // Make sure that we don't get any error messages from using this temporary parser + parser.setErrorHandler(new BailErrorStrategy()); + parser.removeErrorListeners(); + parser.removeParseListeners(); + parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION); + return parser; + } + + /** We want to stop and track the first air but we cannot bail out like + * {@link BailErrorStrategy} as consume() constructs trees. We make sure + * to create an error node during recovery with this strategy. We + * consume() 1 token during the "bail out of rule" mechanism in recover() + * and let it fall out of the rule to finish constructing trees. For + * recovery in line, we throw InputMismatchException to engage recover(). + */ + public static class BailButConsumeErrorStrategy extends DefaultErrorStrategy { + public int firstErrorTokenIndex = -1; + @Override + public void recover(Parser recognizer, RecognitionException e) { + int errIndex = recognizer.getInputStream().index(); + if ( firstErrorTokenIndex == -1 ) { + firstErrorTokenIndex = errIndex; // latch + } +// System.err.println("recover: error at " + errIndex); + TokenStream input = recognizer.getInputStream(); + if ( input.index() ruleNames; + public InterpreterTreeTextProvider(String[] ruleNames) {this.ruleNames = Arrays.asList(ruleNames);} + + @Override + public String getText(Tree node) { + if ( node==null ) return "null"; + String nodeText = Trees.getNodeText(node, ruleNames); + if ( node instanceof GrammarInterpreterRuleContext) { + GrammarInterpreterRuleContext ctx = (GrammarInterpreterRuleContext) node; + return nodeText+":"+ctx.getOuterAltNum(); + } + if ( node instanceof ErrorNode) { + return ""; + } + return nodeText; + } +} diff --git a/tool/test/org/antlr/v4/test/tool/TestAmbigParseTrees.java b/tool/test/org/antlr/v4/test/tool/TestAmbigParseTrees.java index 0b1fac959..148cfc2e2 100644 --- a/tool/test/org/antlr/v4/test/tool/TestAmbigParseTrees.java +++ b/tool/test/org/antlr/v4/test/tool/TestAmbigParseTrees.java @@ -14,6 +14,7 @@ import org.antlr.v4.runtime.atn.PredictionMode; import org.antlr.v4.runtime.atn.RuleStartState; import org.antlr.v4.runtime.atn.Transition; import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.Trees; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.GrammarParserInterpreter; import org.antlr.v4.tool.LexerGrammar; @@ -38,8 +39,8 @@ public class TestAmbigParseTrees { "x : B ; \n", lg); - testInterpAtSpecificAlt(lg, g, "s", 1, "abc", "(s a (x b) c)"); - testInterpAtSpecificAlt(lg, g, "s", 2, "abc", "(s a b c)"); + testInterpAtSpecificAlt(lg, g, "s", 1, "abc", "(s:1 a (x:1 b) c)"); + testInterpAtSpecificAlt(lg, g, "s", 2, "abc", "(s:2 a b c)"); } @Test public void testAmbigAltsAtRoot() throws Exception { @@ -60,12 +61,13 @@ public class TestAmbigParseTrees { String input = "abc"; String expectedAmbigAlts = "{1, 2}"; int decision = 0; - String expectedOverallTree = "(s a (x b) c)"; - String[] expectedParseTrees = {"(s a (x b) c)","(s a b c)"}; + String expectedOverallTree = "(s:1 a (x:1 b) c)"; + String[] expectedParseTrees = {"(s:1 a (x:1 b) c)", + "(s:2 a b c)"}; - testInterp(lg, g, startRule, input, decision, - expectedAmbigAlts, - expectedOverallTree, expectedParseTrees); + testAmbiguousTrees(lg, g, startRule, input, decision, + expectedAmbigAlts, + expectedOverallTree, expectedParseTrees); } @Test public void testAmbigAltsNotAtRoot() throws Exception { @@ -76,24 +78,25 @@ public class TestAmbigParseTrees { "C : 'c' ;\n"); Grammar g = new Grammar( "parser grammar T;\n" + - "s : a ;" + - "a : b ;" + - "b : A x C" + + "s : x ;" + + "x : y ;" + + "y : A z C" + " | A B C" + " ;" + - "x : B ; \n", + "z : B ; \n", lg); String startRule = "s"; String input = "abc"; String expectedAmbigAlts = "{1, 2}"; int decision = 0; - String expectedOverallTree = "(s (a (b a (x b) c)))"; - String[] expectedParseTrees = {"(b a (x b) c)","(b a b c)"}; + String expectedOverallTree = "(s:1 (x:1 (y:1 a (z:1 b) c)))"; + String[] expectedParseTrees = {"(y:1 a (z:1 b) c)", + "(y:2 a b c)"}; - testInterp(lg, g, startRule, input, decision, - expectedAmbigAlts, - expectedOverallTree, expectedParseTrees); + testAmbiguousTrees(lg, g, startRule, input, decision, + expectedAmbigAlts, + expectedOverallTree, expectedParseTrees); } @Test public void testAmbigAltDipsIntoOuterContextToRoot() throws Exception { @@ -104,7 +107,6 @@ public class TestAmbigParseTrees { "DOT : '.' ;\n"); Grammar g = new Grammar( "parser grammar T;\n" + -// "s : e ;\n"+ "e : p (DOT ID)* ;\n"+ "p : SELF" + " | SELF DOT ID" + @@ -114,13 +116,14 @@ public class TestAmbigParseTrees { String startRule = "e"; String input = "self.x"; String expectedAmbigAlts = "{1, 2}"; - int decision = 1; // decision in s - String expectedOverallTree = "(e (p self) . x)"; - String[] expectedParseTrees = {"(e (p self) . x)","(p self . x)"}; + int decision = 1; // decision in p + String expectedOverallTree = "(e:1 (p:1 self) . x)"; + String[] expectedParseTrees = {"(e:1 (p:1 self) . x)", + "(p:2 self . x)"}; - testInterp(lg, g, startRule, input, decision, - expectedAmbigAlts, - expectedOverallTree, expectedParseTrees); + testAmbiguousTrees(lg, g, startRule, input, decision, + expectedAmbigAlts, + expectedOverallTree, expectedParseTrees); } @Test public void testAmbigAltDipsIntoOuterContextBelowRoot() throws Exception { @@ -141,13 +144,14 @@ public class TestAmbigParseTrees { String startRule = "s"; String input = "self.x"; String expectedAmbigAlts = "{1, 2}"; - int decision = 1; // decision in s - String expectedOverallTree = "(s (e (p self) . x))"; - String[] expectedParseTrees = {"(e (p self) . x)","(p self . x)"}; + int decision = 1; // decision in p + String expectedOverallTree = "(s:1 (e:1 (p:1 self) . x))"; + String[] expectedParseTrees = {"(e:1 (p:1 self) . x)", // shouldn't include s + "(p:2 self . x)"}; // shouldn't include e - testInterp(lg, g, startRule, input, decision, - expectedAmbigAlts, - expectedOverallTree, expectedParseTrees); + testAmbiguousTrees(lg, g, startRule, input, decision, + expectedAmbigAlts, + expectedOverallTree, expectedParseTrees); } @Test public void testAmbigAltInLeftRecursiveBelowStartRule() throws Exception { @@ -168,13 +172,14 @@ public class TestAmbigParseTrees { String startRule = "s"; String input = "self.x"; String expectedAmbigAlts = "{1, 2}"; - int decision = 1; // decision in s - String expectedOverallTree = "(s (e (e (p self)) . x))"; - String[] expectedParseTrees = {"(e (e (p self)) . x)","(p self . x)"}; + int decision = 1; // decision in p + String expectedOverallTree = "(s:1 (e:2 (e:1 (p:1 self)) . x))"; + String[] expectedParseTrees = {"(e:2 (e:1 (p:1 self)) . x)", + "(p:2 self . x)"}; - testInterp(lg, g, startRule, input, decision, - expectedAmbigAlts, - expectedOverallTree, expectedParseTrees); + testAmbiguousTrees(lg, g, startRule, input, decision, + expectedAmbigAlts, + expectedOverallTree, expectedParseTrees); } @Test public void testAmbigAltInLeftRecursiveStartRule() throws Exception { @@ -194,21 +199,24 @@ public class TestAmbigParseTrees { String startRule = "e"; String input = "self.x"; String expectedAmbigAlts = "{1, 2}"; - int decision = 1; // decision in s - String expectedOverallTree = "(e (e (p self)) . x)"; - String[] expectedParseTrees = {"(e (e (p self)) . x)","(p self . x)"}; + int decision = 1; // decision in p + String expectedOverallTree = "(e:2 (e:1 (p:1 self)) . x)"; + String[] expectedParseTrees = {"(e:2 (e:1 (p:1 self)) . x)", + "(p:2 self . x)"}; // shows just enough for self.x - testInterp(lg, g, startRule, input, decision, - expectedAmbigAlts, - expectedOverallTree, expectedParseTrees); + testAmbiguousTrees(lg, g, startRule, input, decision, + expectedAmbigAlts, + expectedOverallTree, expectedParseTrees); } - public void testInterp(LexerGrammar lg, Grammar g, - String startRule, String input, int decision, - String expectedAmbigAlts, - String overallTree, - String[] expectedParseTrees) + public void testAmbiguousTrees(LexerGrammar lg, Grammar g, + String startRule, String input, int decision, + String expectedAmbigAlts, + String overallTree, + String[] expectedParseTrees) { + InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(g.getRuleNames()); + LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input)); CommonTokenStream tokens = new CommonTokenStream(lexEngine); final GrammarParserInterpreter parser = g.createGrammarParserInterpreter(tokens); @@ -218,7 +226,7 @@ public class TestAmbigParseTrees { // PARSE int ruleIndex = g.rules.get(startRule).index; ParserRuleContext parseTree = parser.parse(ruleIndex); - assertEquals(overallTree, parseTree.toStringTree(parser)); + assertEquals(overallTree, Trees.toStringTree(parseTree, nodeTextProvider)); System.out.println(); DecisionInfo[] decisionInfo = parser.getParseInfo().getDecisionInfo(); @@ -230,17 +238,17 @@ public class TestAmbigParseTrees { GrammarParserInterpreter.getAllPossibleParseTrees(g, parser, tokens, - ambiguityInfo.decision, + decision, ambiguityInfo.ambigAlts, ambiguityInfo.startIndex, ambiguityInfo.stopIndex, ruleIndex); assertEquals(expectedAmbigAlts, ambiguityInfo.ambigAlts.toString()); - assertEquals(ambiguityInfo.ambigAlts.cardinality(), ambiguousParseTrees.size()); + for (int i = 0; i)"); + testInterp(lg, g, "e", "a", "(e:4 a)"); + testInterp(lg, g, "e", "34", "(e:3 34)"); + testInterp(lg, g, "e", "a+1", "(e:2 (e:4 a) + (e:3 1))"); + testInterp(lg, g, "e", "1+2*a", "(e:2 (e:3 1) + (e:1 (e:3 2) * (e:4 a)))"); + } + + InterpreterRuleContext testInterp(LexerGrammar lg, Grammar g, + String startRule, String input, + String expectedParseTree) + { + LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input)); + CommonTokenStream tokens = new CommonTokenStream(lexEngine); + GrammarParserInterpreter parser = g.createGrammarParserInterpreter(tokens); + ParseTree t = parser.parse(g.rules.get(startRule).index); + InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(g.getRuleNames()); + String treeStr = Trees.toStringTree(t, nodeTextProvider); + System.out.println("parse tree: "+treeStr); + assertEquals(expectedParseTree, treeStr); + return (InterpreterRuleContext)t; + } +} diff --git a/tool/test/org/antlr/v4/test/tool/TestLookaheadTrees.java b/tool/test/org/antlr/v4/test/tool/TestLookaheadTrees.java new file mode 100644 index 000000000..be047ac9c --- /dev/null +++ b/tool/test/org/antlr/v4/test/tool/TestLookaheadTrees.java @@ -0,0 +1,144 @@ +package org.antlr.v4.test.tool; + +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.LexerInterpreter; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.atn.DecisionInfo; +import org.antlr.v4.runtime.atn.LookaheadEventInfo; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.Trees; +import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.GrammarParserInterpreter; +import org.antlr.v4.tool.LexerGrammar; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class TestLookaheadTrees { + public static final String lexerText = + "lexer grammar L;\n" + + "DOT : '.' ;\n" + + "SEMI : ';' ;\n" + + "BANG : '!' ;\n" + + "PLUS : '+' ;\n" + + "LPAREN : '(' ;\n" + + "RPAREN : ')' ;\n" + + "MULT : '*' ;\n" + + "ID : [a-z]+ ;\n" + + "INT : [0-9]+ ;\n" + + "WS : [ \\r\\t\\n]+ ;\n"; + + @Test + public void testAlts() throws Exception { + LexerGrammar lg = new LexerGrammar(lexerText); + Grammar g = new Grammar( + "parser grammar T;\n" + + "s : e SEMI EOF ;\n" + + "e : ID DOT ID\n"+ + " | ID LPAREN RPAREN\n"+ + " ;\n", + lg); + + String startRuleName = "s"; + int decision = 0; + + testLookaheadTrees(lg, g, "a.b;", startRuleName, decision, + new String[] {"(e:1 a . b)", "(e:2 a )"}); + } + + @Test + public void testAlts2() throws Exception { + LexerGrammar lg = new LexerGrammar(lexerText); + Grammar g = new Grammar( + "parser grammar T;\n" + + "s : e? SEMI EOF ;\n" + + "e : ID\n" + + " | e BANG" + + " ;\n", + lg); + + String startRuleName = "s"; + int decision = 1; // (...)* in e. + + testLookaheadTrees(lg, g, "a;", startRuleName, decision, + new String[] {"(e:2 (e:1 a) )", // Decision for alt 1 is error as no ! char, but alt 2 (exit) is good. + "(s:1 (e:1 a) ; )"}); // root s:1 is included to show ';' node + } + + @Test + public void testIncludeEOF() throws Exception { + LexerGrammar lg = new LexerGrammar(lexerText); + Grammar g = new Grammar( + "parser grammar T;\n" + + "s : e ;\n" + + "e : ID DOT ID EOF\n"+ + " | ID DOT ID EOF\n"+ + " ;\n", + lg); + + int decision = 0; + testLookaheadTrees(lg, g, "a.b", "s", decision, + new String[] {"(e:1 a . b )", "(e:2 a . b )"}); + } + + @Test + public void testCallLeftRecursiveRule() throws Exception { + LexerGrammar lg = new LexerGrammar(lexerText); + Grammar g = new Grammar( + "parser grammar T;\n" + + "s : a BANG EOF;\n" + + "a : e SEMI \n" + + " | ID SEMI \n" + + " ;" + + "e : e MULT e\n" + + " | e PLUS e\n" + + " | e DOT e\n" + + " | ID\n" + + " | INT\n" + + " ;\n", + lg); + + int decision = 0; + testLookaheadTrees(lg, g, "x;!", "s", decision, + new String[] {"(a:1 (e:4 x) ;)", + "(a:2 x ;)"}); // shouldn't include BANG, EOF + decision = 2; // (...)* in e + testLookaheadTrees(lg, g, "x+1;!", "s", decision, + new String[] {"(e:1 (e:4 x) )", + "(e:2 (e:4 x) + (e:5 1))", + "(e:3 (e:4 x) )"}); + } + + public void testLookaheadTrees(LexerGrammar lg, Grammar g, + String input, + String startRuleName, + int decision, + String[] expectedTrees) + { + int startRuleIndex = g.getRule(startRuleName).index; + InterpreterTreeTextProvider nodeTextProvider = + new InterpreterTreeTextProvider(g.getRuleNames()); + + LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input)); + CommonTokenStream tokens = new CommonTokenStream(lexEngine); + GrammarParserInterpreter parser = g.createGrammarParserInterpreter(tokens); + parser.setProfile(true); + ParseTree t = parser.parse(startRuleIndex); + + DecisionInfo decisionInfo = parser.getParseInfo().getDecisionInfo()[decision]; + LookaheadEventInfo lookaheadEventInfo = decisionInfo.SLL_MaxLookEvent; + + List lookaheadParseTrees = + GrammarParserInterpreter.getLookaheadParseTrees(g, parser, tokens, startRuleIndex, lookaheadEventInfo.decision, + lookaheadEventInfo.startIndex, lookaheadEventInfo.stopIndex); + + assertEquals(expectedTrees.length, lookaheadParseTrees.size()); + for (int i = 0; i < lookaheadParseTrees.size(); i++) { + ParserRuleContext lt = lookaheadParseTrees.get(i); + assertEquals(expectedTrees[i], Trees.toStringTree(lt, nodeTextProvider)); + } + } +}