From 088cd4a9bcb33cc67688f00daac9e1731d338efb Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Wed, 11 Sep 2013 14:59:53 -0700 Subject: [PATCH] got test rig start. added support test rig stuff to exec parser. add findAll to ParseTree interface. --- CHANGES.txt | 2 + .../antlr/v4/runtime/ParserRuleContext.java | 2 +- .../src/org/antlr/v4/runtime/RuleContext.java | 10 ++- .../org/antlr/v4/runtime/tree/ParseTree.java | 11 ++- .../v4/runtime/tree/TerminalNodeImpl.java | 10 ++- tool/playground/TestXPath.java | 10 ++- tool/test/org/antlr/v4/test/BaseTest.java | 82 +++++++++++++++++-- tool/test/org/antlr/v4/test/TestXPath.java | 60 ++++++++++++++ 8 files changed, 171 insertions(+), 16 deletions(-) create mode 100644 tool/test/org/antlr/v4/test/TestXPath.java diff --git a/CHANGES.txt b/CHANGES.txt index c37a7d842..f98f7ca6e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,8 @@ ANTLR v4 Honey Badger September 11, 2013 * Copy lots of find node stuff from v3 GrammarAST to Trees class in runtime. +* Add to ParseTree [BREAKING CHANGE]: + Collection findAll(String xpath); September 10, 2013 diff --git a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java index 695c1ee35..0096aad85 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java @@ -194,7 +194,7 @@ public class ParserRuleContext extends RuleContext { } @Override - public List getChildren() { + public List getChildren() { return children; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java index 660b0b460..a72ef197f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java @@ -36,11 +36,13 @@ import org.antlr.v4.runtime.tree.ParseTreeVisitor; import org.antlr.v4.runtime.tree.RuleNode; import org.antlr.v4.runtime.tree.Trees; import org.antlr.v4.runtime.tree.gui.TreeViewer; +import org.antlr.v4.runtime.tree.xpath.XPath; import javax.print.PrintException; import javax.swing.*; import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.concurrent.Future; @@ -151,10 +153,16 @@ public class RuleContext implements RuleNode { } @Override - public List getChildren() { + public List getChildren() { return null; } + @Override + public Collection findAll(Parser parser, String xpath) { + XPath p = new XPath(parser, xpath); + return p.evaluate(this); + } + @Override public T accept(ParseTreeVisitor visitor) { return visitor.visitChildren(this); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java index 96c3b3e9f..3df4afed1 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java @@ -34,6 +34,7 @@ import org.antlr.v4.runtime.Parser; import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.Token; +import java.util.Collection; import java.util.List; /** An interface to access the tree of {@link RuleContext} objects created @@ -53,19 +54,21 @@ public interface ParseTree extends SyntaxTree { /** Return ordered list of all children of this node; redefine to * refine type. */ - List getChildren(); + List getChildren(); /** The {@link ParseTreeVisitor} needs a double dispatch method. */ - public T accept(ParseTreeVisitor visitor); + T accept(ParseTreeVisitor visitor); + + Collection findAll(Parser parser, String xpath); /** Return the combined text of all leaf nodes. Does not get any * off-channel tokens (if any) so won't return whitespace and * comments if they are sent to parser on hidden channel. */ - public String getText(); + String getText(); /** Specialize toStringTree so that it can print out more information * based upon the parser. */ - public String toStringTree(Parser parser); + String toStringTree(Parser parser); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNodeImpl.java b/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNodeImpl.java index 62c9f61c9..2212a3420 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNodeImpl.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNodeImpl.java @@ -33,7 +33,9 @@ package org.antlr.v4.runtime.tree; import org.antlr.v4.runtime.Parser; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.misc.Interval; +import org.antlr.v4.runtime.tree.xpath.XPath; +import java.util.Collection; import java.util.List; public class TerminalNodeImpl implements TerminalNode { @@ -66,10 +68,16 @@ public class TerminalNodeImpl implements TerminalNode { public int getChildCount() { return 0; } @Override - public List getChildren() { + public List getChildren() { return null; } + @Override + public Collection findAll(Parser parser, String xpath) { + XPath p = new XPath(parser, xpath); + return p.evaluate(this); + } + @Override public T accept(ParseTreeVisitor visitor) { return visitor.visitTerminal(this); diff --git a/tool/playground/TestXPath.java b/tool/playground/TestXPath.java index e15462451..3ac1f900e 100644 --- a/tool/playground/TestXPath.java +++ b/tool/playground/TestXPath.java @@ -24,15 +24,17 @@ public class TestXPath { // XPath p = new XPath(parser, "//blockStatement"); // XPath p = new XPath(parser, "//StringLiteral"); // XPath p = new XPath(parser, "//Identifier"); - XPath p = new XPath(parser, "//expression/primary/Identifier"); - for (ParseTree t : p.evaluate(tree) ) { +// XPath p = new XPath(parser, "//expression/primary/Identifier"); +// XPath p = new XPath(parser, "//primary/*"); + XPath p = new XPath(parser, "//expression//Identifier"); + for (ParseTree t : tree.findAll(parser, "//expression//Identifier") ) { if ( t instanceof RuleContext ) { RuleContext r = (RuleContext)t; - System.out.println(" "+parser.ruleNames[r.getRuleIndex()]); + System.out.println(" "+parser.getRuleNames()[r.getRuleIndex()]); } else { TerminalNode token = (TerminalNode)t; - System.out.println(token.getText()); + System.out.println(" "+token.getText()); } } diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java index feeeae30f..c02e17a36 100644 --- a/tool/test/org/antlr/v4/test/BaseTest.java +++ b/tool/test/org/antlr/v4/test/BaseTest.java @@ -42,6 +42,7 @@ import org.antlr.v4.runtime.CommonToken; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.IntStream; import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.Parser; import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.TokenSource; @@ -57,6 +58,8 @@ import org.antlr.v4.runtime.misc.IntegerList; import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.Nullable; +import org.antlr.v4.runtime.misc.Pair; +import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.semantics.SemanticPipeline; import org.antlr.v4.tool.ANTLRMessage; import org.antlr.v4.tool.DOTGenerator; @@ -64,7 +67,11 @@ import org.antlr.v4.tool.DefaultToolListener; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.GrammarSemanticsMessage; import org.antlr.v4.tool.LexerGrammar; +import org.antlr.v4.tool.Rule; import org.junit.Before; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; import org.stringtemplate.v4.ST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.STGroupString; @@ -73,10 +80,6 @@ import javax.tools.JavaCompiler; import javax.tools.JavaFileObject; import javax.tools.StandardJavaFileManager; import javax.tools.ToolProvider; -import org.antlr.v4.tool.Rule; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; @@ -87,6 +90,8 @@ import java.io.InputStreamReader; import java.io.PipedInputStream; import java.io.PipedOutputStream; import java.io.PrintStream; +import java.io.StringReader; +import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.MalformedURLException; @@ -104,7 +109,11 @@ import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; -import static org.junit.Assert.*; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; public abstract class BaseTest { // -J-Dorg.antlr.v4.test.BaseTest.level=FINE @@ -478,6 +487,69 @@ public abstract class BaseTest { return output; } + public ParseTree execParser(String startRuleName, String input, + String parserName, String lexerName) + throws Exception + { + Pair pl = getParserAndLexer(input, parserName, lexerName); + Parser parser = pl.a; + return execStartRule(startRuleName, parser); + } + + public ParseTree execStartRule(String startRuleName, Parser parser) + throws IllegalAccessException, InvocationTargetException, + NoSuchMethodException + { + Method startRule = null; + Object[] args = null; + try { + startRule = parser.getClass().getMethod(startRuleName); + } + catch (NoSuchMethodException nsme) { + // try with int _p arg for recursive func + startRule = parser.getClass().getMethod(startRuleName, int.class); + args = new Integer[] {0}; + } + ParseTree result = (ParseTree)startRule.invoke(parser, args); + System.out.println("parse tree = "+result.toStringTree(parser)); + return result; + } + + public Pair getParserAndLexer(String input, + String parserName, String lexerName) + throws Exception + { + final Class lexerClass = loadLexerClassFromTempDir(lexerName); + final Class parserClass = loadParserClassFromTempDir(parserName); + + ANTLRInputStream in = new ANTLRInputStream(new StringReader(input)); + + Class c = lexerClass.asSubclass(Lexer.class); + Constructor ctor = c.getConstructor(CharStream.class); + Lexer lexer = ctor.newInstance(in); + + Class pc = parserClass.asSubclass(Parser.class); + Constructor pctor = pc.getConstructor(TokenStream.class); + CommonTokenStream tokens = new CommonTokenStream(lexer); + Parser parser = pctor.newInstance(tokens); + return new Pair(parser, lexer); + } + + public Class loadClassFromTempDir(String name) throws Exception { + ClassLoader loader = + new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() }, + ClassLoader.getSystemClassLoader()); + return loader.loadClass(name); + } + + public Class loadLexerClassFromTempDir(String name) throws Exception { + return (Class)loadClassFromTempDir(name); + } + + public Class loadParserClassFromTempDir(String name) throws Exception { + return (Class)loadClassFromTempDir(name); + } + protected String execParser(String grammarFileName, String grammarStr, String parserName, diff --git a/tool/test/org/antlr/v4/test/TestXPath.java b/tool/test/org/antlr/v4/test/TestXPath.java new file mode 100644 index 000000000..f07a4ce2c --- /dev/null +++ b/tool/test/org/antlr/v4/test/TestXPath.java @@ -0,0 +1,60 @@ +package org.antlr.v4.test; + +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.RuleContext; +import org.antlr.v4.runtime.misc.Pair; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.TerminalNode; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +public class TestXPath extends BaseTest { + @Test public void test() throws Exception { + String grammar = + "grammar Expr;\n" + + "prog: func+ ;\n" + + "func: 'def' ID '(' arg (',' arg)* ')' '{' stat+ '}' ;\n" + + "arg : ID ;\n" + + "stat: expr ';' # printExpr\n" + + " | ID '=' expr ';' # assign\n" + + " | ';' # blank\n" + + " ;\n" + + "expr: expr ('*'|'/') expr # MulDiv\n" + + " | expr ('+'|'-') expr # AddSub\n" + + " | INT # int\n" + + " | ID # id\n" + + " | '(' expr ')' # parens\n" + + " ;\n" + + "\n" + + "MUL : '*' ; // assigns token name to '*' used above in grammar\n" + + "DIV : '/' ;\n" + + "ADD : '+' ;\n" + + "SUB : '-' ;\n" + + "ID : [a-zA-Z]+ ; // match identifiers\n" + + "INT : [0-9]+ ; // match integers\n" + + "NEWLINE:'\\r'? '\\n' -> skip; // return newlines to parser (is end-statement signal)\n" + + "WS : [ \\t]+ -> skip ; // toss out whitespace\n"; + + boolean ok = + rawGenerateAndBuildRecognizer("Expr.g4", grammar, "ExprParser", "ExprLexer", false); + assertTrue(ok); + + String input = "def f(x,y) { x = 3+4; y; ; }"; + Pair pl = getParserAndLexer(input, "ExprParser", "ExprLexer"); + Parser parser = pl.a; + ParseTree tree = execStartRule("prog", parser); + + for (ParseTree t : tree.findAll(parser, "/prog/func") ) { + if ( t instanceof RuleContext) { + RuleContext r = (RuleContext)t; + System.out.println(" "+parser.getRuleNames()[r.getRuleIndex()]); + } + else { + TerminalNode token = (TerminalNode)t; + System.out.println(" "+token.getText()); + } + } + } +}