jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

cleaned up test rig. more tests. fix bugs.

This commit is contained in:
Terence Parr 2013-09-11 17:49:25 -07:00
parent 088cd4a9bc
commit 09ddded3ed
5 changed files with 134 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java
View File

@ -32,7 +32,6 @@ package org.antlr.v4.runtime.tree;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
@ -188,11 +187,11 @@ public class Trees {
List<? super ParseTree> nodes)
{
// check this node (the root) first
if ( t instanceof TerminalNode ) {
if ( findTokens && t instanceof TerminalNode ) {
TerminalNode tnode = (TerminalNode)t;
if ( tnode.getSymbol().getType()==index ) nodes.add(t);
}
else {
else if ( !findTokens && t instanceof ParserRuleContext ) {
ParserRuleContext ctx = (ParserRuleContext)t;
if ( ctx.getRuleIndex() == index ) nodes.add(t);
}
@ -202,8 +201,8 @@ public class Trees {
}
}
public static List<Tree> descendants(ParseTree t){
List<Tree> nodes = new ArrayList<Tree>();
public static List<ParseTree> descendants(ParseTree t){
List<ParseTree> nodes = new ArrayList<ParseTree>();
nodes.add(t);
int n = t.getChildCount();

40
runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPath.java
View File

@ -20,22 +20,7 @@ import java.util.regex.Pattern;
* At each separator-word pair, find set of nodes. Next stage uses those as
* work list.
*
* //ID all IDs anywhere
* /ID an ID node if at root
* /classdef/field all field children of classdef at root.
* ID INVALID (must have // in front or /)
* //classdef//funcdef all funcs under classdef somewhere
* //classdef/* all children of classdefs anywhere in tree
* * INVALID
* /* root node
* //* every node
* /*slash* All children of root
* /* slash * INVALID
*
* these are all the same: returns t if t is classdef root node
* [9/10/13 6:35:13 PM] Terence Parr: eval(t, "classdef")
[9/10/13 6:35:45 PM] Terence Parr: eval(t, "/classdef")
[9/10/13 6:36:44 PM] Terence Parr: eval(t, "/*")
* See TestXPath
*
* The "root" is relative to the node passed to evaluate().
*/
@ -58,7 +43,7 @@ public class XPath {
public XPathElement[] split(String path) {
Map<String, Integer> ruleIndexes = toMap(parser.getRuleNames());
Map<String, Integer> tokenTypes = toMap(parser.getTokenNames());
Pattern pattern = Pattern.compile("//|/|\\w+|\\*");
Pattern pattern = Pattern.compile("//|/|\\w+|'.+?'|\\*"); // TODO: handle escapes in strings?
Matcher matcher = pattern.matcher(path);
List<String> pathStrings = new ArrayList<String>();
while (matcher.find()) {
@ -79,7 +64,7 @@ public class XPath {
if ( next.equals(WILDCARD) ) {
elements.add(new XPathWildcardElement());
}
else if ( Character.isUpperCase(next.charAt(0)) ) {
else if ( next.charAt(0)=='\'' || Character.isUpperCase(next.charAt(0)) ) {
elements.add(new XPathTokenElement(next, tokenTypes.get(next)));
}
else {
@ -93,7 +78,10 @@ public class XPath {
System.out.println("missing element name after operator");
}
String next = pathStrings.get(i);
if ( Character.isUpperCase(next.charAt(0)) ) {
if ( next.equals(WILDCARD) ) {
elements.add(new XPathWildcardAnywhereElement());
}
else if ( next.charAt(0)=='\'' || Character.isUpperCase(next.charAt(0)) ) {
elements.add(new XPathTokenAnywhereElement(next, tokenTypes.get(next)));
}
else {
@ -102,7 +90,10 @@ public class XPath {
i++;
}
else {
if ( Character.isUpperCase(el.charAt(0)) ) {
if ( el.equals(WILDCARD) ) {
elements.add(new XPathWildcardElement());
}
else if ( el.charAt(0)=='\'' || Character.isUpperCase(el.charAt(0)) ) {
elements.add(new XPathTokenElement(el, tokenTypes.get(el)));
}
else {
@ -128,8 +119,13 @@ public class XPath {
while ( i < elements.length ) {
Collection<ParseTree> next = new ArrayList<ParseTree>();
for (ParseTree node : work) {
Collection<? extends ParseTree> matching = elements[i].evaluate(node);
next.addAll(matching);
if ( node.getChildCount()>0 ) {
// only try to match next element if it has children
// e.g., //func/*/stat might have a token node for which
// we can't go looking for stat nodes.
Collection<? extends ParseTree> matching = elements[i].evaluate(node);
next.addAll(matching);
}
}
i++;
work = next;

3
runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleElement.java
View File

@ -18,6 +18,9 @@ public class XPathRuleElement extends XPathElement {
public Collection<ParseTree> evaluate(ParseTree t) {
// return all children of t that match nodeName
List<ParseTree> nodes = new ArrayList<ParseTree>();
if ( t.getChildren()==null) {
System.out.println();
}
for (ParseTree c : t.getChildren()) {
if ( c instanceof ParserRuleContext ) {
ParserRuleContext ctx = (ParserRuleContext)c;

17
runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardAnywhereElement.java Normal file
View File

@ -0,0 +1,17 @@
package org.antlr.v4.runtime.tree.xpath;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.Trees;
import java.util.Collection;
public class XPathWildcardAnywhereElement extends XPathElement {
public XPathWildcardAnywhereElement() {
super(XPath.WILDCARD);
}
@Override
public Collection<ParseTree> evaluate(ParseTree t) {
return Trees.descendants(t);
}
}

124
tool/test/org/antlr/v4/test/TestXPath.java
View File

@ -8,53 +8,113 @@ import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class TestXPath extends BaseTest {
@Test public void test() throws Exception {
String grammar =
"grammar Expr;\n" +
"prog: func+ ;\n" +
"func: 'def' ID '(' arg (',' arg)* ')' '{' stat+ '}' ;\n" +
"arg : ID ;\n" +
"stat: expr ';' # printExpr\n" +
" | ID '=' expr ';' # assign\n" +
" | ';' # blank\n" +
" ;\n" +
"expr: expr ('*'|'/') expr # MulDiv\n" +
" | expr ('+'|'-') expr # AddSub\n" +
" | INT # int\n" +
" | ID # id\n" +
" | '(' expr ')' # parens\n" +
" ;\n" +
"\n" +
"MUL : '*' ; // assigns token name to '*' used above in grammar\n" +
"DIV : '/' ;\n" +
"ADD : '+' ;\n" +
"SUB : '-' ;\n" +
"ID : [a-zA-Z]+ ; // match identifiers\n" +
"INT : [0-9]+ ; // match integers\n" +
"NEWLINE:'\\r'? '\\n' -> skip; // return newlines to parser (is end-statement signal)\n" +
"WS : [ \\t]+ -> skip ; // toss out whitespace\n";
public static final String grammar =
"grammar Expr;\n" +
"prog: func+ ;\n" +
"func: 'def' ID '(' arg (',' arg)* ')' body ;\n" +
"body: '{' stat+ '}' ;\n" +
"arg : ID ;\n" +
"stat: expr ';' # printExpr\n" +
" | ID '=' expr ';' # assign\n" +
" | 'return' expr ';' # ret\n" +
" | ';' # blank\n" +
" ;\n" +
"expr: expr ('*'|'/') expr # MulDiv\n" +
" | expr ('+'|'-') expr # AddSub\n" +
" | primary # prim\n" +
" ;\n" +
"primary" +
" : INT # int\n" +
" | ID # id\n" +
" | '(' expr ')' # parens\n" +
" ;" +
"\n" +
"MUL : '*' ; // assigns token name to '*' used above in grammar\n" +
"DIV : '/' ;\n" +
"ADD : '+' ;\n" +
"SUB : '-' ;\n" +
"ID : [a-zA-Z]+ ; // match identifiers\n" +
"INT : [0-9]+ ; // match integers\n" +
"NEWLINE:'\\r'? '\\n' -> skip; // return newlines to parser (is end-statement signal)\n" +
"WS : [ \\t]+ -> skip ; // toss out whitespace\n";
@Test public void test() throws Exception {
boolean ok =
rawGenerateAndBuildRecognizer("Expr.g4", grammar, "ExprParser", "ExprLexer", false);
assertTrue(ok);
String input = "def f(x,y) { x = 3+4; y; ; }";
Pair<Parser, Lexer> pl = getParserAndLexer(input, "ExprParser", "ExprLexer");
Parser parser = pl.a;
ParseTree tree = execStartRule("prog", parser);
String input =
"def f(x,y) { x = 3+4; y; ; }\n" +
"def g(x) { return 1+2*x; }\n";
String xpath[] = {
"/prog/func", // all funcs under prog at root
"/prog/*", // all children of prog at root
"/*/func", // all func kids of any root node
"prog", // prog must be root node
"/prog", // prog must be root node
"/*", // any root
"*", // any root
"//ID", // any ID in tree
"//expr/primary/ID",// any ID child of a primary under any expr
"//body//ID", // any ID under a body
"//'return'", // any 'return' literal in tree
"//primary/*", // all kids of any primary
"//func/*/stat", // all stat nodes grandkids of any func node
"/prog/func/'def'", // all def literal kids of func kid of prog
"//stat/';'" // all ';' under any stat node
};
String expected[] = {
"[func, func]",
"[func, func]",
"[func, func]",
"[prog]",
"[prog]",
"[prog]",
"[prog]",
"[f, x, y, x, y, g, x, x]",
"[y, x]",
"[x, y, x]",
"[return]",
"[3, 4, y, 1, 2, x]",
"[stat, stat, stat, stat]",
"[def, def]",
"[;, ;, ;, ;]"
};
for (ParseTree t : tree.findAll(parser, "/prog/func") ) {
for (int i=0; i<xpath.length; i++) {
List<String> nodes = getNodeStrings(input, xpath[i], "prog", "ExprParser", "ExprLexer");
String result = nodes.toString();
assertEquals("path "+xpath[i]+" failed", expected[i], result);
}
}
public List<String> getNodeStrings(String input, String xpath,
String startRuleName,
String parserName, String lexerName)
throws Exception
{
Pair<Parser, Lexer> pl = getParserAndLexer(input, parserName, lexerName);
Parser parser = pl.a;
ParseTree tree = execStartRule(startRuleName, parser);
List<String> nodes = new ArrayList<String>();
for (ParseTree t : tree.findAll(parser, xpath) ) {
if ( t instanceof RuleContext) {
RuleContext r = (RuleContext)t;
System.out.println(" "+parser.getRuleNames()[r.getRuleIndex()]);
nodes.add(parser.getRuleNames()[r.getRuleIndex()]);
}
else {
TerminalNode token = (TerminalNode)t;
System.out.println(" "+token.getText());
nodes.add(token.getText());
}
}
return nodes;
}
}