jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

got decent test rig

This commit is contained in:
Terence Parr 2013-09-04 17:09:14 -07:00
parent 731b0f6528
commit 4b87f84b99
4 changed files with 188 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternErrorStrategy.java
View File

@ -53,10 +53,6 @@ import org.antlr.v4.runtime.Token;
*/
public class ParseTreePatternErrorStrategy extends DefaultErrorStrategy {
public boolean isRuleToken(Token t) {
return t instanceof RuleTagToken;
}
@Override
public Token recoverInline(Parser recognizer) throws RecognitionException {
throw new InputMismatchException(recognizer);
@ -64,7 +60,7 @@ public class ParseTreePatternErrorStrategy extends DefaultErrorStrategy {
@Override
public void reportError(Parser recognizer, RecognitionException e) {
if ( isRuleToken(e.getOffendingToken()) ) {
if (e.getOffendingToken() instanceof RuleTagToken) {
System.out.println("match <ruletag>");
}
else {
@ -74,7 +70,7 @@ public class ParseTreePatternErrorStrategy extends DefaultErrorStrategy {
@Override
public void recover(Parser recognizer, RecognitionException e) {
if ( isRuleToken(e.getOffendingToken()) ) {
if (e.getOffendingToken() instanceof RuleTagToken) {
ParserRuleContext ctx = recognizer.getContext();
RuleSubtreeNode sub = new RuleSubtreeNode(ctx);
ParserRuleContext parent = ctx.getParent();

111
runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java
View File

@ -41,11 +41,11 @@ import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
import org.antlr.v4.runtime.tree.RuleNode;
import org.antlr.v4.runtime.tree.TerminalNode;
import java.io.IOException;
import java.io.StringReader;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashMap;
@ -53,6 +53,17 @@ import java.util.List;
import java.util.Map;
public class ParseTreePatternMatcher {
public static class CannotCreateLexerOrParser extends RuntimeException {
public CannotCreateLexerOrParser(Throwable e) {
super(e);
}
}
public static class CannotInvokeStartRule extends RuntimeException {
public CannotInvokeStartRule(Throwable e) {
super(e);
}
}
protected Class<? extends Lexer> lexerClass;
protected Class<? extends Parser> parserClass;
@ -71,48 +82,89 @@ public class ParseTreePatternMatcher {
this.parserClass = parserClass;
}
// public ParseTreePatternMatcher(String lexerName, String parseName) {
// final Class<? extends Lexer> lexerClass = (Class<? extends Lexer>)loader.loadClass("TLexer");
// final Class<? extends Parser> parserClass = (Class<? extends Parser>)loader.loadClass("TParser");
//
// }
public void setDelimiters(String start, String stop, String escapeLeft) {
this.start = start;
this.stop = stop;
this.escape = escapeLeft;
}
public void lazyInit()
throws IllegalAccessException, InvocationTargetException,
InstantiationException, NoSuchMethodException
{
if ( lexer==null ) {
Class<? extends Lexer> c = lexerClass.asSubclass(Lexer.class);
Constructor<? extends Lexer> ctor = c.getConstructor(CharStream.class);
lexer = ctor.newInstance((CharStream)null);
}
public void lazyInit() {
try {
if ( lexer==null ) {
Class<? extends Lexer> c = lexerClass.asSubclass(Lexer.class);
Constructor<? extends Lexer> ctor = c.getConstructor(CharStream.class);
lexer = ctor.newInstance((CharStream)null);
}
if ( parser==null ) {
Class<? extends Parser> pc = parserClass.asSubclass(Parser.class);
Constructor<? extends Parser> pctor = pc.getConstructor(TokenStream.class);
parser = pctor.newInstance((TokenStream)null);
if ( parser==null ) {
Class<? extends Parser> pc = parserClass.asSubclass(Parser.class);
Constructor<? extends Parser> pctor = pc.getConstructor(TokenStream.class);
parser = pctor.newInstance((TokenStream)null);
}
}
catch (Exception e) {
throw new CannotCreateLexerOrParser(e);
}
}
public ParseTree compilePattern(String patternRuleName, String pattern)
throws InstantiationException, IllegalAccessException, NoSuchMethodException,
InvocationTargetException
{
public boolean matches(ParseTree tree, String patternRuleName, String pattern) {
ParseTree patternTree = compilePattern(patternRuleName, pattern);
return matches_(tree, patternTree);
}
protected boolean matches_(ParseTree tree, ParseTree patternTree) {
if ( tree==null || patternTree==null ) return false;
// x and <ID>
if ( tree instanceof TerminalNode && patternTree instanceof TerminalNode ) {
TerminalNode t1 = (TerminalNode)tree;
TerminalNode t2 = (TerminalNode)patternTree;
return t1.getSymbol().getType() == t2.getSymbol().getType();
}
if ( tree instanceof RuleNode && patternTree instanceof RuleNode ) {
RuleNode r1 = (RuleNode)tree;
RuleNode r2 = (RuleNode)patternTree;
// (expr ...) and <expr>
if ( r2 instanceof RuleSubtreeNode ) {
return r1.getRuleContext().getRuleIndex() == r2.getRuleContext().getRuleIndex();
}
// (expr ...) and (expr ...)
if ( r1.getChildCount()!=r2.getChildCount() ) return false;
int n = r1.getChildCount();
for (int i = 0; i<n; i++) {
boolean childrenMatch = matches_(r1.getChild(i), patternTree.getChild(i));
if ( !childrenMatch ) return false;
}
return true;
}
// if nodes aren't both tokens or both rule nodes, can't match
return false;
}
public ParseTree compilePattern(String patternRuleName, String pattern) {
List<? extends Token> tokenList = tokenizePattern(pattern);
ListTokenSource tokenSrc = new ListTokenSource(tokenList);
CommonTokenStream tokens = new CommonTokenStream(tokenSrc);
parser.setTokenStream(tokens);
parser.setErrorHandler(new ParseTreePatternErrorStrategy());
Method startRule = parserClass.getMethod(patternRuleName);
ParserRuleContext tree = (ParserRuleContext)startRule.invoke(parser, (Object[])null);
System.out.println(tree.toStringTree(parser));
ParserRuleContext tree = null;
try {
Method startRule = parserClass.getMethod(patternRuleName);
tree = (ParserRuleContext)startRule.invoke(parser, (Object[])null);
System.out.println("pattern tree = "+tree.toStringTree(parser));
}
catch (Exception e) {
throw new CannotInvokeStartRule(e);
}
return tree;
}
public List<? extends Token> tokenizePattern(String pattern)
throws InstantiationException, IllegalAccessException, NoSuchMethodException,
InvocationTargetException
{
public List<? extends Token> tokenizePattern(String pattern) {
lazyInit();
// make maps for quick look up
Map<String, Integer> tokenNameToType = toMap(parser.getTokenNames(), 0);
@ -147,11 +199,12 @@ public class ParseTreePatternMatcher {
}
catch (IOException ioe) {
// -----------------
System.err.println("what?-----------------");
}
}
}
System.out.println(tokens);
System.out.println("tokens="+tokens);
return tokens;
}
@ -231,9 +284,9 @@ public class ParseTreePatternMatcher {
}
}
if ( ntags>0 ) {
int endOfLastTag = stops.get(ntags - 1) + stop.length();
if ( endOfLastTag < n-1 ) { // copy text from end of last tag to end
String text = pattern.substring(endOfLastTag+stop.length(), n);
int afterLastTag = stops.get(ntags - 1) + stop.length();
if ( afterLastTag < n ) { // copy text from end of last tag to end
String text = pattern.substring(afterLastTag, n);
chunks.add(new TextChunk(text));
}
}

57
tool/test/org/antlr/v4/test/BaseTest.java
View File

@ -42,6 +42,7 @@ import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.IntStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenSource;
@ -57,6 +58,7 @@ import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.semantics.SemanticPipeline;
import org.antlr.v4.tool.ANTLRMessage;
import org.antlr.v4.tool.DOTGenerator;
@ -64,7 +66,11 @@ import org.antlr.v4.tool.DefaultToolListener;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.GrammarSemanticsMessage;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
import org.junit.Before;
import org.junit.rules.TestRule;
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupString;
@ -73,10 +79,6 @@ import javax.tools.JavaCompiler;
import javax.tools.JavaFileObject;
import javax.tools.StandardJavaFileManager;
import javax.tools.ToolProvider;
import org.antlr.v4.tool.Rule;
import org.junit.rules.TestRule;
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
@ -87,6 +89,8 @@ import java.io.InputStreamReader;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.io.PrintStream;
import java.io.StringReader;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.MalformedURLException;
@ -104,7 +108,11 @@ import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import static org.junit.Assert.*;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
public abstract class BaseTest {
// -J-Dorg.antlr.v4.test.BaseTest.level=FINE
@ -498,6 +506,45 @@ public abstract class BaseTest {
debug);
}
public ParseTree execParser(String startRuleName, String input,
String parserName, String lexerName)
throws Exception
{
final Class<? extends Lexer> lexerClass = loadLexerClassFromTempDir(lexerName);
final Class<? extends Parser> parserClass = loadParserClassFromTempDir(parserName);
ANTLRInputStream in = new ANTLRInputStream(new StringReader(input));
Class<? extends Lexer> c = lexerClass.asSubclass(Lexer.class);
Constructor<? extends Lexer> ctor = c.getConstructor(CharStream.class);
Lexer lexer = ctor.newInstance(in);
Class<? extends Parser> pc = parserClass.asSubclass(Parser.class);
Constructor<? extends Parser> pctor = pc.getConstructor(TokenStream.class);
CommonTokenStream tokens = new CommonTokenStream(lexer);
Parser parser = pctor.newInstance(tokens);
Method startRule = parserClass.getMethod(startRuleName);
ParseTree result = (ParseTree)startRule.invoke(parser, (Object[])null);
System.out.println("parse tree = "+result.toStringTree(parser));
return result;
}
public Class<?> loadClassFromTempDir(String name) throws Exception {
ClassLoader loader =
new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() },
ClassLoader.getSystemClassLoader());
return loader.loadClass(name);
}
public Class<? extends Lexer> loadLexerClassFromTempDir(String name) throws Exception {
return (Class<? extends Lexer>)loadClassFromTempDir(name);
}
public Class<? extends Parser> loadParserClassFromTempDir(String name) throws Exception {
return (Class<? extends Parser>)loadClassFromTempDir(name);
}
/** Return true if all is well */
protected boolean rawGenerateAndBuildRecognizer(String grammarFileName,
String grammarStr,

73
tool/test/org/antlr/v4/test/TestParseTreeMatcher.java
View File

@ -1,23 +1,17 @@
package org.antlr.v4.test;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.pattern.ParseTreePatternMatcher;
import org.junit.Test;
import java.io.File;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class TestParseTreeMatcher extends BaseTest {
@Test
public void testChunking() throws Exception {
@Test public void testChunking() throws Exception {
// tests
ParseTreePatternMatcher p = new ParseTreePatternMatcher();
System.out.println( p.split("<ID> = <expr> ;") );
@ -32,8 +26,7 @@ public class TestParseTreeMatcher extends BaseTest {
System.out.println(p.split("<<ID>> = <<expr>> ;$<< ick $>>"));
}
@Test
public void testTokenizingPattern() throws Exception {
@Test public void testTokenizingPattern() throws Exception {
String grammar =
"grammar T;\n" +
"s : ID '=' expr ';' ;\n" +
@ -45,13 +38,10 @@ public class TestParseTreeMatcher extends BaseTest {
rawGenerateAndBuildRecognizer("T.g4", grammar, "TParser", "TLexer", false);
assertTrue(ok);
ClassLoader loader =
new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() },
ClassLoader.getSystemClassLoader());
final Class<? extends Lexer> lexerClass = (Class<? extends Lexer>)loader.loadClass("TLexer");
final Class<? extends Parser> parserClass = (Class<? extends Parser>)loader.loadClass("TParser");
ParseTreePatternMatcher p =
new ParseTreePatternMatcher(loadLexerClassFromTempDir("TLexer"),
loadParserClassFromTempDir("TParser"));
ParseTreePatternMatcher p = new ParseTreePatternMatcher(lexerClass, parserClass);
List<? extends Token> tokens = p.tokenizePattern("<ID> = <expr> ;");
String results = tokens.toString();
String expected = "[ID:3, [@-1,1:1='=',<1>,1:1], expr:1, [@-1,0:0=';',<2>,1:0]]";
@ -71,16 +61,57 @@ public class TestParseTreeMatcher extends BaseTest {
rawGenerateAndBuildRecognizer("T.g4", grammar, "TParser", "TLexer", false);
assertTrue(ok);
ClassLoader loader =
new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() },
ClassLoader.getSystemClassLoader());
final Class<? extends Lexer> lexerClass = (Class<? extends Lexer>)loader.loadClass("TLexer");
final Class<? extends Parser> parserClass = (Class<? extends Parser>)loader.loadClass("TParser");
ParseTreePatternMatcher p =
new ParseTreePatternMatcher(loadLexerClassFromTempDir("TLexer"),
loadParserClassFromTempDir("TParser"));
ParseTreePatternMatcher p = new ParseTreePatternMatcher(lexerClass, parserClass);
ParseTree t = p.compilePattern("s", "<ID> = <expr> ;");
String results = t.toStringTree(p.getParser());
String expected = "(s <ID> = expr ;)";
assertEquals(expected, results);
}
public void checkPatternMatch(String grammarName, String grammar, String startRule,
String input, String pattern,
String parserName, String lexerName)
throws Exception
{
boolean ok =
rawGenerateAndBuildRecognizer(grammarName, grammar, parserName, lexerName, false);
assertTrue(ok);
ParseTree result = execParser(startRule, input, parserName, lexerName);
ParseTreePatternMatcher p =
new ParseTreePatternMatcher(loadLexerClassFromTempDir(lexerName),
loadParserClassFromTempDir(parserName));
boolean matches = p.matches(result, startRule, pattern);
assertTrue(matches);
}
@Test public void testIDNodeMatches() throws Exception {
String grammar =
"grammar T;\n" +
"s : ID ';' ;\n" +
"ID : [a-z]+ ;\n" +
"WS : [ \\r\\n\\t]+ -> skip ;\n";
String input = "x ;";
String pattern = "<ID>;";
checkPatternMatch("T.g4", grammar, "s", input, pattern, "TParser", "TLexer");
}
@Test public void testTokenAndRuleMatch() throws Exception {
String grammar =
"grammar T;\n" +
"s : ID '=' expr ';' ;\n" +
"expr : ID | INT ;\n" +
"ID : [a-z]+ ;\n" +
"INT : [0-9]+ ;\n" +
"WS : [ \\r\\n\\t]+ -> skip ;\n";
String input = "x = 99;";
String pattern = "<ID> = <expr> ;";
checkPatternMatch("T.g4", grammar, "s", input, pattern, "TParser", "TLexer");
}
}