diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternErrorStrategy.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternErrorStrategy.java index a850279a5..d068d60e4 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternErrorStrategy.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternErrorStrategy.java @@ -53,10 +53,6 @@ import org.antlr.v4.runtime.Token; */ public class ParseTreePatternErrorStrategy extends DefaultErrorStrategy { - public boolean isRuleToken(Token t) { - return t instanceof RuleTagToken; - } - @Override public Token recoverInline(Parser recognizer) throws RecognitionException { throw new InputMismatchException(recognizer); @@ -64,7 +60,7 @@ public class ParseTreePatternErrorStrategy extends DefaultErrorStrategy { @Override public void reportError(Parser recognizer, RecognitionException e) { - if ( isRuleToken(e.getOffendingToken()) ) { + if (e.getOffendingToken() instanceof RuleTagToken) { System.out.println("match "); } else { @@ -74,7 +70,7 @@ public class ParseTreePatternErrorStrategy extends DefaultErrorStrategy { @Override public void recover(Parser recognizer, RecognitionException e) { - if ( isRuleToken(e.getOffendingToken()) ) { + if (e.getOffendingToken() instanceof RuleTagToken) { ParserRuleContext ctx = recognizer.getContext(); RuleSubtreeNode sub = new RuleSubtreeNode(ctx); ParserRuleContext parent = ctx.getParent(); diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java index f29d8a2a1..201e19ec7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java @@ -41,11 +41,11 @@ import org.antlr.v4.runtime.TokenStream; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.antlr.v4.runtime.tree.RuleNode; +import org.antlr.v4.runtime.tree.TerminalNode; import java.io.IOException; import java.io.StringReader; import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.HashMap; @@ -53,6 +53,17 @@ import java.util.List; import java.util.Map; public class ParseTreePatternMatcher { + public static class CannotCreateLexerOrParser extends RuntimeException { + public CannotCreateLexerOrParser(Throwable e) { + super(e); + } + } + public static class CannotInvokeStartRule extends RuntimeException { + public CannotInvokeStartRule(Throwable e) { + super(e); + } + } + protected Class lexerClass; protected Class parserClass; @@ -71,48 +82,89 @@ public class ParseTreePatternMatcher { this.parserClass = parserClass; } +// public ParseTreePatternMatcher(String lexerName, String parseName) { +// final Class lexerClass = (Class)loader.loadClass("TLexer"); +// final Class parserClass = (Class)loader.loadClass("TParser"); +// +// } + public void setDelimiters(String start, String stop, String escapeLeft) { this.start = start; this.stop = stop; this.escape = escapeLeft; } - public void lazyInit() - throws IllegalAccessException, InvocationTargetException, - InstantiationException, NoSuchMethodException - { - if ( lexer==null ) { - Class c = lexerClass.asSubclass(Lexer.class); - Constructor ctor = c.getConstructor(CharStream.class); - lexer = ctor.newInstance((CharStream)null); - } + public void lazyInit() { + try { + if ( lexer==null ) { + Class c = lexerClass.asSubclass(Lexer.class); + Constructor ctor = c.getConstructor(CharStream.class); + lexer = ctor.newInstance((CharStream)null); + } - if ( parser==null ) { - Class pc = parserClass.asSubclass(Parser.class); - Constructor pctor = pc.getConstructor(TokenStream.class); - parser = pctor.newInstance((TokenStream)null); + if ( parser==null ) { + Class pc = parserClass.asSubclass(Parser.class); + Constructor pctor = pc.getConstructor(TokenStream.class); + parser = pctor.newInstance((TokenStream)null); + } + } + catch (Exception e) { + throw new CannotCreateLexerOrParser(e); } } - public ParseTree compilePattern(String patternRuleName, String pattern) - throws InstantiationException, IllegalAccessException, NoSuchMethodException, - InvocationTargetException - { + public boolean matches(ParseTree tree, String patternRuleName, String pattern) { + ParseTree patternTree = compilePattern(patternRuleName, pattern); + return matches_(tree, patternTree); + } + + protected boolean matches_(ParseTree tree, ParseTree patternTree) { + if ( tree==null || patternTree==null ) return false; + // x and + if ( tree instanceof TerminalNode && patternTree instanceof TerminalNode ) { + TerminalNode t1 = (TerminalNode)tree; + TerminalNode t2 = (TerminalNode)patternTree; + return t1.getSymbol().getType() == t2.getSymbol().getType(); + } + if ( tree instanceof RuleNode && patternTree instanceof RuleNode ) { + RuleNode r1 = (RuleNode)tree; + RuleNode r2 = (RuleNode)patternTree; + // (expr ...) and + if ( r2 instanceof RuleSubtreeNode ) { + return r1.getRuleContext().getRuleIndex() == r2.getRuleContext().getRuleIndex(); + } + // (expr ...) and (expr ...) + if ( r1.getChildCount()!=r2.getChildCount() ) return false; + int n = r1.getChildCount(); + for (int i = 0; i tokenList = tokenizePattern(pattern); ListTokenSource tokenSrc = new ListTokenSource(tokenList); CommonTokenStream tokens = new CommonTokenStream(tokenSrc); parser.setTokenStream(tokens); parser.setErrorHandler(new ParseTreePatternErrorStrategy()); - Method startRule = parserClass.getMethod(patternRuleName); - ParserRuleContext tree = (ParserRuleContext)startRule.invoke(parser, (Object[])null); - System.out.println(tree.toStringTree(parser)); + ParserRuleContext tree = null; + try { + Method startRule = parserClass.getMethod(patternRuleName); + tree = (ParserRuleContext)startRule.invoke(parser, (Object[])null); + System.out.println("pattern tree = "+tree.toStringTree(parser)); + } + catch (Exception e) { + throw new CannotInvokeStartRule(e); + } return tree; } - public List tokenizePattern(String pattern) - throws InstantiationException, IllegalAccessException, NoSuchMethodException, - InvocationTargetException - { + public List tokenizePattern(String pattern) { lazyInit(); // make maps for quick look up Map tokenNameToType = toMap(parser.getTokenNames(), 0); @@ -147,11 +199,12 @@ public class ParseTreePatternMatcher { } catch (IOException ioe) { // ----------------- + System.err.println("what?-----------------"); } } } - System.out.println(tokens); + System.out.println("tokens="+tokens); return tokens; } @@ -231,9 +284,9 @@ public class ParseTreePatternMatcher { } } if ( ntags>0 ) { - int endOfLastTag = stops.get(ntags - 1) + stop.length(); - if ( endOfLastTag < n-1 ) { // copy text from end of last tag to end - String text = pattern.substring(endOfLastTag+stop.length(), n); + int afterLastTag = stops.get(ntags - 1) + stop.length(); + if ( afterLastTag < n ) { // copy text from end of last tag to end + String text = pattern.substring(afterLastTag, n); chunks.add(new TextChunk(text)); } } diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java index feeeae30f..09d554ece 100644 --- a/tool/test/org/antlr/v4/test/BaseTest.java +++ b/tool/test/org/antlr/v4/test/BaseTest.java @@ -42,6 +42,7 @@ import org.antlr.v4.runtime.CommonToken; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.IntStream; import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.Parser; import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.TokenSource; @@ -57,6 +58,7 @@ import org.antlr.v4.runtime.misc.IntegerList; import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.Nullable; +import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.semantics.SemanticPipeline; import org.antlr.v4.tool.ANTLRMessage; import org.antlr.v4.tool.DOTGenerator; @@ -64,7 +66,11 @@ import org.antlr.v4.tool.DefaultToolListener; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.GrammarSemanticsMessage; import org.antlr.v4.tool.LexerGrammar; +import org.antlr.v4.tool.Rule; import org.junit.Before; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; import org.stringtemplate.v4.ST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.STGroupString; @@ -73,10 +79,6 @@ import javax.tools.JavaCompiler; import javax.tools.JavaFileObject; import javax.tools.StandardJavaFileManager; import javax.tools.ToolProvider; -import org.antlr.v4.tool.Rule; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; @@ -87,6 +89,8 @@ import java.io.InputStreamReader; import java.io.PipedInputStream; import java.io.PipedOutputStream; import java.io.PrintStream; +import java.io.StringReader; +import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.MalformedURLException; @@ -104,7 +108,11 @@ import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; -import static org.junit.Assert.*; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; public abstract class BaseTest { // -J-Dorg.antlr.v4.test.BaseTest.level=FINE @@ -498,6 +506,45 @@ public abstract class BaseTest { debug); } + public ParseTree execParser(String startRuleName, String input, + String parserName, String lexerName) + throws Exception + { + final Class lexerClass = loadLexerClassFromTempDir(lexerName); + final Class parserClass = loadParserClassFromTempDir(parserName); + + ANTLRInputStream in = new ANTLRInputStream(new StringReader(input)); + + Class c = lexerClass.asSubclass(Lexer.class); + Constructor ctor = c.getConstructor(CharStream.class); + Lexer lexer = ctor.newInstance(in); + + Class pc = parserClass.asSubclass(Parser.class); + Constructor pctor = pc.getConstructor(TokenStream.class); + CommonTokenStream tokens = new CommonTokenStream(lexer); + Parser parser = pctor.newInstance(tokens); + + Method startRule = parserClass.getMethod(startRuleName); + ParseTree result = (ParseTree)startRule.invoke(parser, (Object[])null); + System.out.println("parse tree = "+result.toStringTree(parser)); + return result; + } + + public Class loadClassFromTempDir(String name) throws Exception { + ClassLoader loader = + new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() }, + ClassLoader.getSystemClassLoader()); + return loader.loadClass(name); + } + + public Class loadLexerClassFromTempDir(String name) throws Exception { + return (Class)loadClassFromTempDir(name); + } + + public Class loadParserClassFromTempDir(String name) throws Exception { + return (Class)loadClassFromTempDir(name); + } + /** Return true if all is well */ protected boolean rawGenerateAndBuildRecognizer(String grammarFileName, String grammarStr, diff --git a/tool/test/org/antlr/v4/test/TestParseTreeMatcher.java b/tool/test/org/antlr/v4/test/TestParseTreeMatcher.java index c9237b6ed..dbb1a8dcc 100644 --- a/tool/test/org/antlr/v4/test/TestParseTreeMatcher.java +++ b/tool/test/org/antlr/v4/test/TestParseTreeMatcher.java @@ -1,23 +1,17 @@ package org.antlr.v4.test; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Parser; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.pattern.ParseTreePatternMatcher; import org.junit.Test; -import java.io.File; -import java.net.URL; -import java.net.URLClassLoader; import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class TestParseTreeMatcher extends BaseTest { - @Test - public void testChunking() throws Exception { + @Test public void testChunking() throws Exception { // tests ParseTreePatternMatcher p = new ParseTreePatternMatcher(); System.out.println( p.split(" = ;") ); @@ -32,8 +26,7 @@ public class TestParseTreeMatcher extends BaseTest { System.out.println(p.split("<> = <> ;$<< ick $>>")); } - @Test - public void testTokenizingPattern() throws Exception { + @Test public void testTokenizingPattern() throws Exception { String grammar = "grammar T;\n" + "s : ID '=' expr ';' ;\n" + @@ -45,13 +38,10 @@ public class TestParseTreeMatcher extends BaseTest { rawGenerateAndBuildRecognizer("T.g4", grammar, "TParser", "TLexer", false); assertTrue(ok); - ClassLoader loader = - new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() }, - ClassLoader.getSystemClassLoader()); - final Class lexerClass = (Class)loader.loadClass("TLexer"); - final Class parserClass = (Class)loader.loadClass("TParser"); + ParseTreePatternMatcher p = + new ParseTreePatternMatcher(loadLexerClassFromTempDir("TLexer"), + loadParserClassFromTempDir("TParser")); - ParseTreePatternMatcher p = new ParseTreePatternMatcher(lexerClass, parserClass); List tokens = p.tokenizePattern(" = ;"); String results = tokens.toString(); String expected = "[ID:3, [@-1,1:1='=',<1>,1:1], expr:1, [@-1,0:0=';',<2>,1:0]]"; @@ -71,16 +61,57 @@ public class TestParseTreeMatcher extends BaseTest { rawGenerateAndBuildRecognizer("T.g4", grammar, "TParser", "TLexer", false); assertTrue(ok); - ClassLoader loader = - new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() }, - ClassLoader.getSystemClassLoader()); - final Class lexerClass = (Class)loader.loadClass("TLexer"); - final Class parserClass = (Class)loader.loadClass("TParser"); + ParseTreePatternMatcher p = + new ParseTreePatternMatcher(loadLexerClassFromTempDir("TLexer"), + loadParserClassFromTempDir("TParser")); - ParseTreePatternMatcher p = new ParseTreePatternMatcher(lexerClass, parserClass); ParseTree t = p.compilePattern("s", " = ;"); String results = t.toStringTree(p.getParser()); String expected = "(s = expr ;)"; assertEquals(expected, results); } + + public void checkPatternMatch(String grammarName, String grammar, String startRule, + String input, String pattern, + String parserName, String lexerName) + throws Exception + { + boolean ok = + rawGenerateAndBuildRecognizer(grammarName, grammar, parserName, lexerName, false); + assertTrue(ok); + + ParseTree result = execParser(startRule, input, parserName, lexerName); + + ParseTreePatternMatcher p = + new ParseTreePatternMatcher(loadLexerClassFromTempDir(lexerName), + loadParserClassFromTempDir(parserName)); + boolean matches = p.matches(result, startRule, pattern); + assertTrue(matches); + } + + @Test public void testIDNodeMatches() throws Exception { + String grammar = + "grammar T;\n" + + "s : ID ';' ;\n" + + "ID : [a-z]+ ;\n" + + "WS : [ \\r\\n\\t]+ -> skip ;\n"; + + String input = "x ;"; + String pattern = ";"; + checkPatternMatch("T.g4", grammar, "s", input, pattern, "TParser", "TLexer"); + } + + @Test public void testTokenAndRuleMatch() throws Exception { + String grammar = + "grammar T;\n" + + "s : ID '=' expr ';' ;\n" + + "expr : ID | INT ;\n" + + "ID : [a-z]+ ;\n" + + "INT : [0-9]+ ;\n" + + "WS : [ \\r\\n\\t]+ -> skip ;\n"; + + String input = "x = 99;"; + String pattern = " = ;"; + checkPatternMatch("T.g4", grammar, "s", input, pattern, "TParser", "TLexer"); + } }