forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

pass a lexer and a parser to the tree pattern match or instead of the classes.

This commit is contained in:
Terence Parr 2013-11-21 09:32:45 -08:00
parent 599896cf1b
commit e68757717f
3 changed files with 42 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
runtime/Java/src/org/antlr/v4/runtime/Recognizer.java
View File

@ -62,8 +62,6 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
/** If this recognizer was generated, it will have a serialized ATN /** If this recognizer was generated, it will have a serialized ATN
* representation of the grammar. * representation of the grammar.
* *
* Provide a dummy return value here to support backward compatibility.
*
* For interpreters, we don't know their serialized ATN despite having * For interpreters, we don't know their serialized ATN despite having
* created the interpreter from it. * created the interpreter from it.
*/ */

33
runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java
View File

@ -31,7 +31,6 @@
package org.antlr.v4.runtime.tree.pattern; package org.antlr.v4.runtime.tree.pattern;
import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.ListTokenSource; import org.antlr.v4.runtime.ListTokenSource;
@ -39,7 +38,6 @@ import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.ParserInterpreter; import org.antlr.v4.runtime.ParserInterpreter;
import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNDeserializationOptions; import org.antlr.v4.runtime.atn.ATNDeserializationOptions;
import org.antlr.v4.runtime.atn.ATNDeserializer; import org.antlr.v4.runtime.atn.ATNDeserializer;
@ -51,7 +49,6 @@ import org.antlr.v4.runtime.tree.TerminalNode;
import java.io.IOException; import java.io.IOException;
import java.io.StringReader; import java.io.StringReader;
import java.lang.reflect.Constructor;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
@ -86,13 +83,9 @@ public class ParseTreePatternMatcher {
public ParseTreePatternMatcher() { } public ParseTreePatternMatcher() { }
public ParseTreePatternMatcher(Class<? extends Lexer> lexerClass, public ParseTreePatternMatcher(Lexer lexer, Parser parser) {
Class<? extends Parser> parserClass) this.lexer = lexer;
{ this.parser = parser;
this.lexerClass = lexerClass;
this.parserClass = parserClass;
lazyInit();
String sATN = parser.getSerializedATN(); String sATN = parser.getSerializedATN();
ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions(); ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions();
deserializationOptions.setGenerateRuleBypassTransitions(true); deserializationOptions.setGenerateRuleBypassTransitions(true);
@ -106,25 +99,6 @@ public class ParseTreePatternMatcher {
this.escape = escapeLeft; this.escape = escapeLeft;
} }
public void lazyInit() {
try {
if ( lexer==null ) {
Class<? extends Lexer> c = lexerClass.asSubclass(Lexer.class);
Constructor<? extends Lexer> ctor = c.getConstructor(CharStream.class);
lexer = ctor.newInstance((CharStream)null);
}
if ( parser==null ) {
Class<? extends Parser> pc = parserClass.asSubclass(Parser.class);
Constructor<? extends Parser> pctor = pc.getConstructor(TokenStream.class);
parser = pctor.newInstance((TokenStream)null);
}
}
catch (Exception e) {
throw new CannotCreateLexerOrParser(e);
}
}
public boolean matches(ParseTree tree, String patternRuleName, String pattern) { public boolean matches(ParseTree tree, String patternRuleName, String pattern) {
ParseTreePattern p = compile(patternRuleName, pattern); ParseTreePattern p = compile(patternRuleName, pattern);
ParseTreeMatch m = matches_(tree, p.patternTree, p); ParseTreeMatch m = matches_(tree, p.patternTree, p);
@ -243,7 +217,6 @@ public class ParseTreePatternMatcher {
} }
public List<? extends Token> tokenize(String pattern) { public List<? extends Token> tokenize(String pattern) {
lazyInit();
// make maps for quick look up // make maps for quick look up
Map<String, Integer> tokenNameToType = toMap(parser.getTokenNames(), 0); Map<String, Integer> tokenNameToType = toMap(parser.getTokenNames(), 0);
Map<String, Integer> ruleNameToIndex = toMap(parser.getRuleNames(), 0); Map<String, Integer> ruleNameToIndex = toMap(parser.getRuleNames(), 0);

66
tool/test/org/antlr/v4/test/TestParseTreeMatcher.java
View File

@ -1,11 +1,16 @@
package org.antlr.v4.test; package org.antlr.v4.test;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.pattern.ParseTreePattern; import org.antlr.v4.runtime.tree.pattern.ParseTreePattern;
import org.antlr.v4.runtime.tree.pattern.ParseTreePatternMatcher; import org.antlr.v4.runtime.tree.pattern.ParseTreePatternMatcher;
import org.junit.Test; import org.junit.Test;
import java.lang.reflect.Constructor;
import java.util.List; import java.util.List;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
@ -81,9 +86,7 @@ public class TestParseTreeMatcher extends BaseTest {
rawGenerateAndBuildRecognizer("X1.g4", grammar, "X1Parser", "X1Lexer", false); rawGenerateAndBuildRecognizer("X1.g4", grammar, "X1Parser", "X1Lexer", false);
assertTrue(ok); assertTrue(ok);
ParseTreePatternMatcher p = ParseTreePatternMatcher p = getMatcher("X1");
new ParseTreePatternMatcher(loadLexerClassFromTempDir("X1Lexer"),
loadParserClassFromTempDir("X1Parser"));
List<? extends Token> tokens = p.tokenize("<ID> = <expr> ;"); List<? extends Token> tokens = p.tokenize("<ID> = <expr> ;");
String results = tokens.toString(); String results = tokens.toString();
@ -104,9 +107,7 @@ public class TestParseTreeMatcher extends BaseTest {
rawGenerateAndBuildRecognizer("X2.g4", grammar, "X2Parser", "X2Lexer", false); rawGenerateAndBuildRecognizer("X2.g4", grammar, "X2Parser", "X2Lexer", false);
assertTrue(ok); assertTrue(ok);
ParseTreePatternMatcher p = ParseTreePatternMatcher p = getMatcher("X2");
new ParseTreePatternMatcher(loadLexerClassFromTempDir("X2Lexer"),
loadParserClassFromTempDir("X2Parser"));
ParseTreePattern t = p.compile("s", "<ID> = <expr> ;"); ParseTreePattern t = p.compile("s", "<ID> = <expr> ;");
String results = t.patternTree.toStringTree(p.getParser()); String results = t.patternTree.toStringTree(p.getParser());
@ -125,9 +126,7 @@ public class TestParseTreeMatcher extends BaseTest {
rawGenerateAndBuildRecognizer("X2.g4", grammar, "X2Parser", "X2Lexer", false); rawGenerateAndBuildRecognizer("X2.g4", grammar, "X2Parser", "X2Lexer", false);
assertTrue(ok); assertTrue(ok);
ParseTreePatternMatcher p = ParseTreePatternMatcher p = getMatcher("X2");
new ParseTreePatternMatcher(loadLexerClassFromTempDir("X2Lexer"),
loadParserClassFromTempDir("X2Parser"));
ParseTreePattern t = p.compile("s", "<ID> = <ID> ;"); ParseTreePattern t = p.compile("s", "<ID> = <ID> ;");
String results = t.patternTree.toStringTree(p.getParser()); String results = t.patternTree.toStringTree(p.getParser());
@ -135,6 +134,20 @@ public class TestParseTreeMatcher extends BaseTest {
assertEquals(expected, results); assertEquals(expected, results);
} }
public ParseTreePatternMatcher getMatcher(String name) throws Exception {
Class<? extends Lexer> lexerClass = loadLexerClassFromTempDir(name+"Lexer");
Class<? extends Parser> parserClass = loadParserClassFromTempDir(name+"Parser");
Class<? extends Lexer> c = lexerClass.asSubclass(Lexer.class);
Constructor<? extends Lexer> ctor = c.getConstructor(CharStream.class);
Lexer lexer = ctor.newInstance((CharStream)null);
Class<? extends Parser> pc = parserClass.asSubclass(Parser.class);
Constructor<? extends Parser> pctor = pc.getConstructor(TokenStream.class);
Parser parser = pctor.newInstance((TokenStream)null);
return new ParseTreePatternMatcher(lexer, parser);
}
@Test public void testIDNodeMatches() throws Exception { @Test public void testIDNodeMatches() throws Exception {
String grammar = String grammar =
"grammar X3;\n" + "grammar X3;\n" +
@ -144,7 +157,7 @@ public class TestParseTreeMatcher extends BaseTest {
String input = "x ;"; String input = "x ;";
String pattern = "<ID>;"; String pattern = "<ID>;";
checkPatternMatch("X3.g4", grammar, "s", input, pattern, "X3Parser", "X3Lexer"); checkPatternMatch("X3.g4", grammar, "s", input, pattern, "X3");
} }
@Test public void testTokenAndRuleMatch() throws Exception { @Test public void testTokenAndRuleMatch() throws Exception {
@ -158,7 +171,7 @@ public class TestParseTreeMatcher extends BaseTest {
String input = "x = 99;"; String input = "x = 99;";
String pattern = "<ID> = <expr> ;"; String pattern = "<ID> = <expr> ;";
checkPatternMatch("X4.g4", grammar, "s", input, pattern, "X4Parser", "X4Lexer"); checkPatternMatch("X4.g4", grammar, "s", input, pattern, "X4");
} }
@Test public void testTokenTextMatch() throws Exception { @Test public void testTokenTextMatch() throws Exception {
@ -173,22 +186,22 @@ public class TestParseTreeMatcher extends BaseTest {
String input = "x = 0;"; String input = "x = 0;";
String pattern = "<ID> = 1;"; String pattern = "<ID> = 1;";
boolean invertMatch = true; // 0!=1 boolean invertMatch = true; // 0!=1
checkPatternMatch("X4.g4", grammar, "s", input, pattern, "X4Parser", "X4Lexer", invertMatch); checkPatternMatch("X4.g4", grammar, "s", input, pattern, "X4", invertMatch);
input = "x = 0;"; input = "x = 0;";
pattern = "<ID> = 0;"; pattern = "<ID> = 0;";
invertMatch = false; invertMatch = false;
checkPatternMatch("X4.g4", grammar, "s", input, pattern, "X4Parser", "X4Lexer", invertMatch); checkPatternMatch("X4.g4", grammar, "s", input, pattern, "X4", invertMatch);
input = "x = 0;"; input = "x = 0;";
pattern = "x = 0;"; pattern = "x = 0;";
invertMatch = false; invertMatch = false;
checkPatternMatch("X4.g4", grammar, "s", input, pattern, "X4Parser", "X4Lexer", invertMatch); checkPatternMatch("X4.g4", grammar, "s", input, pattern, "X4", invertMatch);
input = "x = 0;"; input = "x = 0;";
pattern = "y = 0;"; pattern = "y = 0;";
invertMatch = true; invertMatch = true;
checkPatternMatch("X4.g4", grammar, "s", input, pattern, "X4Parser", "X4Lexer", invertMatch); checkPatternMatch("X4.g4", grammar, "s", input, pattern, "X4", invertMatch);
} }
@Test public void testAssign() throws Exception { @Test public void testAssign() throws Exception {
@ -209,7 +222,7 @@ public class TestParseTreeMatcher extends BaseTest {
String input = "x = 99;"; String input = "x = 99;";
String pattern = "<ID> = <expr>;"; String pattern = "<ID> = <expr>;";
checkPatternMatch("X5.g4", grammar, "s", input, pattern, "X5Parser", "X5Lexer"); checkPatternMatch("X5.g4", grammar, "s", input, pattern, "X5");
} }
@Test public void testLRecursiveExpr() throws Exception { @Test public void testLRecursiveExpr() throws Exception {
@ -229,32 +242,31 @@ public class TestParseTreeMatcher extends BaseTest {
String input = "3*4*5"; String input = "3*4*5";
String pattern = "<expr> * <expr> * <expr>"; String pattern = "<expr> * <expr> * <expr>";
checkPatternMatch("X6.g4", grammar, "expr", input, pattern, "X6Parser", "X6Lexer"); checkPatternMatch("X6.g4", grammar, "expr", input, pattern, "X6");
} }
public void checkPatternMatch(String grammarName, String grammar, String startRule, public void checkPatternMatch(String grammarFileName, String grammar, String startRule,
String input, String pattern, String input, String pattern,
String parserName, String lexerName) String grammarName)
throws Exception throws Exception
{ {
checkPatternMatch(grammarName, grammar, startRule, input, pattern, parserName, lexerName, false); checkPatternMatch(grammarFileName, grammar, startRule, input, pattern, grammarName, false);
} }
public void checkPatternMatch(String grammarName, String grammar, String startRule, public void checkPatternMatch(String grammarFileName, String grammar, String startRule,
String input, String pattern, String input, String pattern,
String parserName, String lexerName, String grammarName, boolean invertMatch)
boolean invertMatch)
throws Exception throws Exception
{ {
String parserName = grammarName+"Parser";
String lexerName = grammarName+"Lexer";
boolean ok = boolean ok =
rawGenerateAndBuildRecognizer(grammarName, grammar, parserName, lexerName, false); rawGenerateAndBuildRecognizer(grammarFileName, grammar, parserName, lexerName, false);
assertTrue(ok); assertTrue(ok);
ParseTree result = execParser(startRule, input, parserName, lexerName); ParseTree result = execParser(startRule, input, parserName, lexerName);
ParseTreePatternMatcher p = ParseTreePatternMatcher p = getMatcher(grammarName);
new ParseTreePatternMatcher(loadLexerClassFromTempDir(lexerName),
loadParserClassFromTempDir(parserName));
boolean matches = p.matches(result, startRule, pattern); boolean matches = p.matches(result, startRule, pattern);
if ( invertMatch ) assertFalse(matches); if ( invertMatch ) assertFalse(matches);
else assertTrue(matches); else assertTrue(matches);