forked from jasder/antlr
add getTokenTypeMap(), getRuleIndexMap() to recognizer. Gen new fields for that an ATN with bypass alts. Then methods for that: getATNWithBypassAlts(). Big changes to interface for ParseTreeMatch; create Parser.compileParseTreePattern() method. Convert rule names to rule indexes.
This commit is contained in:
Terence Parr
parent
4c52a103e1
commit
bd91dc166d
18
CHANGES.txt
18
CHANGES.txt
|
|
@ -1,8 +1,22 @@
|
|||
ANTLR v4 Honey Badger
|
||||
|
||||
November 22, 2013
|
||||
November 24, 2013
|
||||
|
||||
* Ter adds tree pattern matching.
|
||||
* Ter adds tree pattern matching. Preferred interface:
|
||||
|
||||
ParseTree t = parser.expr();
|
||||
ParseTreePattern p = parser.compileParseTreePattern("<ID>+0", MyParser.RULE_expr);
|
||||
ParseTreeMatch m = p.match(t);
|
||||
String id = m.get("ID");
|
||||
|
||||
or
|
||||
|
||||
String xpath = "//blockStatement/*";
|
||||
String treePattern = "int <Identifier> = <expression>;";
|
||||
ParseTreePattern p =
|
||||
parser.compileParseTreePattern(treePattern,
|
||||
JavaParser.RULE_localVariableDeclarationStatement);
|
||||
List<ParseTreeMatch> matches = p.findAll(tree, xpath);
|
||||
|
||||
November 20, 2013
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@
|
|||
package org.antlr.v4.runtime;
|
||||
|
||||
import org.antlr.v4.runtime.atn.ATN;
|
||||
import org.antlr.v4.runtime.atn.ATNDeserializationOptions;
|
||||
import org.antlr.v4.runtime.atn.ATNDeserializer;
|
||||
import org.antlr.v4.runtime.atn.ATNSimulator;
|
||||
import org.antlr.v4.runtime.atn.ATNState;
|
||||
import org.antlr.v4.runtime.atn.ParserATNSimulator;
|
||||
|
|
@ -43,6 +45,8 @@ import org.antlr.v4.runtime.tree.ErrorNode;
|
|||
import org.antlr.v4.runtime.tree.ParseTreeListener;
|
||||
import org.antlr.v4.runtime.tree.ParseTreeWalker;
|
||||
import org.antlr.v4.runtime.tree.TerminalNode;
|
||||
import org.antlr.v4.runtime.tree.pattern.ParseTreePattern;
|
||||
import org.antlr.v4.runtime.tree.pattern.ParseTreePatternMatcher;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
|
|
@ -133,6 +137,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
|
|||
*/
|
||||
protected boolean _buildParseTrees = true;
|
||||
|
||||
|
||||
/**
|
||||
* When {@link #setTrace}{@code (true)} is called, a reference to the
|
||||
* {@link TraceListener} is stored here so it can be easily removed in a
|
||||
|
|
@ -430,6 +435,59 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
|
|||
_input.getTokenSource().setTokenFactory(factory);
|
||||
}
|
||||
|
||||
/** The ATN with bypass alternatives is expensive to create so we create it lazily.
|
||||
* The actual generated parsers override this method. These are just
|
||||
* setters/getters. createATNWithBypassAlts() does the creation.
|
||||
*/
|
||||
public void setATNWithBypassAlts(ATN atn) { }
|
||||
public ATN getATNWithBypassAlts() { return null; }
|
||||
|
||||
/** Create and cache the ATN with bypass alternatives. This is not
|
||||
* part of the typical API--use compileParseTreePattern().
|
||||
*/
|
||||
public void createATNWithBypassAlts() {
|
||||
if ( getATNWithBypassAlts()==null ) {
|
||||
synchronized (Parser.class) { // create just one pattern matcher
|
||||
if ( getATNWithBypassAlts()==null ) { // double-check
|
||||
String sATN = getSerializedATN();
|
||||
ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions();
|
||||
deserializationOptions.setGenerateRuleBypassTransitions(true);
|
||||
setATNWithBypassAlts( new ATNDeserializer(deserializationOptions).deserialize(sATN.toCharArray()) );
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The preferred method of getting a tree pattern. For example,
|
||||
* here's a sample use:
|
||||
*
|
||||
* ParseTree t = parser.expr();
|
||||
* ParseTreePattern p = parser.compileParseTreePattern("<ID>+0", MyParser.RULE_expr);
|
||||
* ParseTreeMatch m = p.match(t);
|
||||
* String id = m.get("ID");
|
||||
*/
|
||||
public ParseTreePattern compileParseTreePattern(String pattern, int patternRuleIndex) {
|
||||
if ( getTokenStream()!=null ) {
|
||||
TokenSource tokenSource = getTokenStream().getTokenSource();
|
||||
if ( tokenSource instanceof Lexer ) {
|
||||
Lexer lexer = (Lexer)tokenSource;
|
||||
return compileParseTreePattern(pattern, patternRuleIndex, lexer);
|
||||
}
|
||||
}
|
||||
throw new UnsupportedOperationException("Parser can't discover a lexer to use");
|
||||
}
|
||||
|
||||
/** The same as compileParseTreePattern(pattern,patternRuleName) but
|
||||
* specify a lexer rather than trying to deduce it from this parser.
|
||||
*/
|
||||
public ParseTreePattern compileParseTreePattern(String pattern, int patternRuleIndex,
|
||||
Lexer lexer)
|
||||
{
|
||||
createATNWithBypassAlts();
|
||||
ParseTreePatternMatcher m = new ParseTreePatternMatcher(lexer, this);
|
||||
return m.compile(pattern, patternRuleIndex);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
public ANTLRErrorStrategy getErrorHandler() {
|
||||
return _errHandler;
|
||||
|
|
@ -725,15 +783,12 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
|
|||
return atn.nextTokens(s);
|
||||
}
|
||||
|
||||
// /** Compute the set of valid tokens reachable from the current
|
||||
// * position in the parse.
|
||||
// */
|
||||
// public IntervalSet nextTokens(@NotNull RuleContext ctx) {
|
||||
// ATN atn = getInterpreter().atn;
|
||||
// ATNState s = atn.states.get(ctx.s);
|
||||
// if ( s == null ) return null;
|
||||
// return atn.nextTokens(s, ctx);
|
||||
// }
|
||||
/** Get a rule's index (i.e., RULE_ruleName field) or -1 if not found. */
|
||||
public int getRuleIndex(String ruleName) {
|
||||
Integer ruleIndex = getRuleIndexMap().get(ruleName);
|
||||
if ( ruleIndex!=null ) return ruleIndex;
|
||||
return -1;
|
||||
}
|
||||
|
||||
public ParserRuleContext getRuleContext() { return _ctx; }
|
||||
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import org.antlr.v4.runtime.misc.NotNull;
|
|||
import org.antlr.v4.runtime.misc.Nullable;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
|
||||
public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
|
||||
|
|
@ -59,6 +60,22 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
|
|||
|
||||
public abstract String[] getRuleNames();
|
||||
|
||||
/** Used for xpath, tree pattern compilation */
|
||||
public Map<String, Integer> getTokenTypeMap() {
|
||||
throw new UnsupportedOperationException("recognizer implementation must implement this");
|
||||
}
|
||||
|
||||
/** Used for xpath, tree pattern compilation */
|
||||
public Map<String, Integer> getRuleIndexMap() {
|
||||
throw new UnsupportedOperationException("recognizer implementation must implement this");
|
||||
}
|
||||
|
||||
public int getTokenType(String tokenName) {
|
||||
Integer ttype = getTokenTypeMap().get(tokenName);
|
||||
if ( ttype!=null ) return ttype;
|
||||
return Token.INVALID_TYPE;
|
||||
}
|
||||
|
||||
/** If this recognizer was generated, it will have a serialized ATN
|
||||
* representation of the grammar.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -30,8 +30,6 @@
|
|||
|
||||
package org.antlr.v4.runtime.tree.pattern;
|
||||
|
||||
import org.antlr.v4.runtime.Lexer;
|
||||
import org.antlr.v4.runtime.Parser;
|
||||
import org.antlr.v4.runtime.tree.ParseTree;
|
||||
import org.antlr.v4.runtime.tree.xpath.XPath;
|
||||
|
||||
|
|
@ -43,28 +41,36 @@ import java.util.List;
|
|||
* ParseTreePatternMatcher.compile().
|
||||
*/
|
||||
public class ParseTreePattern {
|
||||
public String patternRuleName;
|
||||
public String pattern;
|
||||
public ParseTree patternTree;
|
||||
protected int patternRuleIndex;
|
||||
protected String pattern;
|
||||
protected ParseTree patternTree;
|
||||
public ParseTreePatternMatcher matcher;
|
||||
|
||||
public ParseTreePattern(String patternRuleName, String pattern, ParseTree patternTree) {
|
||||
this.patternRuleName = patternRuleName;
|
||||
public ParseTreePattern(ParseTreePatternMatcher matcher,
|
||||
String pattern, int patternRuleIndex, ParseTree patternTree)
|
||||
{
|
||||
this.matcher = matcher;
|
||||
this.patternRuleIndex = patternRuleIndex;
|
||||
this.pattern = pattern;
|
||||
this.patternTree = patternTree;
|
||||
}
|
||||
|
||||
/** Find all nodes in the tree that match xpath and also the tree
|
||||
* pattern. Return the list of ParseTreeMatch objects for all matches.
|
||||
public ParseTreeMatch match(ParseTree tree) {
|
||||
return matcher.match(tree, this);
|
||||
}
|
||||
|
||||
public boolean matches(ParseTree tree) {
|
||||
return matcher.match(tree, this).succeeded();
|
||||
}
|
||||
|
||||
/** Find all nodes using xpath and then try to match those subtrees
|
||||
* against this tree pattern
|
||||
*/
|
||||
public static List<ParseTreeMatch> findAll(ParseTree tree, String xpath,
|
||||
String pattern, String patternRuleName,
|
||||
Lexer lexer, Parser parser)
|
||||
{
|
||||
Collection<ParseTree> subtrees = XPath.findAll(tree, xpath, parser);
|
||||
public List<ParseTreeMatch> findAll(ParseTree tree, String xpath) {
|
||||
Collection<ParseTree> subtrees = XPath.findAll(tree, xpath, matcher.getParser());
|
||||
List<ParseTreeMatch> matches = new ArrayList<ParseTreeMatch>();
|
||||
ParseTreePatternMatcher p = new ParseTreePatternMatcher(lexer, parser);
|
||||
for (ParseTree t : subtrees) {
|
||||
ParseTreeMatch match = p.match(t, pattern, patternRuleName);
|
||||
ParseTreeMatch match = match(t);
|
||||
if ( match.succeeded() ) {
|
||||
matches.add(match);
|
||||
}
|
||||
|
|
@ -72,10 +78,19 @@ public class ParseTreePattern {
|
|||
return matches;
|
||||
}
|
||||
|
||||
/** Does the tree match the pattern matched as a patternRuleName? */
|
||||
public static ParseTreeMatch match(ParseTree tree, String pattern, String patternRuleName,
|
||||
Lexer lexer, Parser parser) {
|
||||
ParseTreePatternMatcher p = new ParseTreePatternMatcher(lexer, parser);
|
||||
return p.match(tree, pattern, patternRuleName);
|
||||
public ParseTreePatternMatcher getParseTreePattern() {
|
||||
return matcher;
|
||||
}
|
||||
|
||||
public String getPattern() {
|
||||
return pattern;
|
||||
}
|
||||
|
||||
public int getPatternRuleIndex() {
|
||||
return patternRuleIndex;
|
||||
}
|
||||
|
||||
public ParseTree getPatternTree() {
|
||||
return patternTree;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,10 +38,6 @@ import org.antlr.v4.runtime.Parser;
|
|||
import org.antlr.v4.runtime.ParserInterpreter;
|
||||
import org.antlr.v4.runtime.ParserRuleContext;
|
||||
import org.antlr.v4.runtime.Token;
|
||||
import org.antlr.v4.runtime.atn.ATN;
|
||||
import org.antlr.v4.runtime.atn.ATNDeserializationOptions;
|
||||
import org.antlr.v4.runtime.atn.ATNDeserializer;
|
||||
import org.antlr.v4.runtime.misc.Utils;
|
||||
import org.antlr.v4.runtime.tree.ParseTree;
|
||||
import org.antlr.v4.runtime.tree.RuleNode;
|
||||
import org.antlr.v4.runtime.tree.TerminalNode;
|
||||
|
|
@ -51,7 +47,6 @@ import java.io.StringReader;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/** A tree pattern matching mechanism for ANTLR ParseTrees.
|
||||
*
|
||||
|
|
@ -79,9 +74,7 @@ import java.util.Map;
|
|||
* not match.
|
||||
*
|
||||
* For efficiency, you can compile a tree pattern in string form to a
|
||||
* ParseTreePattern object. It is also expensive to create a
|
||||
* ParseTreePatternMatcher object, so create one of those and reuse
|
||||
* it.
|
||||
* ParseTreePattern object.
|
||||
*
|
||||
* See TestParseTreeMatcher for lots of examples. ParseTreePattern
|
||||
* has two static helper methods: findAll() and match() that are easy
|
||||
|
|
@ -129,14 +122,6 @@ public class ParseTreePatternMatcher {
|
|||
protected String start = "<", stop=">";
|
||||
protected String escape = "\\"; // e.g., \< and \> must escape BOTH!
|
||||
|
||||
/** This ATN has alternatives to match special imaginary tokens for rules like <expr> */
|
||||
protected ATN atnWithBypassAlts;
|
||||
|
||||
/** Maps the rule name to rule index; computed during the constructor
|
||||
* for efficient use later.
|
||||
*/
|
||||
protected Map<String, Integer> ruleToIndex;
|
||||
|
||||
public ParseTreePatternMatcher() { } // used for testing only
|
||||
|
||||
/** Constructs a pattern match or from a lecture and parser object.
|
||||
|
|
@ -147,11 +132,9 @@ public class ParseTreePatternMatcher {
|
|||
public ParseTreePatternMatcher(Lexer lexer, Parser parser) {
|
||||
this.lexer = lexer;
|
||||
this.parser = parser;
|
||||
String sATN = parser.getSerializedATN();
|
||||
ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions();
|
||||
deserializationOptions.setGenerateRuleBypassTransitions(true);
|
||||
atnWithBypassAlts = new ATNDeserializer(deserializationOptions).deserialize(sATN.toCharArray());
|
||||
ruleToIndex = Utils.toMap(parser.getRuleNames());
|
||||
if ( parser!=null ) {
|
||||
parser.createATNWithBypassAlts();
|
||||
}
|
||||
}
|
||||
|
||||
public void setDelimiters(String start, String stop, String escapeLeft) {
|
||||
|
|
@ -160,13 +143,13 @@ public class ParseTreePatternMatcher {
|
|||
this.escape = escapeLeft;
|
||||
}
|
||||
|
||||
/** Does pattern matched as a patternRuleName match tree? */
|
||||
public boolean matches(ParseTree tree, String pattern, String patternRuleName) {
|
||||
ParseTreePattern p = compile(pattern, patternRuleName);
|
||||
/** Does pattern matched as rule patternRuleIndex match tree? */
|
||||
public boolean matches(ParseTree tree, String pattern, int patternRuleIndex) {
|
||||
ParseTreePattern p = compile(pattern, patternRuleIndex);
|
||||
return matches(tree, p);
|
||||
}
|
||||
|
||||
/** Does pattern matched as a patternRuleName match tree? Pass in a
|
||||
/** Does pattern matched as rule patternRuleIndex match tree? Pass in a
|
||||
* compiled pattern instead of a string representation of a tree pattern.
|
||||
*/
|
||||
public boolean matches(ParseTree tree, ParseTreePattern pattern) {
|
||||
|
|
@ -175,16 +158,16 @@ public class ParseTreePatternMatcher {
|
|||
return match.succeeded();
|
||||
}
|
||||
|
||||
/** Compare pattern matched as a patternRuleName against tree and
|
||||
/** Compare pattern matched as rule patternRuleIndex against tree and
|
||||
* return a ParseTreeMatch object that contains the matched elements,
|
||||
* or the node at which the match failed.
|
||||
*/
|
||||
public ParseTreeMatch match(ParseTree tree, String pattern, String patternRuleName) {
|
||||
ParseTreePattern p = compile(pattern, patternRuleName);
|
||||
public ParseTreeMatch match(ParseTree tree, String pattern, int patternRuleIndex) {
|
||||
ParseTreePattern p = compile(pattern, patternRuleIndex);
|
||||
return match(tree, p);
|
||||
}
|
||||
|
||||
/** Compare pattern matched as a patternRuleName against tree and
|
||||
/** Compare pattern matched against tree and
|
||||
* return a ParseTreeMatch object that contains the matched elements,
|
||||
* or the node at which the match failed. Pass in a compiled pattern
|
||||
* instead of a string representation of a tree pattern.
|
||||
|
|
@ -198,7 +181,7 @@ public class ParseTreePatternMatcher {
|
|||
/** For repeated use of a tree pattern, compile it to a ParseTreePattern
|
||||
* using this method.
|
||||
*/
|
||||
public ParseTreePattern compile(String pattern, String patternRuleName) {
|
||||
public ParseTreePattern compile(String pattern, int patternRuleIndex) {
|
||||
List<? extends Token> tokenList = tokenize(pattern);
|
||||
ListTokenSource tokenSrc = new ListTokenSource(tokenList);
|
||||
CommonTokenStream tokens = new CommonTokenStream(tokenSrc);
|
||||
|
|
@ -206,20 +189,19 @@ public class ParseTreePatternMatcher {
|
|||
ParserInterpreter parserInterp = new ParserInterpreter(parser.getGrammarFileName(),
|
||||
Arrays.asList(parser.getTokenNames()),
|
||||
Arrays.asList(parser.getRuleNames()),
|
||||
atnWithBypassAlts,
|
||||
parser.getATNWithBypassAlts(),
|
||||
tokens);
|
||||
|
||||
ParseTree tree = null;
|
||||
try {
|
||||
Integer ruleIndex = ruleToIndex.get(patternRuleName);
|
||||
tree = parserInterp.parse(ruleIndex);
|
||||
tree = parserInterp.parse(patternRuleIndex);
|
||||
// System.out.println("pattern tree = "+tree.toStringTree(parserInterp));
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new CannotInvokeStartRule(e);
|
||||
}
|
||||
|
||||
return new ParseTreePattern(patternRuleName, pattern, tree);
|
||||
return new ParseTreePattern(this, pattern, patternRuleIndex, tree);
|
||||
}
|
||||
|
||||
public Lexer getLexer() {
|
||||
|
|
@ -319,10 +301,6 @@ public class ParseTreePatternMatcher {
|
|||
}
|
||||
|
||||
public List<? extends Token> tokenize(String pattern) {
|
||||
// make maps for quick look up
|
||||
Map<String, Integer> tokenNameToType = Utils.toMap(parser.getTokenNames());
|
||||
Map<String, Integer> ruleNameToIndex = Utils.toMap(parser.getRuleNames());
|
||||
|
||||
// split pattern into chunks: sea (raw input) and islands (<ID>, <expr>)
|
||||
List<Chunk> chunks = split(pattern);
|
||||
|
||||
|
|
@ -333,19 +311,19 @@ public class ParseTreePatternMatcher {
|
|||
TagChunk tagChunk = (TagChunk)chunk;
|
||||
// add special rule token or conjure up new token from name
|
||||
if ( Character.isUpperCase(tagChunk.tag.charAt(0)) ) {
|
||||
Integer ttype = tokenNameToType.get(tagChunk.tag);
|
||||
if ( ttype==null ) {
|
||||
Integer ttype = parser.getTokenType(tagChunk.tag);
|
||||
if ( ttype==Token.INVALID_TYPE ) {
|
||||
throw new IllegalArgumentException("Unknown token "+tagChunk.tag+" in pattern: "+pattern);
|
||||
}
|
||||
TokenTagToken t = new TokenTagToken(tagChunk.tag, ttype, tagChunk.label);
|
||||
tokens.add(t);
|
||||
}
|
||||
else if ( Character.isLowerCase(tagChunk.tag.charAt(0)) ) {
|
||||
Integer ruleIndex = ruleNameToIndex.get(tagChunk.tag);
|
||||
if ( ruleIndex==null ) {
|
||||
int ruleIndex = parser.getRuleIndex(tagChunk.tag);
|
||||
if ( ruleIndex==-1 ) {
|
||||
throw new IllegalArgumentException("Unknown rule "+tagChunk.tag+" in pattern: "+pattern);
|
||||
}
|
||||
int ruleImaginaryTokenType = atnWithBypassAlts.ruleToTokenType[ruleIndex];
|
||||
int ruleImaginaryTokenType = parser.getATNWithBypassAlts().ruleToTokenType[ruleIndex];
|
||||
tokens.add(new RuleTagToken(tagChunk.tag, ruleImaginaryTokenType, tagChunk.label));
|
||||
}
|
||||
else {
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ import org.antlr.v4.runtime.tree.*;
|
|||
import java.util.List;
|
||||
import java.util.Iterator;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Map;
|
||||
|
||||
<parser>
|
||||
>>
|
||||
|
|
@ -224,11 +225,13 @@ public class <parser.name> extends <superClass> {
|
|||
public static final String[] tokenNames = {
|
||||
<parser.tokenNames:{t | <t>}; null="\"\<INVALID>\"", separator=", ", wrap, anchor>
|
||||
};
|
||||
public static final Map\<String, Integer> tokenNameToType = Utils.toMap(tokenNames);
|
||||
public static final int
|
||||
<parser.rules:{r | RULE_<r.name> = <r.index>}; separator=", ", wrap, anchor>;
|
||||
public static final String[] ruleNames = {
|
||||
<parser.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor>
|
||||
};
|
||||
public static final Map\<String, Integer> ruleNameToIndex = Utils.toMap(ruleNames);
|
||||
|
||||
@Override
|
||||
public String getGrammarFileName() { return "<parser.grammarFileName; format="java-escape">"; }
|
||||
|
|
@ -244,6 +247,15 @@ public class <parser.name> extends <superClass> {
|
|||
|
||||
@Override
|
||||
public ATN getATN() { return _ATN; }
|
||||
@Override
|
||||
public ATN getATNWithBypassAlts() { return _ATNWithBypassAlts; }
|
||||
@Override
|
||||
public void setATNWithBypassAlts(ATN atn) { _ATNWithBypassAlts = atn; }
|
||||
|
||||
@Override
|
||||
public Map\<String, Integer> getTokenTypeMap() { return tokenNameToType; }
|
||||
@Override
|
||||
public Map\<String, Integer> getRuleIndexMap() { return ruleNameToIndex; }
|
||||
|
||||
<namedActions.members>
|
||||
<parser:(ctor)()>
|
||||
|
|
@ -841,6 +853,7 @@ import org.antlr.v4.runtime.*;
|
|||
import org.antlr.v4.runtime.atn.*;
|
||||
import org.antlr.v4.runtime.dfa.DFA;
|
||||
import org.antlr.v4.runtime.misc.*;
|
||||
import java.util.Map;
|
||||
|
||||
<lexer>
|
||||
>>
|
||||
|
|
@ -862,9 +875,16 @@ public class <lexer.name> extends <superClass> {
|
|||
"\<INVALID>",
|
||||
<lexer.tokenNames:{t | <t>}; separator=", ", wrap, anchor>
|
||||
};
|
||||
public static final Map\<String, Integer> tokenNameToType = Utils.toMap(tokenNames);
|
||||
public static final String[] ruleNames = {
|
||||
<lexer.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor>
|
||||
};
|
||||
public static final Map\<String, Integer> ruleNameToIndex = Utils.toMap(ruleNames);
|
||||
|
||||
@Override
|
||||
public Map\<String, Integer> getTokenTypeMap() { return tokenNameToType; }
|
||||
@Override
|
||||
public Map\<String, Integer> getRuleIndexMap() { return ruleNameToIndex; }
|
||||
|
||||
<namedActions.members>
|
||||
|
||||
|
|
@ -915,6 +935,7 @@ public static final String _serializedATN =
|
|||
<endif>
|
||||
public static final ATN _ATN =
|
||||
new ATNDeserializer().deserialize(_serializedATN.toCharArray());
|
||||
public static ATN _ATNWithBypassAlts;
|
||||
static {
|
||||
_decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
|
||||
for (int i = 0; i \< _ATN.getNumberOfDecisions(); i++) {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package org.antlr.v4.test;
|
||||
|
||||
import org.antlr.v4.runtime.CharStream;
|
||||
import org.antlr.v4.runtime.CommonTokenStream;
|
||||
import org.antlr.v4.runtime.Lexer;
|
||||
import org.antlr.v4.runtime.Parser;
|
||||
import org.antlr.v4.runtime.Token;
|
||||
|
|
@ -22,27 +23,27 @@ import static org.junit.Assert.assertTrue;
|
|||
|
||||
public class TestParseTreeMatcher extends BaseTest {
|
||||
@Test public void testChunking() throws Exception {
|
||||
ParseTreePatternMatcher p = new ParseTreePatternMatcher();
|
||||
assertEquals("[ID, ' = ', expr, ' ;']", p.split("<ID> = <expr> ;").toString());
|
||||
assertEquals("[' ', ID, ' = ', expr]", p.split(" <ID> = <expr>").toString());
|
||||
assertEquals("[ID, ' = ', expr]", p.split("<ID> = <expr>").toString());
|
||||
assertEquals("[expr]", p.split("<expr>").toString());
|
||||
assertEquals("['<x> foo']", p.split("\\<x\\> foo").toString());
|
||||
assertEquals("['foo <x> bar ', tag]", p.split("foo \\<x\\> bar <tag>").toString());
|
||||
ParseTreePatternMatcher m = new ParseTreePatternMatcher();
|
||||
assertEquals("[ID, ' = ', expr, ' ;']", m.split("<ID> = <expr> ;").toString());
|
||||
assertEquals("[' ', ID, ' = ', expr]", m.split(" <ID> = <expr>").toString());
|
||||
assertEquals("[ID, ' = ', expr]", m.split("<ID> = <expr>").toString());
|
||||
assertEquals("[expr]", m.split("<expr>").toString());
|
||||
assertEquals("['<x> foo']", m.split("\\<x\\> foo").toString());
|
||||
assertEquals("['foo <x> bar ', tag]", m.split("foo \\<x\\> bar <tag>").toString());
|
||||
}
|
||||
|
||||
@Test public void testDelimiters() throws Exception {
|
||||
ParseTreePatternMatcher p = new ParseTreePatternMatcher();
|
||||
p.setDelimiters("<<", ">>", "$");
|
||||
String result = p.split("<<ID>> = <<expr>> ;$<< ick $>>").toString();
|
||||
ParseTreePatternMatcher m = new ParseTreePatternMatcher();
|
||||
m.setDelimiters("<<", ">>", "$");
|
||||
String result = m.split("<<ID>> = <<expr>> ;$<< ick $>>").toString();
|
||||
assertEquals("[ID, ' = ', expr, ' ;<< ick >>']", result);
|
||||
}
|
||||
|
||||
@Test public void testInvertedTags() throws Exception {
|
||||
ParseTreePatternMatcher p = new ParseTreePatternMatcher();
|
||||
ParseTreePatternMatcher m= new ParseTreePatternMatcher();
|
||||
String result = null;
|
||||
try {
|
||||
p.split(">expr<");
|
||||
m.split(">expr<");
|
||||
}
|
||||
catch (IllegalArgumentException iae) {
|
||||
result = iae.getMessage();
|
||||
|
|
@ -52,10 +53,10 @@ public class TestParseTreeMatcher extends BaseTest {
|
|||
}
|
||||
|
||||
@Test public void testUnclosedTag() throws Exception {
|
||||
ParseTreePatternMatcher p = new ParseTreePatternMatcher();
|
||||
ParseTreePatternMatcher m = new ParseTreePatternMatcher();
|
||||
String result = null;
|
||||
try {
|
||||
p.split("<expr hi mom");
|
||||
m.split("<expr hi mom");
|
||||
}
|
||||
catch (IllegalArgumentException iae) {
|
||||
result = iae.getMessage();
|
||||
|
|
@ -65,10 +66,10 @@ public class TestParseTreeMatcher extends BaseTest {
|
|||
}
|
||||
|
||||
@Test public void testExtraClose() throws Exception {
|
||||
ParseTreePatternMatcher p = new ParseTreePatternMatcher();
|
||||
ParseTreePatternMatcher m = new ParseTreePatternMatcher();
|
||||
String result = null;
|
||||
try {
|
||||
p.split("<expr> >");
|
||||
m.split("<expr> >");
|
||||
}
|
||||
catch (IllegalArgumentException iae) {
|
||||
result = iae.getMessage();
|
||||
|
|
@ -89,9 +90,9 @@ public class TestParseTreeMatcher extends BaseTest {
|
|||
rawGenerateAndBuildRecognizer("X1.g4", grammar, "X1Parser", "X1Lexer", false);
|
||||
assertTrue(ok);
|
||||
|
||||
ParseTreePatternMatcher p = getMatcher("X1");
|
||||
ParseTreePatternMatcher m = getPatternMatcher("X1");
|
||||
|
||||
List<? extends Token> tokens = p.tokenize("<ID> = <expr> ;");
|
||||
List<? extends Token> tokens = m.tokenize("<ID> = <expr> ;");
|
||||
String results = tokens.toString();
|
||||
String expected = "[ID:3, [@-1,1:1='=',<1>,1:1], expr:7, [@-1,1:1=';',<2>,1:1]]";
|
||||
assertEquals(expected, results);
|
||||
|
|
@ -110,10 +111,10 @@ public class TestParseTreeMatcher extends BaseTest {
|
|||
rawGenerateAndBuildRecognizer("X2.g4", grammar, "X2Parser", "X2Lexer", false);
|
||||
assertTrue(ok);
|
||||
|
||||
ParseTreePatternMatcher p = getMatcher("X2");
|
||||
ParseTreePatternMatcher m = getPatternMatcher("X2");
|
||||
|
||||
ParseTreePattern t = p.compile("<ID> = <expr> ;", "s");
|
||||
String results = t.patternTree.toStringTree(p.getParser());
|
||||
ParseTreePattern t = m.compile("<ID> = <expr> ;", m.getParser().getRuleIndex("s"));
|
||||
String results = t.getPatternTree().toStringTree(m.getParser());
|
||||
String expected = "(s <ID> = (expr <expr>) ;)";
|
||||
assertEquals(expected, results);
|
||||
}
|
||||
|
|
@ -131,10 +132,10 @@ public class TestParseTreeMatcher extends BaseTest {
|
|||
rawGenerateAndBuildRecognizer("X2.g4", grammar, "X2Parser", "X2Lexer", false);
|
||||
assertTrue(ok);
|
||||
|
||||
ParseTreePatternMatcher p = getMatcher("X2");
|
||||
ParseTreePatternMatcher m = getPatternMatcher("X2");
|
||||
|
||||
ParseTreePattern t = p.compile("<ID> = <expr> ;", "s");
|
||||
String results = t.patternTree.toStringTree(p.getParser());
|
||||
ParseTreePattern t = m.compile("<ID> = <expr> ;", m.getParser().getRuleIndex("s"));
|
||||
String results = t.getPatternTree().toStringTree(m.getParser());
|
||||
String expected = "(s <ID> = (expr <expr>) ;)";
|
||||
assertEquals(expected, results);
|
||||
}
|
||||
|
|
@ -150,10 +151,10 @@ public class TestParseTreeMatcher extends BaseTest {
|
|||
rawGenerateAndBuildRecognizer("X2.g4", grammar, "X2Parser", "X2Lexer", false);
|
||||
assertTrue(ok);
|
||||
|
||||
ParseTreePatternMatcher p = getMatcher("X2");
|
||||
ParseTreePatternMatcher m = getPatternMatcher("X2");
|
||||
|
||||
ParseTreePattern t = p.compile("<ID> = <ID> ;", "s");
|
||||
String results = t.patternTree.toStringTree(p.getParser());
|
||||
ParseTreePattern t = m.compile("<ID> = <ID> ;", m.getParser().getRuleIndex("s"));
|
||||
String results = t.getPatternTree().toStringTree(m.getParser());
|
||||
String expected = "(s <ID> = <ID> ;)";
|
||||
assertEquals(expected, results);
|
||||
}
|
||||
|
|
@ -326,24 +327,38 @@ public class TestParseTreeMatcher extends BaseTest {
|
|||
|
||||
ParseTree result = execParser(startRule, input, parserName, lexerName);
|
||||
|
||||
ParseTreePatternMatcher p = getMatcher(grammarName);
|
||||
ParseTreeMatch match = p.match(result, pattern, startRule);
|
||||
ParseTreePattern p = getPattern(grammarName, pattern, startRule);
|
||||
ParseTreeMatch match = p.match(result);
|
||||
boolean matched = match.succeeded();
|
||||
if ( invertMatch ) assertFalse(matched);
|
||||
else assertTrue(matched);
|
||||
return match;
|
||||
}
|
||||
|
||||
public ParseTreePatternMatcher getMatcher(String name) throws Exception {
|
||||
Class<? extends Lexer> lexerClass = loadLexerClassFromTempDir(name+"Lexer");
|
||||
Class<? extends Parser> parserClass = loadParserClassFromTempDir(name + "Parser");
|
||||
Class<? extends Lexer> c = lexerClass.asSubclass(Lexer.class);
|
||||
Constructor<? extends Lexer> ctor = c.getConstructor(CharStream.class);
|
||||
public ParseTreePattern getPattern(String grammarName, String pattern, String ruleName)
|
||||
throws Exception
|
||||
{
|
||||
Class<? extends Lexer> lexerClass = loadLexerClassFromTempDir(grammarName + "Lexer");
|
||||
Constructor<? extends Lexer> ctor = lexerClass.getConstructor(CharStream.class);
|
||||
Lexer lexer = ctor.newInstance((CharStream) null);
|
||||
|
||||
Class<? extends Parser> pc = parserClass.asSubclass(Parser.class);
|
||||
Constructor<? extends Parser> pctor = pc.getConstructor(TokenStream.class);
|
||||
Parser parser = pctor.newInstance((TokenStream)null);
|
||||
Class<? extends Parser> parserClass = loadParserClassFromTempDir(grammarName + "Parser");
|
||||
Constructor<? extends Parser> pctor = parserClass.getConstructor(TokenStream.class);
|
||||
Parser parser = pctor.newInstance(new CommonTokenStream(lexer));
|
||||
|
||||
return parser.compileParseTreePattern(pattern, parser.getRuleIndex(ruleName));
|
||||
}
|
||||
|
||||
public ParseTreePatternMatcher getPatternMatcher(String grammarName)
|
||||
throws Exception
|
||||
{
|
||||
Class<? extends Lexer> lexerClass = loadLexerClassFromTempDir(grammarName + "Lexer");
|
||||
Constructor<? extends Lexer> ctor = lexerClass.getConstructor(CharStream.class);
|
||||
Lexer lexer = ctor.newInstance((CharStream) null);
|
||||
|
||||
Class<? extends Parser> parserClass = loadParserClassFromTempDir(grammarName + "Parser");
|
||||
Constructor<? extends Parser> pctor = parserClass.getConstructor(TokenStream.class);
|
||||
Parser parser = pctor.newInstance(new CommonTokenStream(lexer));
|
||||
|
||||
return new ParseTreePatternMatcher(lexer, parser);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue