jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Updated documentation and API encapsulation for tree patterns

This commit is contained in:
Sam Harwell 2013-12-19 19:04:20 -06:00
parent 40bbd66231
commit 8449b9258f
11 changed files with 1039 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

205
runtime/Java/src/org/antlr/v4/runtime/ListTokenSource.java
View File

@ -1,63 +1,230 @@
package org.antlr.v4.runtime;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.CommonTokenFactory;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenFactory;
import org.antlr.v4.runtime.TokenSource;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Pair;
import java.util.List;
/** Convert a list of token objects to a source of tokens for a TokenStream. */
/**
* Provides an implementation of {@link TokenSource} as a wrapper around a list
* of {@link Token} objects.
* <p/>
* If the final token in the list is an {@link Token#EOF} token, it will be used
* as the EOF token for every call to {@link #nextToken} after the end of the
* list is reached. Otherwise, an EOF token will be created.
*/
public class ListTokenSource implements TokenSource {
protected List<? extends Token> tokens;
protected int i = 0;
protected TokenFactory<?> _factory = CommonTokenFactory.DEFAULT;
/**
* The wrapped collection of {@link Token} objects to return.
*/
protected final List<? extends Token> tokens;
public ListTokenSource(List<? extends Token> tokens) {
assert tokens!=null;
this.tokens = tokens;
/**
* The name of the input source. If this value is {@code null}, a call to
* {@link #getSourceName} should return the source name used to create the
* the next token in {@link #tokens} (or the previous token if the end of
* the input has been reached).
*/
private final String sourceName;
/**
* The index into {@link #tokens} of token to return by the next call to
* {@link #nextToken}. The end of the input is indicated by this value
* being greater than or equal to the number of items in {@link #tokens}.
*/
protected int i;
/**
* This field caches the EOF token for the token source.
*/
protected Token eofToken;
/**
* This is the backing field for {@link #getTokenFactory} and
* {@link setTokenFactory}.
*/
private TokenFactory<?> _factory = CommonTokenFactory.DEFAULT;
/**
* Constructs a new {@link ListTokenSource} instance from the specified
* collection of {@link Token} objects.
*
* @param tokens The collection of {@link Token} objects to provide as a
* {@link TokenSource}.
* @exception NullPointerException if {@code tokens} is {@code null}
*/
public ListTokenSource(@NotNull List<? extends Token> tokens) {
this(tokens, null);
}
/**
* Constructs a new {@link ListTokenSource} instance from the specified
* collection of {@link Token} objects and source name.
*
* @param tokens The collection of {@link Token} objects to provide as a
* {@link TokenSource}.
* @param sourceName The name of the {@link TokenSource}. If this value is
* {@code null}, {@link #getSourceName} will attempt to infer the name from
* the next {@link Token} (or the previous token if the end of the input has
* been reached).
*
* @exception NullPointerException if {@code tokens} is {@code null}
*/
public ListTokenSource(@NotNull List<? extends Token> tokens, String sourceName) {
if (tokens == null) {
throw new NullPointerException("tokens cannot be null");
}
this.tokens = tokens;
this.sourceName = sourceName;
}
/**
* @inheritDoc
*/
@Override
public int getCharPositionInLine() {
return -1;
if (i < tokens.size()) {
return tokens.get(i).getCharPositionInLine();
}
else if (eofToken != null) {
return eofToken.getCharPositionInLine();
}
else if (tokens.size() > 0) {
// have to calculate the result from the line/column of the previous
// token, along with the text of the token.
Token lastToken = tokens.get(tokens.size() - 1);
String tokenText = lastToken.getText();
if (tokenText != null) {
int lastNewLine = tokenText.lastIndexOf('\n');
if (lastNewLine >= 0) {
return tokenText.length() - lastNewLine - 1;
}
}
return lastToken.getCharPositionInLine() + lastToken.getStopIndex() - lastToken.getStartIndex() + 1;
}
// only reach this if tokens is empty, meaning EOF occurs at the first
// position in the input
return 0;
}
/**
* @inheritDoc
*/
@Override
public Token nextToken() {
if ( i>=tokens.size() ) {
Token eof = new CommonToken(Token.EOF); // ignore factory
return eof;
if (i >= tokens.size()) {
if (eofToken == null) {
int start = -1;
if (tokens.size() > 0) {
int previousStop = tokens.get(tokens.size() - 1).getStopIndex();
if (previousStop != -1) {
start = previousStop + 1;
}
}
int stop = Math.max(-1, start - 1);
eofToken = _factory.create(new Pair<TokenSource, CharStream>(this, getInputStream()), Token.EOF, "EOF", Token.DEFAULT_CHANNEL, start, stop, getLine(), getCharPositionInLine());
}
return eofToken;
}
Token t = tokens.get(i);
if (i == tokens.size() - 1 && t.getType() == Token.EOF) {
eofToken = t;
}
i++;
return t;
}
/**
* @inheritDoc
*/
@Override
public int getLine() {
return 0;
if (i < tokens.size()) {
return tokens.get(i).getLine();
}
else if (eofToken != null) {
return eofToken.getLine();
}
else if (tokens.size() > 0) {
// have to calculate the result from the line/column of the previous
// token, along with the text of the token.
Token lastToken = tokens.get(tokens.size() - 1);
int line = lastToken.getLine();
String tokenText = lastToken.getText();
if (tokenText != null) {
for (int i = 0; i < tokenText.length(); i++) {
if (tokenText.charAt(i) == '\n') {
line++;
}
}
}
// if no text is available, assume the token did not contain any newline characters.
return line;
}
// only reach this if tokens is empty, meaning EOF occurs at the first
// position in the input
return 1;
}
/**
* @inheritDoc
*/
@Override
public CharStream getInputStream() {
if (i < tokens.size()) {
return tokens.get(i).getInputStream();
}
else if (eofToken != null) {
return eofToken.getInputStream();
}
else if (tokens.size() > 0) {
return tokens.get(tokens.size() - 1).getInputStream();
}
// no input stream information is available
return null;
}
/**
* @inheritDoc
*/
@Override
public String getSourceName() {
return "<List>";
if (sourceName != null) {
return sourceName;
}
CharStream inputStream = getInputStream();
if (inputStream != null) {
return inputStream.getSourceName();
}
return "List";
}
/**
* @inheritDoc
*/
@Override
public void setTokenFactory(@NotNull TokenFactory<?> factory) {
this._factory = factory;
}
/**
* @inheritDoc
*/
@Override
@NotNull
public TokenFactory<?> getTokenFactory() {
return _factory;
}

26
runtime/Java/src/org/antlr/v4/runtime/Parser.java
View File

@ -473,13 +473,16 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
}
}
/** The preferred method of getting a tree pattern. For example,
* here's a sample use:
/**
* The preferred method of getting a tree pattern. For example, here's a
* sample use:
*
* ParseTree t = parser.expr();
* ParseTreePattern p = parser.compileParseTreePattern("<ID>+0", MyParser.RULE_expr);
* ParseTreeMatch m = p.match(t);
* String id = m.get("ID");
* <pre>
* ParseTree t = parser.expr();
* ParseTreePattern p = parser.compileParseTreePattern("<ID>+0", MyParser.RULE_expr);
* ParseTreeMatch m = p.match(t);
* String id = m.get("ID");
* </pre>
*/
public ParseTreePattern compileParseTreePattern(String pattern, int patternRuleIndex) {
if ( getTokenStream()!=null ) {
@ -492,8 +495,9 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
throw new UnsupportedOperationException("Parser can't discover a lexer to use");
}
/** The same as compileParseTreePattern(pattern,patternRuleName) but
* specify a lexer rather than trying to deduce it from this parser.
/**
* The same as {@link #compileParseTreePattern(String, int)} but specify a
* {@link Lexer} rather than trying to deduce it from this parser.
*/
public ParseTreePattern compileParseTreePattern(String pattern, int patternRuleIndex,
Lexer lexer)
@ -644,8 +648,8 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
}
/**
* @deprecated Use {@link #enterRecursionRule(ParserRuleContext, int, int)}
* instead.
* @deprecated Use
* {@link #enterRecursionRule(ParserRuleContext, int, int, int)} instead.
*/
@Deprecated
public void enterRecursionRule(ParserRuleContext localctx, int ruleIndex) {
@ -797,7 +801,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
return atn.nextTokens(s);
}
/** Get a rule's index (i.e., RULE_ruleName field) or -1 if not found. */
/** Get a rule's index (i.e., {@code RULE_ruleName} field) or -1 if not found. */
public int getRuleIndex(String ruleName) {
Integer ruleIndex = getRuleIndexMap().get(ruleName);
if ( ruleIndex!=null ) return ruleIndex;

49
runtime/Java/src/org/antlr/v4/runtime/Recognizer.java
View File

@ -68,7 +68,11 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
public abstract String[] getRuleNames();
/** Used for xpath, tree pattern compilation */
/**
* Get a map from token names to token types.
* <p/>
* Used for XPath and tree pattern compilation.
*/
@NotNull
public Map<String, Integer> getTokenTypeMap() {
String[] tokenNames = getTokenNames();
@ -89,7 +93,11 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
}
}
/** Used for xpath, tree pattern compilation */
/**
* Get a map from rule names to rule indexes.
* <p/>
* Used for XPath and tree pattern compilation.
*/
@NotNull
public Map<String, Integer> getRuleIndexMap() {
String[] ruleNames = getRuleNames();
@ -114,12 +122,14 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
return Token.INVALID_TYPE;
}
/** If this recognizer was generated, it will have a serialized ATN
* representation of the grammar.
*
* For interpreters, we don't know their serialized ATN despite having
* created the interpreter from it.
/**
* If this recognizer was generated, it will have a serialized ATN
* representation of the grammar.
* <p/>
* For interpreters, we don't know their serialized ATN despite having
* created the interpreter from it.
*/
@NotNull
public String getSerializedATN() {
throw new UnsupportedOperationException("there is no serialized ATN");
}
@ -129,18 +139,37 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
*/
public abstract String getGrammarFileName();
/**
* Get the {@link ATN} used by the recognizer for prediction.
*
* @return The {@link ATN} used by the recognizer for prediction.
*/
@NotNull
public abstract ATN getATN();
/**
* Get the ATN interpreter used by the recognizer for prediction.
*
* @return The ATN interpreter used by the recognizer for prediction.
*/
@NotNull
public ATNInterpreter getInterpreter() {
return _interp;
}
public void setInterpreter(ATNInterpreter interpreter) {
/**
* Set the ATN interpreter used by the recognizer for prediction.
*
* @param interpreter The ATN interpreter used by the recognizer for
* prediction.
*/
public void setInterpreter(@NotNull ATNInterpreter interpreter) {
_interp = interpreter;
}
/** What is the error header, normally line/character position information? */
public String getErrorHeader(RecognitionException e) {
@NotNull
public String getErrorHeader(@NotNull RecognitionException e) {
int line = e.getOffendingToken().getLine();
int charPositionInLine = e.getOffendingToken().getCharPositionInLine();
return "line "+line+":"+charPositionInLine;
@ -172,7 +201,7 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
}
/**
* @throws NullPointerException if {@code listener} is {@code null}.
* @exception NullPointerException if {@code listener} is {@code null}.
*/
public void addErrorListener(@NotNull ANTLRErrorListener listener) {
if (listener == null) {

16
runtime/Java/src/org/antlr/v4/runtime/tree/pattern/Chunk.java
View File

@ -30,12 +30,16 @@
package org.antlr.v4.runtime.tree.pattern;
/** A chunk is either a token reference, a rule reference, or some plaintext
* within a tree pattern. Function split() in the pattern matcher returns
* a list of chunks in preparation for creating a token stream by tokenize().
* From there, we get a parse tree from with compile(). These chunks are
* converted to RuleTagToken, TokenTagToken, or the regular tokens
* of the text surrounding the tags.
/**
* A chunk is either a token tag, a rule tag, or a span of literal text within a
* tree pattern.
* <p/>
* The method {@link ParseTreePatternMatcher#split(String)} returns a list of
* chunks in preparation for creating a token stream by
* {@link ParseTreePatternMatcher#tokenize(String)}. From there, we get a parse
* tree from with {@link ParseTreePatternMatcher#compile(String, int)}. These
* chunks are converted to {@link RuleTagToken}, {@link TokenTagToken}, or the
* regular tokens of the text surrounding the tags.
*/
abstract class Chunk {
}

175
runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreeMatch.java
View File

@ -31,91 +31,194 @@
package org.antlr.v4.runtime.tree.pattern;
import org.antlr.v4.runtime.misc.MultiMap;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.tree.ParseTree;
import java.util.Collections;
import java.util.List;
/** The result of matching a tree against a tree pattern.
* If there is no match, field mismatchedNode is set to the offending
* treenode.
/**
* Represents the result of matching a {@link ParseTree} against a tree pattern.
*/
public class ParseTreeMatch {
/** Tree we tried to match */
protected ParseTree tree;
/**
* This is the backing field for {@link #getTree()}.
*/
private final ParseTree tree;
/** To what pattern? */
protected ParseTreePattern pattern;
/**
* This is the backing field for {@link #getPattern()}.
*/
private final ParseTreePattern pattern;
/** Map a label or token/rule name to List of nodes in tree we match */
protected MultiMap<String, ParseTree> labels;
/**
* This is the backing field for {@link #getLabels()}.
*/
private final MultiMap<String, ParseTree> labels;
/** Where in actual parse tree we failed if we can't match pattern */
protected ParseTree mismatchedNode;
/**
* This is the backing field for {@link #getMismatchedNode()}.
*/
private final ParseTree mismatchedNode;
/**
* Constructs a new instance of {@link ParseTreeMatch} from the specified
* parse tree and pattern.
*
* @param tree The parse tree to match against the pattern.
* @param pattern The parse tree pattern.
* @param labels A mapping from label names to collections of
* {@link ParseTree} objects located by the tree pattern matching process.
* @param mismatchedNode The first node which failed to match the tree
* pattern during the matching process.
*
* @exception IllegalArgumentException if {@code tree} is {@code null}
* @exception IllegalArgumentException if {@code pattern} is {@code null}
* @exception IllegalArgumentException if {@code labels} is {@code null}
*/
public ParseTreeMatch(@NotNull ParseTree tree, @NotNull ParseTreePattern pattern, @NotNull MultiMap<String, ParseTree> labels, @Nullable ParseTree mismatchedNode) {
if (tree == null) {
throw new IllegalArgumentException("tree cannot be null");
}
if (pattern == null) {
throw new IllegalArgumentException("pattern cannot be null");
}
if (labels == null) {
throw new IllegalArgumentException("labels cannot be null");
}
public ParseTreeMatch(ParseTree tree, ParseTreePattern pattern) {
this.tree = tree;
this.pattern = pattern;
this.labels = new MultiMap<String, ParseTree>();
this.labels = labels;
this.mismatchedNode = mismatchedNode;
}
/** Get the node associated with label. E.g., for pattern <id:ID>,
* get("id") returns the node matched for that ID. If there are more than
* one nodes matched with that label, this returns the first one matched.
* If there is no node associated with the label, this returns null.
/**
* Get the first node associated with a specific {@code label}.
* <p/>
* For example, for pattern {@code <id:ID>}, {@code get("id")} returns the
* node matched for that {@code ID}. If more than one node
* matched the specified label, only the first is returned. If there is
* no node associated with the label, this returns {@code null}.
* <p/>
* Pattern tags like {@code <ID>} and {@code <expr>} without labels are
* considered to be labeled with {@code ID} and {@code expr}, respectively.
*
* Pattern tags like <ID> and <expr> without labels are considered to be
* labeled with ID and expr, respectively.
* @param label The label to check.
*
* @return The first {@link ParseTree} to match a tag with the specified
* label, or {@code null} if no parse tree matched a tag with the label.
*/
@Nullable
public ParseTree get(String label) {
List<ParseTree> parseTrees = labels.get(label);
if ( parseTrees==null ) return null;
if ( parseTrees==null ) {
return null;
}
return parseTrees.get(0); // return first if multiple
}
/** Return all nodes matched that are labeled with parameter label.
* If there is only one such node, return a list with one element.
* If there are no nodes matched with that label, return an empty list.
/**
* Return all nodes matching a rule or token tag with the specified label.
* <p/>
* If the {@code label} is the name of a parser rule or token in the
* grammar, the resulting list will contain both the parse trees matching
* rule or tags explicitly labeled with the label and the complete set of
* parse trees matching the labeled and unlabeled tags in the pattern for
* the parser rule or token. For example, if {@code label} is {@code "foo"},
* the result will contain <em>all</em> of the following.
*
* Pattern tags like <ID> and <expr> without labels are considered to be
* labeled with ID and expr, respectively.
* <ul>
* <li>Parse tree nodes matching tags of the form {@code <foo:anyRuleName>} and
* {@code <foo:AnyTokenName>}.</li>
* <li>Parse tree nodes matching tags of the form {@code <anyLabel:foo>}.</li>
* <li>Parse tree nodes matching tags of the form {@code <foo>}.</li>
* </ul>
*
* @param label The label.
*
* @return A collection of all {@link ParseTree} nodes matching tags with
* the specified {@code label}. If no nodes matched the label, an empty list
* is returned.
*/
public List<ParseTree> getAll(String label) {
@NotNull
public List<ParseTree> getAll(@NotNull String label) {
List<ParseTree> nodes = labels.get(label);
if ( nodes==null ) return Collections.emptyList();
if ( nodes==null ) {
return Collections.emptyList();
}
return nodes;
}
/** Return a mapping from label->[list of nodes]. This includes
* token and rule references such as <ID> and <expr>
/**
* Return a mapping from label &rarr; [list of nodes].
* <p/>
* The map includes special entries corresponding to the names of rules and
* tokens referenced in tags in the original pattern. For additional
* information, see the description of {@link #getAll(String)}.
*
* @return A mapping from labels to parse tree nodes. If the parse tree
* pattern did not contain any rule or token tags, this map will be empty.
*/
@NotNull
public MultiMap<String, ParseTree> getLabels() {
return labels;
}
/** Return the node at which we first detected a mismatch. Return
* null if we have not found an error.
/**
* Get the node at which we first detected a mismatch.
*
* @return the node at which we first detected a mismatch, or {@code null}
* if the match was successful.
*/
@Nullable
public ParseTree getMismatchedNode() {
return mismatchedNode;
}
/** Did the tree vs pattern match? */
/**
* Gets a value indicating whether the match operation succeeded.
*
* @return {@code true} if the match operation succeeded; otherwise,
* {@code false}.
*/
public boolean succeeded() {
return mismatchedNode==null;
return mismatchedNode == null;
}
/** Return the tree pattern we are matching against */
/**
* Get the tree pattern we are matching against.
*
* @return The tree pattern we are matching against.
*/
@NotNull
public ParseTreePattern getPattern() {
return pattern;
}
/** Return the parse tree we are trying to match to a pattern */
/**
* Get the parse tree we are trying to match to a pattern.
*
* @return The {@link ParseTree} we are trying to match to a pattern.
*/
@NotNull
public ParseTree getTree() {
return tree;
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
return tree.getText();
return String.format(
"Match %s; found %d labels",
succeeded() ? "succeeded" : "failed",
getLabels().size());
}
}

111
runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePattern.java
View File

@ -30,6 +30,7 @@
package org.antlr.v4.runtime.tree.pattern;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.xpath.XPath;
@ -37,17 +38,46 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/** A pattern like "<ID> = <expr>;" converted to a ParseTree by
* ParseTreePatternMatcher.compile().
/**
* A pattern like {@code <ID> = <expr>;} converted to a {@link ParseTree} by
* {@link ParseTreePatternMatcher#compile(String, int)}.
*/
public class ParseTreePattern {
protected int patternRuleIndex;
protected String pattern;
protected ParseTree patternTree;
public ParseTreePatternMatcher matcher;
/**
* This is the backing field for {@link #getPatternRuleIndex()}.
*/
private final int patternRuleIndex;
public ParseTreePattern(ParseTreePatternMatcher matcher,
String pattern, int patternRuleIndex, ParseTree patternTree)
/**
* This is the backing field for {@link #getPattern()}.
*/
@NotNull
private final String pattern;
/**
* This is the backing field for {@link #getPatternTree()}.
*/
@NotNull
private final ParseTree patternTree;
/**
* This is the backing field for {@link #getMatcher()}.
*/
@NotNull
private final ParseTreePatternMatcher matcher;
/**
* Construct a new instance of the {@link ParseTreePattern} class.
*
* @param matcher The {@link ParseTreePatternMatcher} which created this
* tree pattern.
* @param pattern The tree pattern in concrete syntax form.
* @param patternRuleIndex The parser rule which serves as the root of the
* tree pattern.
* @param patternTree The tree pattern in {@link ParseTree} form.
*/
public ParseTreePattern(@NotNull ParseTreePatternMatcher matcher,
@NotNull String pattern, int patternRuleIndex, @NotNull ParseTree patternTree)
{
this.matcher = matcher;
this.patternRuleIndex = patternRuleIndex;
@ -55,18 +85,43 @@ public class ParseTreePattern {
this.patternTree = patternTree;
}
public ParseTreeMatch match(ParseTree tree) {
/**
* Match a specific parse tree against this tree pattern.
*
* @param tree The parse tree to match against this tree pattern.
* @return A {@link ParseTreeMatch} object describing the result of the
* match operation. The {@link ParseTreeMatch#succeeded()} method can be
* used to determine whether or not the match was successful.
*/
@NotNull
public ParseTreeMatch match(@NotNull ParseTree tree) {
return matcher.match(tree, this);
}
public boolean matches(ParseTree tree) {
/**
* Determine whether or not a parse tree matches this tree pattern.
*
* @param tree The parse tree to match against this tree pattern.
* @return {@code true} if {@code tree} is a match for the current tree
* pattern; otherwise, {@code false}.
*/
public boolean matches(@NotNull ParseTree tree) {
return matcher.match(tree, this).succeeded();
}
/** Find all nodes using xpath and then try to match those subtrees
* against this tree pattern
/**
* Find all nodes using XPath and then try to match those subtrees against
* this tree pattern.
*
* @param tree The {@link ParseTree} to match against this pattern.
* @param xpath An expression matching the nodes
*
* @return A collection of {@link ParseTreeMatch} objects describing the
* successful matches. Unsuccessful matches are omitted from the result,
* regardless of the reason for the failure.
*/
public List<ParseTreeMatch> findAll(ParseTree tree, String xpath) {
@NotNull
public List<ParseTreeMatch> findAll(@NotNull ParseTree tree, @NotNull String xpath) {
Collection<ParseTree> subtrees = XPath.findAll(tree, xpath, matcher.getParser());
List<ParseTreeMatch> matches = new ArrayList<ParseTreeMatch>();
for (ParseTree t : subtrees) {
@ -78,18 +133,46 @@ public class ParseTreePattern {
return matches;
}
public ParseTreePatternMatcher getParseTreePattern() {
/**
* Get the {@link ParseTreePatternMatcher} which created this tree pattern.
*
* @return The {@link ParseTreePatternMatcher} which created this tree
* pattern.
*/
@NotNull
public ParseTreePatternMatcher getMatcher() {
return matcher;
}
/**
* Get the tree pattern in concrete syntax form.
*
* @return The tree pattern in concrete syntax form.
*/
@NotNull
public String getPattern() {
return pattern;
}
/**
* Get the parser rule which serves as the outermost rule for the tree
* pattern.
*
* @return The parser rule which serves as the outermost rule for the tree
* pattern.
*/
public int getPatternRuleIndex() {
return patternRuleIndex;
}
/**
* Get the tree pattern as a {@link ParseTree}. The rule and token tags from
* the pattern are present in the parse tree as terminal nodes with a symbol
* of type {@link RuleTagToken} or {@link TokenTagToken}.
*
* @return The tree pattern as a {@link ParseTree}.
*/
@NotNull
public ParseTree getPatternTree() {
return patternTree;
}

347
runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java
View File

@ -38,69 +38,74 @@ import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.ParserInterpreter;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.misc.MultiMap;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.RuleNode;
import org.antlr.v4.runtime.tree.TerminalNode;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/** A tree pattern matching mechanism for ANTLR ParseTrees.
*
* Patterns are strings of source input text with special tags
* representing token or rule references such as:
*
* "<ID> = <expr>;"
*
* Given a pattern start rule such as statement, this object
* construct a parse tree with placeholders for the identifier and
* expression subtree. Then the match() routines can compare an
* actual parse tree from a parse with this pattern. Tag <ID> matches
* any ID token and tag <expr> references any expression subtree.
*
* Pattern "x = 0;" is a similar pattern that matches the same
* pattern except that it requires the identifier to be x and the
* expression to be 0.
*
* The matches() routines return true or false based upon a match for
* the tree rooted at the parameter sent in. The match() routines
* return a ParseTreeMatch object that contains the parse tree, the
* parse tree pattern, and a map from tag name to matched nodes (more
* below). A subtree that fails to match, returns with
* ParseTreeMatch.mismatchedNode set to the first tree node that did
* not match.
*
* For efficiency, you can compile a tree pattern in string form to a
* ParseTreePattern object.
*
* See TestParseTreeMatcher for lots of examples. ParseTreePattern
* has two static helper methods: findAll() and match() that are easy
* to use but not superefficient because they create new
* ParseTreePatternMatcher objects each time and have to compile the
* pattern in string form before using it.
*
* The lexer and parser that you pass into the
* ParseTreePatternMatcher constructor are used to parse the pattern
* in string form. The lexer converts the "<ID> = <expr>;" into a
* sequence of four tokens (assuming lexer throws out whitespace or
* puts it on a hidden channel). Be aware that the input stream is
* reset for the lexer (but not the parser; a ParserInterpreter is
* created to parse the input.). Any user-defined fields you have put
* into the lexer might get changed when this mechanism asks it to
* scan the pattern string.
*
* Normally a parser does not accept token "<expr>" as a valid expr
* but, from the parser passed in, we create a special version of the
* underlying grammar representation (an ATN) that allows imaginary
* tokens representing rules (<expr>) to match entire rules.
* We call these bypass alternatives.
*
* Delimiters are < and > with \ as the escape string by default, but
* you can set them to whatever you want using setDelimiters(). You
* must escape both start and stop strings \< and \>.
/**
* A tree pattern matching mechanism for ANTLR {@link ParseTree}s.
* <p/>
* Patterns are strings of source input text with special tags representing
* token or rule references such as:
* <p/>
* {@code <ID> = <expr>;}
* <p/>
* Given a pattern start rule such as {@code statement}, this object constructs
* a {@link ParseTree} with placeholders for the {@code ID} and {@code expr}
* subtree. Then the {@link #match} routines can compare an actual
* {@link ParseTree} from a parse with this pattern. Tag {@code <ID>} matches
* any {@code ID} token and tag {@code <expr>} references the result of the
* {@code expr} rule (generally an instance of {@code ExprContext}.
* <p/>
* Pattern {@code x = 0;} is a similar pattern that matches the same pattern
* except that it requires the identifier to be {@code x} and the expression to
* be {@code 0}.
* <p/>
* The {@link #matches} routines return {@code true} or {@code false} based
* upon a match for the tree rooted at the parameter sent in. The
* {@link #match} routines return a {@link ParseTreeMatch} object that
* contains the parse tree, the parse tree pattern, and a map from tag name to
* matched nodes (more below). A subtree that fails to match, returns with
* {@link ParseTreeMatch#mismatchedNode} set to the first tree node that did not
* match.
* <p/>
* For efficiency, you can compile a tree pattern in string form to a
* {@link ParseTreePattern} object.
* <p/>
* See {@code TestParseTreeMatcher} for lots of examples.
* {@link ParseTreePattern} has two static helper methods:
* {@link ParseTreePattern#findAll} and {@link ParseTreePattern#match} that
* are easy to use but not super efficient because they create new
* {@link ParseTreePatternMatcher} objects each time and have to compile the
* pattern in string form before using it.
* <p/>
* The lexer and parser that you pass into the {@link ParseTreePatternMatcher}
* constructor are used to parse the pattern in string form. The lexer converts
* the {@code <ID> = <expr>;} into a sequence of four tokens (assuming lexer
* throws out whitespace or puts it on a hidden channel). Be aware that the
* input stream is reset for the lexer (but not the parser; a
* {@link ParserInterpreter} is created to parse the input.). Any user-defined
* fields you have put into the lexer might get changed when this mechanism asks
* it to scan the pattern string.
* <p/>
* Normally a parser does not accept token {@code <expr>} as a valid
* {@code expr} but, from the parser passed in, we create a special version of
* the underlying grammar representation (an {@link ATN}) that allows imaginary
* tokens representing rules ({@code <expr>}) to match entire rules. We call
* these <em>bypass alternatives</em>.
* <p/>
* Delimiters are {@code <} and {@code >}, with {@code \} as the escape string
* by default, but you can set them to whatever you want using
* {@link #setDelimiters}. You must escape both start and stop strings
* {@code \<} and {@code \>}.
*/
public class ParseTreePatternMatcher {
public static class CannotInvokeStartRule extends RuntimeException {
@ -109,72 +114,97 @@ public class ParseTreePatternMatcher {
}
}
/** Used to convert the tree pattern string into a series of tokens.
* The input stream is reset.
/**
* This is the backing field for {@link #getLexer()}.
*/
protected Lexer lexer;
private final Lexer lexer;
/** Used to collect to the grammar file name, token names, rule names
* for used to parse the pattern into a parse tree.
/**
* This is the backing field for {@link #getParser()}.
*/
protected Parser parser;
private final Parser parser;
protected String start = "<", stop=">";
protected String start = "<";
protected String stop = ">";
protected String escape = "\\"; // e.g., \< and \> must escape BOTH!
/** Constructs a pattern match or from a lecture and parser object.
* The lexer input stream is altered for tokenizing the tree patterns.
* The parser is used as a convenient mechanism to get the grammar name,
* plus token, rule names.
/**
* Constructs a {@link ParseTreePatternMatcher} or from a {@link Lexer} and
* {@link Parser} object. The lexer input stream is altered for tokenizing
* the tree patterns. The parser is used as a convenient mechanism to get
* the grammar name, plus token, rule names.
*/
public ParseTreePatternMatcher(Lexer lexer, Parser parser) {
this.lexer = lexer;
this.parser = parser;
}
/**
* Set the delimiters used for marking rule and token tags within concrete
* syntax used by the tree pattern parser.
*
* @param start The start delimiter.
* @param stop The stop delimiter.
* @param escapeLeft The escape sequence to use for escaping a start or stop delimiter.
*
* @exception IllegalArgumentException if {@code start} is {@code null} or empty.
* @exception IllegalArgumentException if {@code stop} is {@code null} or empty.
*/
public void setDelimiters(String start, String stop, String escapeLeft) {
if (start == null || start.isEmpty()) {
throw new IllegalArgumentException("start cannot be null or empty");
}
if (stop == null || stop.isEmpty()) {
throw new IllegalArgumentException("stop cannot be null or empty");
}
this.start = start;
this.stop = stop;
this.escape = escapeLeft;
}
/** Does pattern matched as rule patternRuleIndex match tree? */
/** Does {@code pattern} matched as rule {@code patternRuleIndex} match {@code tree}? */
public boolean matches(ParseTree tree, String pattern, int patternRuleIndex) {
ParseTreePattern p = compile(pattern, patternRuleIndex);
return matches(tree, p);
}
/** Does pattern matched as rule patternRuleIndex match tree? Pass in a
/** Does {@code pattern} matched as rule patternRuleIndex match tree? Pass in a
* compiled pattern instead of a string representation of a tree pattern.
*/
public boolean matches(ParseTree tree, ParseTreePattern pattern) {
ParseTreeMatch match = new ParseTreeMatch(tree, pattern);
matches_(tree, pattern.patternTree, match);
return match.succeeded();
MultiMap<String, ParseTree> labels = new MultiMap<String, ParseTree>();
ParseTree mismatchedNode = matchImpl(tree, pattern.getPatternTree(), labels);
return mismatchedNode == null;
}
/** Compare pattern matched as rule patternRuleIndex against tree and
* return a ParseTreeMatch object that contains the matched elements,
* or the node at which the match failed.
/**
* Compare {@code pattern} matched as rule {@code patternRuleIndex} against
* {@code tree} and return a {@link ParseTreeMatch} object that contains the
* matched elements, or the node at which the match failed.
*/
public ParseTreeMatch match(ParseTree tree, String pattern, int patternRuleIndex) {
ParseTreePattern p = compile(pattern, patternRuleIndex);
return match(tree, p);
}
/** Compare pattern matched against tree and
* return a ParseTreeMatch object that contains the matched elements,
* or the node at which the match failed. Pass in a compiled pattern
* instead of a string representation of a tree pattern.
/**
* Compare {@code pattern} matched against {@code tree} and return a
* {@link ParseTreeMatch} object that contains the matched elements, or the
* node at which the match failed. Pass in a compiled pattern instead of a
* string representation of a tree pattern.
*/
public ParseTreeMatch match(ParseTree tree, ParseTreePattern pattern) {
ParseTreeMatch match = new ParseTreeMatch(tree, pattern);
matches_(tree, pattern.patternTree, match);
return match;
@NotNull
public ParseTreeMatch match(@NotNull ParseTree tree, @NotNull ParseTreePattern pattern) {
MultiMap<String, ParseTree> labels = new MultiMap<String, ParseTree>();
ParseTree mismatchedNode = matchImpl(tree, pattern.getPatternTree(), labels);
return new ParseTreeMatch(tree, pattern, labels, mismatchedNode);
}
/** For repeated use of a tree pattern, compile it to a ParseTreePattern
* using this method.
/**
* For repeated use of a tree pattern, compile it to a
* {@link ParseTreePattern} using this method.
*/
public ParseTreePattern compile(String pattern, int patternRuleIndex) {
List<? extends Token> tokenList = tokenize(pattern);
@ -199,88 +229,131 @@ public class ParseTreePatternMatcher {
return new ParseTreePattern(this, pattern, patternRuleIndex, tree);
}
/**
* Used to convert the tree pattern string into a series of tokens. The
* input stream is reset.
*/
@NotNull
public Lexer getLexer() {
return lexer;
}
/**
* Used to collect to the grammar file name, token names, rule names for
* used to parse the pattern into a parse tree.
*/
@NotNull
public Parser getParser() {
return parser;
}
// ---- SUPPORT CODE ----
/** Recursively walk tree against patternTree, filling match.labels */
protected boolean matches_(ParseTree tree,
ParseTree patternTree,
ParseTreeMatch match)
/**
* Recursively walk {@code tree} against {@code patternTree}, filling
* {@code match.}{@link ParseTreeMatch#labels labels}.
*
* @return the first node encountered in {@code tree} which does not match
* a corresponding node in {@code patternTree}, or {@code null} if the match
* was successful. The specific node returned depends on the matching
* algorithm used by the implementation, and may be overridden.
*/
@Nullable
protected ParseTree matchImpl(@NotNull ParseTree tree,
@NotNull ParseTree patternTree,
@NotNull MultiMap<String, ParseTree> labels)
{
if ( tree==null || patternTree==null ) {
return false;
if (tree == null) {
throw new IllegalArgumentException("tree cannot be null");
}
if (patternTree == null) {
throw new IllegalArgumentException("patternTree cannot be null");
}
// x and <ID>, x and y, or x and x; or could be mismatched types
if ( tree instanceof TerminalNode && patternTree instanceof TerminalNode ) {
TerminalNode t1 = (TerminalNode)tree;
TerminalNode t2 = (TerminalNode)patternTree;
ParseTreeMatch m = null;
ParseTree mismatchedNode = null;
// both are tokens and they have same type
if ( t1.getSymbol().getType() == t2.getSymbol().getType() ) {
if ( t2.getSymbol() instanceof TokenTagToken ) { // x and <ID>
TokenTagToken tokenTagToken = (TokenTagToken)t2.getSymbol();
// track label->list-of-nodes for both token name and label (if any)
match.labels.map(tokenTagToken.tokenName, tree);
if ( tokenTagToken.label!=null ) {
match.labels.map(tokenTagToken.label, tree);
labels.map(tokenTagToken.getTokenName(), tree);
if ( tokenTagToken.getLabel()!=null ) {
labels.map(tokenTagToken.getLabel(), tree);
}
}
else if ( t1.getText().equals(t2.getText()) ) { // x and x
else if ( t1.getText().equals(t2.getText()) ) {
// x and x
}
else { // x and y
match.mismatchedNode = t1;
else {
// x and y
if (mismatchedNode == null) {
mismatchedNode = t1;
}
}
}
else {
match.mismatchedNode = t1;
if (mismatchedNode == null) {
mismatchedNode = t1;
}
}
return match.succeeded();
return mismatchedNode;
}
if ( tree instanceof ParserRuleContext && patternTree instanceof ParserRuleContext ) {
ParserRuleContext r1 = (ParserRuleContext)tree;
ParserRuleContext r2 = (ParserRuleContext)patternTree;
ParseTree mismatchedNode = null;
// (expr ...) and <expr>
RuleTagToken ruleTagToken = getRuleTagToken(r2);
if ( ruleTagToken!=null ) {
ParseTreeMatch m = null;
if ( r1.getRuleContext().getRuleIndex() == r2.getRuleContext().getRuleIndex() ) {
// track label->list-of-nodes for both rule name and label (if any)
match.labels.map(ruleTagToken.ruleName, tree);
if ( ruleTagToken.label!=null ) {
match.labels.map(ruleTagToken.label, tree);
labels.map(ruleTagToken.getRuleName(), tree);
if ( ruleTagToken.getLabel()!=null ) {
labels.map(ruleTagToken.getLabel(), tree);
}
}
else {
match.mismatchedNode = r1;
if (mismatchedNode == null) {
mismatchedNode = r1;
}
}
return match.succeeded();
return mismatchedNode;
}
// (expr ...) and (expr ...)
if ( r1.getChildCount()!=r2.getChildCount() ) {
match.mismatchedNode = r1;
return false;
if (mismatchedNode == null) {
mismatchedNode = r1;
}
return mismatchedNode;
}
int n = r1.getChildCount();
for (int i = 0; i<n; i++) {
boolean childMatch =
matches_(r1.getChild(i), patternTree.getChild(i), match);
if ( !childMatch ) return false;
ParseTree childMatch = matchImpl(r1.getChild(i), patternTree.getChild(i), labels);
if ( childMatch != null ) {
return childMatch;
}
}
return true;
return mismatchedNode;
}
// if nodes aren't both tokens or both rule nodes, can't match
match.mismatchedNode = tree;
return false;
return tree;
}
/** Is t (expr <expr>) subtree? */
/** Is {@code t} {@code (expr <expr>)} subtree? */
protected RuleTagToken getRuleTagToken(ParseTree t) {
if ( t instanceof RuleNode ) {
RuleNode r = (RuleNode)t;
@ -305,40 +378,34 @@ public class ParseTreePatternMatcher {
if ( chunk instanceof TagChunk ) {
TagChunk tagChunk = (TagChunk)chunk;
// add special rule token or conjure up new token from name
if ( Character.isUpperCase(tagChunk.tag.charAt(0)) ) {
Integer ttype = parser.getTokenType(tagChunk.tag);
if ( Character.isUpperCase(tagChunk.getTag().charAt(0)) ) {
Integer ttype = parser.getTokenType(tagChunk.getTag());
if ( ttype==Token.INVALID_TYPE ) {
throw new IllegalArgumentException("Unknown token "+tagChunk.tag+" in pattern: "+pattern);
throw new IllegalArgumentException("Unknown token "+tagChunk.getTag()+" in pattern: "+pattern);
}
TokenTagToken t = new TokenTagToken(tagChunk.tag, ttype, tagChunk.label);
TokenTagToken t = new TokenTagToken(tagChunk.getTag(), ttype, tagChunk.getLabel());
tokens.add(t);
}
else if ( Character.isLowerCase(tagChunk.tag.charAt(0)) ) {
int ruleIndex = parser.getRuleIndex(tagChunk.tag);
else if ( Character.isLowerCase(tagChunk.getTag().charAt(0)) ) {
int ruleIndex = parser.getRuleIndex(tagChunk.getTag());
if ( ruleIndex==-1 ) {
throw new IllegalArgumentException("Unknown rule "+tagChunk.tag+" in pattern: "+pattern);
throw new IllegalArgumentException("Unknown rule "+tagChunk.getTag()+" in pattern: "+pattern);
}
int ruleImaginaryTokenType = parser.getATNWithBypassAlts().ruleToTokenType[ruleIndex];
tokens.add(new RuleTagToken(tagChunk.tag, ruleImaginaryTokenType, tagChunk.label));
tokens.add(new RuleTagToken(tagChunk.getTag(), ruleImaginaryTokenType, tagChunk.getLabel()));
}
else {
throw new IllegalArgumentException("invalid tag: "+tagChunk.tag+" in pattern: "+pattern);
throw new IllegalArgumentException("invalid tag: "+tagChunk.getTag()+" in pattern: "+pattern);
}
}
else {
TextChunk textChunk = (TextChunk)chunk;
try {
ANTLRInputStream in = new ANTLRInputStream(new StringReader(textChunk.text));
lexer.setInputStream(in);
Token t = lexer.nextToken();
while ( t.getType()!=Token.EOF ) {
tokens.add(t);
t = lexer.nextToken();
}
}
catch (IOException ioe) {
// -----------------
throw new IllegalArgumentException("IOException lexing pattern: "+pattern, ioe);
ANTLRInputStream in = new ANTLRInputStream(textChunk.getText());
lexer.setInputStream(in);
Token t = lexer.nextToken();
while ( t.getType()!=Token.EOF ) {
tokens.add(t);
t = lexer.nextToken();
}
}
}
@ -347,12 +414,12 @@ public class ParseTreePatternMatcher {
return tokens;
}
/** Split "<ID> = <e:expr> ;" into 4 chunks for tokenizing by tokenize() */
/** Split {@code <ID> = <e:expr> ;} into 4 chunks for tokenizing by {@link #tokenize}. */
public List<Chunk> split(String pattern) {
int p = 0;
int n = pattern.length();
List<Chunk> chunks = new ArrayList<Chunk>();
StringBuffer buf = new StringBuffer();
StringBuilder buf = new StringBuilder();
// find all start and stop indexes first, then collect
List<Integer> starts = new ArrayList<Integer>();
List<Integer> stops = new ArrayList<Integer>();
@ -430,10 +497,14 @@ public class ParseTreePatternMatcher {
}
// strip out the escape sequences from text chunks but not tags
for (Chunk c : chunks) {
for (int i = 0; i < chunks.size(); i++) {
Chunk c = chunks.get(i);
if ( c instanceof TextChunk ) {
TextChunk tc = (TextChunk)c;
tc.text = tc.text.replace(escape, "");
String unescaped = tc.getText().replace(escape, "");
if (unescaped.length() < tc.getText().length()) {
chunks.set(i, new TextChunk(unescaped));
}
}
}

159
runtime/Java/src/org/antlr/v4/runtime/tree/pattern/RuleTagToken.java
View File

@ -33,75 +33,200 @@ package org.antlr.v4.runtime.tree.pattern;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenSource;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
/** A token object representing an entire subtree matched by a rule; e.g., <expr> */
/**
* A {@link Token} object representing an entire subtree matched by a parser
* rule; e.g., {@code <expr>}. These tokens are created for {@link TagChunk}
* chunks where the tag corresponds to a parser rule.
*/
public class RuleTagToken implements Token {
protected String ruleName;
protected int ruleImaginaryTokenType;
protected String label;
/**
* This is the backing field for {@link #getRuleName}.
*/
private final String ruleName;
/**
* The token type for the current token. This is the token type assigned to
* the bypass alternative for the rule during ATN deserialization.
*/
private final int bypassTokenType;
/**
* This is the backing field for {@link #getLabel}.
*/
private final String label;
public RuleTagToken(String ruleName, int ruleImaginaryTokenType) {
this.ruleName = ruleName;
this.ruleImaginaryTokenType = ruleImaginaryTokenType;
/**
* Constructs a new instance of {@link RuleTagToken} with the specified rule
* name and bypass token type and no label.
*
* @param ruleName The name of the parser rule this rule tag matches.
* @param bypassTokenType The bypass token type assigned to the parser rule.
*
* @exception IllegalArgumentException if {@code ruleName} is {@code null}
* or empty.
*/
public RuleTagToken(@NotNull String ruleName, int bypassTokenType) {
this(ruleName, bypassTokenType, null);
}
public RuleTagToken(String ruleName, int ruleImaginaryTokenType, String label) {
this(ruleName, ruleImaginaryTokenType);
/**
* Constructs a new instance of {@link RuleTagToken} with the specified rule
* name, bypass token type, and label.
*
* @param ruleName The name of the parser rule this rule tag matches.
* @param bypassTokenType The bypass token type assigned to the parser rule.
* @param label The label associated with the rule tag, or {@code null} if
* the rule tag is unlabeled.
*
* @exception IllegalArgumentException if {@code ruleName} is {@code null}
* or empty.
*/
public RuleTagToken(@NotNull String ruleName, int bypassTokenType, @Nullable String label) {
if (ruleName == null || ruleName.isEmpty()) {
throw new IllegalArgumentException("ruleName cannot be null or empty.");
}
this.ruleName = ruleName;
this.bypassTokenType = bypassTokenType;
this.label = label;
}
/**
* Gets the name of the rule associated with this rule tag.
*
* @return The name of the parser rule associated with this rule tag.
*/
@NotNull
public final String getRuleName() {
return ruleName;
}
/**
* Gets the label associated with the rule tag.
*
* @return The name of the label associated with the rule tag, or
* {@code null} if this is an unlabeled rule tag.
*/
@Nullable
public final String getLabel() {
return label;
}
/**
* {@inheritDoc}
* <p/>
* Rule tag tokens are always placed on the {@link #DEFAULT_CHANNEL}.
*/
@Override
public int getChannel() {
return 0;
return DEFAULT_CHANNEL;
}
/**
* {@inheritDoc}
* <p/>
* This method returns the rule tag formatted with {@code <} and {@code >}
* delimiters.
*/
@Override
public String getText() {
return "<"+ruleName+">";
if (label != null) {
return "<" + label + ":" + ruleName + ">";
}
return "<" + ruleName + ">";
}
/**
* {@inheritDoc}
* <p/>
* Rule tag tokens have types assigned according to the rule bypass
* transitions created during ATN deserialization.
*/
@Override
public int getType() {
return ruleImaginaryTokenType;
return bypassTokenType;
}
/**
* {@inheritDoc}
* <p/>
* The implementation for {@link RuleTagToken} always returns 0.
*/
@Override
public int getLine() {
return 0;
}
/**
* {@inheritDoc}
* <p/>
* The implementation for {@link RuleTagToken} always returns -1.
*/
@Override
public int getCharPositionInLine() {
return 0;
return -1;
}
/**
* {@inheritDoc}
* <p/>
* The implementation for {@link RuleTagToken} always returns -1.
*/
@Override
public int getTokenIndex() {
return 0;
return -1;
}
/**
* {@inheritDoc}
* <p/>
* The implementation for {@link RuleTagToken} always returns -1.
*/
@Override
public int getStartIndex() {
return 0;
return -1;
}
/**
* {@inheritDoc}
* <p/>
* The implementation for {@link RuleTagToken} always returns -1.
*/
@Override
public int getStopIndex() {
return 0;
return -1;
}
/**
* {@inheritDoc}
* <p/>
* The implementation for {@link RuleTagToken} always returns {@code null}.
*/
@Override
public TokenSource getTokenSource() {
return null;
}
/**
* {@inheritDoc}
* <p/>
* The implementation for {@link RuleTagToken} always returns {@code null}.
*/
@Override
public CharStream getInputStream() {
return null;
}
/**
* {@inheritDoc}
* <p/>
* The implementation for {@link RuleTagToken} returns a string of the form
* {@code ruleName:bypassTokenType}.
*/
@Override
public String toString() {
return ruleName+":"+ ruleImaginaryTokenType;
return ruleName + ":" + bypassTokenType;
}
}

90
runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TagChunk.java
View File

@ -30,22 +30,100 @@
package org.antlr.v4.runtime.tree.pattern;
/** A tag in a tree pattern string such as <expr>, <ID>, <id:ID> etc... */
class TagChunk extends Chunk { // <e:expr> or <ID>
public String tag;
public String label;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
/**
* Represents a placeholder tag in a tree pattern. A tag can have any of the
* following forms.
*
* <ul>
* <li>{@code expr}: An unlabeled placeholder for a parser rule {@code expr}.</li>
* <li>{@code ID}: An unlabeled placeholder for a token of type {@code ID}.</li>
* <li>{@code e:expr}: A labeled placeholder for a parser rule {@code expr}.</li>
* <li>{@code id:ID}: A labeled placeholder for a token of type {@code ID}.</li>
* </ul>
*
* This class does not perform any validation on the tag or label names aside
* from ensuring that the tag is a non-null, non-empty string.
*/
class TagChunk extends Chunk {
/**
* This is the backing field for {@link #getTag}.
*/
private final String tag;
/**
* This is the backing field for {@link #getLabel}.
*/
private final String label;
/**
* Construct a new instance of {@link TagChunk} using the specified tag and
* no label.
*
* @param tag The tag, which should be the name of a parser rule or token
* type.
*
* @exception IllegalArgumentException if {@code tag} is {@code null} or
* empty.
*/
public TagChunk(String tag) {
this.tag = tag;
this(null, tag);
}
/**
* Construct a new instance of {@link TagChunk} using the specified label
* and tag.
*
* @param label The label for the tag. If this is {@code null}, the
* {@link TagChunk} represents an unlabeled tag.
* @param tag The tag, which should be the name of a parser rule or token
* type.
*
* @exception IllegalArgumentException if {@code tag} is {@code null} or
* empty.
*/
public TagChunk(String label, String tag) {
if (tag == null || tag.isEmpty()) {
throw new IllegalArgumentException("tag cannot be null or empty");
}
this.label = label;
this.tag = tag;
}
/**
* Get the tag for this chunk.
*
* @return The tag for the chunk.
*/
@NotNull
public final String getTag() {
return tag;
}
/**
* Get the label, if any, assigned to this chunk.
*
* @return The label assigned to this chunk, or {@code null} if no label is
* assigned to the chunk.
*/
@Nullable
public final String getLabel() {
return label;
}
/**
* This method returns a text representation of the tag chunk. Labeled tags
* are returned in the form {@code label:tag}, and unlabeled tags are
* returned as just the tag name.
*/
@Override
public String toString() {
if ( label!=null ) return label+":"+tag;
if (label != null) {
return label + ":" + tag;
}
return tag;
}
}

42
runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TextChunk.java
View File

@ -30,13 +30,49 @@
package org.antlr.v4.runtime.tree.pattern;
/** The "sea" of text surrounding tags in a tree pattern string */
import org.antlr.v4.runtime.misc.NotNull;
/**
* Represents a span of raw text (concrete syntax) between tags in a tree
* pattern string.
*/
class TextChunk extends Chunk {
public String text;
public TextChunk(String text) {
/**
* This is the backing field for {@link #getText}.
*/
@NotNull
private final String text;
/**
* Constructs a new instance of {@link TextChunk} with the specified text.
*
* @param text The text of this chunk.
* @exception IllegalArgumentException if {@code text} is {@code null}.
*/
public TextChunk(@NotNull String text) {
if (text == null) {
throw new IllegalArgumentException("text cannot be null");
}
this.text = text;
}
/**
* Gets the raw text of this chunk.
*
* @return The text of the chunk.
*/
@NotNull
public final String getText() {
return text;
}
/**
* {@inheritDoc}
* <p/>
* The implementation for {@link TextChunk} returns the result of
* {@link #getText()} in single quotes.
*/
@Override
public String toString() {
return "'"+text+"'";

91
runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TokenTagToken.java
View File

@ -31,29 +31,96 @@
package org.antlr.v4.runtime.tree.pattern;
import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
/** A Token in a tree pattern string representing a token reference; e.g., <ID> */
/**
* A {@link Token} object representing a token of a particular type; e.g.,
* {@code <ID>}. These tokens are created for {@link TagChunk} chunks where the
* tag corresponds to a lexer rule or token type.
*/
public class TokenTagToken extends CommonToken {
protected String tokenName;
protected String label;
/**
* This is the backing field for {@link #getTokenName}.
*/
@NotNull
private final String tokenName;
/**
* This is the backing field for {@link #getLabel}.
*/
@Nullable
private final String label;
public TokenTagToken(String tokenName, int type) {
super(type);
this.tokenName = tokenName;
/**
* Constructs a new instance of {@link TokenTagToken} for an unlabeled tag
* with the specified token name and type.
*
* @param tokenName The token name.
* @param type The token type.
*/
public TokenTagToken(@NotNull String tokenName, int type) {
this(tokenName, type, null);
}
public TokenTagToken(String tokenName, int type, String label) {
this(tokenName, type);
/**
* Constructs a new instance of {@link TokenTagToken} with the specified
* token name, type, and label.
*
* @param tokenName The token name.
* @param type The token type.
* @param label The label associated with the token tag, or {@code null} if
* the token tag is unlabeled.
*/
public TokenTagToken(@NotNull String tokenName, int type, @Nullable String label) {
super(type);
this.tokenName = tokenName;
this.label = label;
}
@Override
public String getText() {
return "<"+tokenName+">";
/**
* Gets the token name.
* @return The token name.
*/
@NotNull
public final String getTokenName() {
return tokenName;
}
/**
* Gets the label associated with the rule tag.
*
* @return The name of the label associated with the rule tag, or
* {@code null} if this is an unlabeled rule tag.
*/
@Nullable
public final String getLabel() {
return label;
}
/**
* {@inheritDoc}
* <p/>
* The implementation for {@link TokenTagToken} returns the token tag
* formatted with {@code <} and {@code >} delimiters.
*/
@Override
public String getText() {
if (label != null) {
return "<" + label + ":" + tokenName + ">";
}
return "<" + tokenName + ">";
}
/**
* {@inheritDoc}
* <p/>
* The implementation for {@link TokenTagToken} returns a string of the form
* {@code tokenName:type}.
*/
@Override
public String toString() {
return tokenName+":"+type;
return tokenName + ":" + type;
}
}