Merge pull request #378 from sharwell/polish

Tree patterns polish

runtime/Java/src/org/antlr/v4/runtime/Lexer.java

@@ -188,14 +188,6 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator>
 		}
 	}
 
-	/** When parsing "x = <expr>;" pattern, we use nextTokenOrRuleToken() not
-	 *  nextToken() so <expr> is converted to RULE token instead of tokenizing.
-	 */
-	public Token nextTokenOrRuleToken() {
-		//if ( '<' id '>' or special pattern ) return RULE token for id
-		return nextToken();
-	}
-
 	/** Instruct the lexer to skip creating a token for current lexer rule
 	 *  and look for another token.  nextToken() knows to keep looking when
 	 *  a lexer rule finishes with token set to SKIP_TOKEN.  Recall that

runtime/Java/src/org/antlr/v4/runtime/ListTokenSource.java

@@ -1,63 +1,230 @@
 package org.antlr.v4.runtime;
 
-import org.antlr.v4.runtime.CharStream;
-import org.antlr.v4.runtime.CommonToken;
-import org.antlr.v4.runtime.CommonTokenFactory;
-import org.antlr.v4.runtime.Token;
-import org.antlr.v4.runtime.TokenFactory;
-import org.antlr.v4.runtime.TokenSource;
 import org.antlr.v4.runtime.misc.NotNull;
+import org.antlr.v4.runtime.misc.Pair;
 
 import java.util.List;
 
-/** Convert a list of token objects to a source of tokens for a TokenStream. */
+/**
+ * Provides an implementation of {@link TokenSource} as a wrapper around a list
+ * of {@link Token} objects.
+ * <p/>
+ * If the final token in the list is an {@link Token#EOF} token, it will be used
+ * as the EOF token for every call to {@link #nextToken} after the end of the
+ * list is reached. Otherwise, an EOF token will be created.
+ */
 public class ListTokenSource implements TokenSource {
-	protected List<? extends Token> tokens;
+	/**
-	protected int i = 0;
+	 * The wrapped collection of {@link Token} objects to return.
-	protected TokenFactory<?> _factory = CommonTokenFactory.DEFAULT;
+	 */
+	protected final List<? extends Token> tokens;
 
-	public ListTokenSource(List<? extends Token> tokens) {
+	/**
-		assert tokens!=null;
+	 * The name of the input source. If this value is {@code null}, a call to
-		this.tokens = tokens;
+	 * {@link #getSourceName} should return the source name used to create the
+	 * the next token in {@link #tokens} (or the previous token if the end of
+	 * the input has been reached).
+	 */
+	private final String sourceName;
+
+	/**
+	 * The index into {@link #tokens} of token to return by the next call to
+	 * {@link #nextToken}. The end of the input is indicated by this value
+	 * being greater than or equal to the number of items in {@link #tokens}.
+	 */
+	protected int i;
+
+	/**
+	 * This field caches the EOF token for the token source.
+	 */
+	protected Token eofToken;
+
+	/**
+	 * This is the backing field for {@link #getTokenFactory} and
+	 * {@link setTokenFactory}.
+	 */
+	private TokenFactory<?> _factory = CommonTokenFactory.DEFAULT;
+
+	/**
+	 * Constructs a new {@link ListTokenSource} instance from the specified
+	 * collection of {@link Token} objects.
+	 *
+	 * @param tokens The collection of {@link Token} objects to provide as a
+	 * {@link TokenSource}.
+	 * @exception NullPointerException if {@code tokens} is {@code null}
+	 */
+	public ListTokenSource(@NotNull List<? extends Token> tokens) {
+		this(tokens, null);
 	}
 
+	/**
+	 * Constructs a new {@link ListTokenSource} instance from the specified
+	 * collection of {@link Token} objects and source name.
+	 *
+	 * @param tokens The collection of {@link Token} objects to provide as a
+	 * {@link TokenSource}.
+	 * @param sourceName The name of the {@link TokenSource}. If this value is
+	 * {@code null}, {@link #getSourceName} will attempt to infer the name from
+	 * the next {@link Token} (or the previous token if the end of the input has
+	 * been reached).
+	 *
+	 * @exception NullPointerException if {@code tokens} is {@code null}
+	 */
+	public ListTokenSource(@NotNull List<? extends Token> tokens, String sourceName) {
+		if (tokens == null) {
+			throw new NullPointerException("tokens cannot be null");
+		}
+
+		this.tokens = tokens;
+		this.sourceName = sourceName;
+	}
+
+	/**
+	 * @inheritDoc
+	 */
 	@Override
 	public int getCharPositionInLine() {
-		return -1;
+		if (i < tokens.size()) {
+			return tokens.get(i).getCharPositionInLine();
+		}
+		else if (eofToken != null) {
+			return eofToken.getCharPositionInLine();
+		}
+		else if (tokens.size() > 0) {
+			// have to calculate the result from the line/column of the previous
+			// token, along with the text of the token.
+			Token lastToken = tokens.get(tokens.size() - 1);
+			String tokenText = lastToken.getText();
+			if (tokenText != null) {
+				int lastNewLine = tokenText.lastIndexOf('\n');
+				if (lastNewLine >= 0) {
+					return tokenText.length() - lastNewLine - 1;
+				}
+			}
+
+			return lastToken.getCharPositionInLine() + lastToken.getStopIndex() - lastToken.getStartIndex() + 1;
+		}
+
+		// only reach this if tokens is empty, meaning EOF occurs at the first
+		// position in the input
+		return 0;
 	}
 
+	/**
+	 * @inheritDoc
+	 */
 	@Override
 	public Token nextToken() {
-		if ( i>=tokens.size() ) {
+		if (i >= tokens.size()) {
-			Token eof = new CommonToken(Token.EOF); // ignore factory
+			if (eofToken == null) {
-			return eof;
+				int start = -1;
+				if (tokens.size() > 0) {
+					int previousStop = tokens.get(tokens.size() - 1).getStopIndex();
+					if (previousStop != -1) {
+						start = previousStop + 1;
+					}
+				}
+
+				int stop = Math.max(-1, start - 1);
+				eofToken = _factory.create(new Pair<TokenSource, CharStream>(this, getInputStream()), Token.EOF, "EOF", Token.DEFAULT_CHANNEL, start, stop, getLine(), getCharPositionInLine());
+			}
+
+			return eofToken;
 		}
+
 		Token t = tokens.get(i);
+		if (i == tokens.size() - 1 && t.getType() == Token.EOF) {
+			eofToken = t;
+		}
+
 		i++;
 		return t;
 	}
 
+	/**
+	 * @inheritDoc
+	 */
 	@Override
 	public int getLine() {
-		return 0;
+		if (i < tokens.size()) {
+			return tokens.get(i).getLine();
+		}
+		else if (eofToken != null) {
+			return eofToken.getLine();
+		}
+		else if (tokens.size() > 0) {
+			// have to calculate the result from the line/column of the previous
+			// token, along with the text of the token.
+			Token lastToken = tokens.get(tokens.size() - 1);
+			int line = lastToken.getLine();
+
+			String tokenText = lastToken.getText();
+			if (tokenText != null) {
+				for (int i = 0; i < tokenText.length(); i++) {
+					if (tokenText.charAt(i) == '\n') {
+						line++;
+					}
+				}
+			}
+
+			// if no text is available, assume the token did not contain any newline characters.
+			return line;
+		}
+
+		// only reach this if tokens is empty, meaning EOF occurs at the first
+		// position in the input
+		return 1;
 	}
 
+	/**
+	 * @inheritDoc
+	 */
 	@Override
 	public CharStream getInputStream() {
+		if (i < tokens.size()) {
+			return tokens.get(i).getInputStream();
+		}
+		else if (eofToken != null) {
+			return eofToken.getInputStream();
+		}
+		else if (tokens.size() > 0) {
+			return tokens.get(tokens.size() - 1).getInputStream();
+		}
+
+		// no input stream information is available
 		return null;
 	}
 
+	/**
+	 * @inheritDoc
+	 */
 	@Override
 	public String getSourceName() {
-		return "<List>";
+		if (sourceName != null) {
+			return sourceName;
+		}
+
+		CharStream inputStream = getInputStream();
+		if (inputStream != null) {
+			return inputStream.getSourceName();
+		}
+
+		return "List";
 	}
 
+	/**
+	 * @inheritDoc
+	 */
 	@Override
 	public void setTokenFactory(@NotNull TokenFactory<?> factory) {
 		this._factory = factory;
 	}
 
+	/**
+	 * @inheritDoc
+	 */
 	@Override
+	@NotNull
 	public TokenFactory<?> getTokenFactory() {
 		return _factory;
 	}

runtime/Java/src/org/antlr/v4/runtime/Parser.java

@@ -51,6 +51,8 @@ import org.antlr.v4.runtime.tree.pattern.ParseTreePatternMatcher;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.WeakHashMap;
 
 /** This is all the parsing support code essentially; most of it is error recovery stuff. */
 public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
@@ -98,6 +100,15 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
 		}
 	}
 
+	/**
+	 * This field maps from the serialized ATN string to the deserialized {@link ATN} with
+	 * bypass alternatives.
+	 *
+	 * @see ATNDeserializationOptions#isGenerateRuleBypassTransitions()
+	 */
+	private static final Map<String, ATN> bypassAltsAtnCache =
+		new WeakHashMap<String, ATN>();
+
 	/**
 	 * The error handling strategy for the parser. The default value is a new
 	 * instance of {@link DefaultErrorStrategy}.
@@ -435,36 +446,43 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
 		_input.getTokenSource().setTokenFactory(factory);
 	}
 
-	/** The ATN with bypass alternatives is expensive to create so we create it lazily.
+	/**
-	 *  The actual generated parsers override this method. These are just
+	 * The ATN with bypass alternatives is expensive to create so we create it
-	 *  setters/getters. createATNWithBypassAlts() does the creation.
+	 * lazily.
+	 *
+	 * @throws UnsupportedOperationException if the current parser does not
+	 * implement the {@link #getSerializedATN()} method.
 	 */
-	public void setATNWithBypassAlts(ATN atn) { }
+	@NotNull
-	public ATN getATNWithBypassAlts() { return null; }
+	public ATN getATNWithBypassAlts() {
+		String serializedAtn = getSerializedATN();
+		if (serializedAtn == null) {
+			throw new UnsupportedOperationException("The current parser does not support an ATN with bypass alternatives.");
+		}
 
-	/** Create and cache the ATN with bypass alternatives. This is not
+		synchronized (bypassAltsAtnCache) {
-	 *  part of the typical API--use compileParseTreePattern().
+			ATN result = bypassAltsAtnCache.get(serializedAtn);
-	 */
+			if (result == null) {
-	public void createATNWithBypassAlts() {
+				ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions();
-		if ( getATNWithBypassAlts()==null ) {
+				deserializationOptions.setGenerateRuleBypassTransitions(true);
-			synchronized (Parser.class) { // create just one pattern matcher
+				result = new ATNDeserializer(deserializationOptions).deserialize(serializedAtn.toCharArray());
-				if ( getATNWithBypassAlts()==null ) { // double-check
+				bypassAltsAtnCache.put(serializedAtn, result);
-					String sATN = getSerializedATN();
-					ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions();
-					deserializationOptions.setGenerateRuleBypassTransitions(true);
-					setATNWithBypassAlts( new ATNDeserializer(deserializationOptions).deserialize(sATN.toCharArray()) );
-				}
 			}
+
+			return result;
 		}
 	}
 
-	/** The preferred method of getting a tree pattern. For example,
+	/**
-	 *  here's a sample use:
+	 * The preferred method of getting a tree pattern. For example, here's a
+	 * sample use:
 	 *
-	 *  ParseTree t = parser.expr();
+	 * <pre>
-	 *  ParseTreePattern p = parser.compileParseTreePattern("<ID>+0", MyParser.RULE_expr);
+	 * ParseTree t = parser.expr();
-	 *  ParseTreeMatch m = p.match(t);
+	 * ParseTreePattern p = parser.compileParseTreePattern("<ID>+0", MyParser.RULE_expr);
-	 *  String id = m.get("ID");
+	 * ParseTreeMatch m = p.match(t);
+	 * String id = m.get("ID");
+	 * </pre>
 	 */
 	public ParseTreePattern compileParseTreePattern(String pattern, int patternRuleIndex) {
 		if ( getTokenStream()!=null ) {
@@ -477,13 +495,13 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
 		throw new UnsupportedOperationException("Parser can't discover a lexer to use");
 	}
 
-	/** The same as compileParseTreePattern(pattern,patternRuleName) but
+	/**
-	 *  specify a lexer rather than trying to deduce it from this parser.
+	 * The same as {@link #compileParseTreePattern(String, int)} but specify a
+	 * {@link Lexer} rather than trying to deduce it from this parser.
 	 */
 	public ParseTreePattern compileParseTreePattern(String pattern, int patternRuleIndex,
 													Lexer lexer)
 	{
-		createATNWithBypassAlts();
 		ParseTreePatternMatcher m = new ParseTreePatternMatcher(lexer, this);
 		return m.compile(pattern, patternRuleIndex);
 	}
@@ -630,8 +648,8 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
 	}
 
 	/**
-	 * @deprecated Use {@link #enterRecursionRule(ParserRuleContext, int, int)}
+	 * @deprecated Use
-	 * instead.
+	 * {@link #enterRecursionRule(ParserRuleContext, int, int, int)} instead.
 	 */
 	@Deprecated
 	public void enterRecursionRule(ParserRuleContext localctx, int ruleIndex) {
@@ -783,7 +801,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
    		return atn.nextTokens(s);
    	}
 
-	/** Get a rule's index (i.e., RULE_ruleName field) or -1 if not found. */
+	/** Get a rule's index (i.e., {@code RULE_ruleName} field) or -1 if not found. */
 	public int getRuleIndex(String ruleName) {
 		Integer ruleIndex = getRuleIndexMap().get(ruleName);
 		if ( ruleIndex!=null ) return ruleIndex;

runtime/Java/src/org/antlr/v4/runtime/Recognizer.java

@@ -34,14 +34,22 @@ import org.antlr.v4.runtime.atn.ATN;
 import org.antlr.v4.runtime.atn.ATNSimulator;
 import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.misc.Nullable;
+import org.antlr.v4.runtime.misc.Utils;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.WeakHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 
 public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
 	public static final int EOF=-1;
 
+	private static final Map<String[], Map<String, Integer>> tokenTypeMapCache =
+		new WeakHashMap<String[], Map<String, Integer>>();
+	private static final Map<String[], Map<String, Integer>> ruleIndexMapCache =
+		new WeakHashMap<String[], Map<String, Integer>>();
+
 	@NotNull
 	private List<ANTLRErrorListener> _listeners =
 		new CopyOnWriteArrayList<ANTLRErrorListener>() {{
@@ -60,14 +68,52 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
 
 	public abstract String[] getRuleNames();
 
-	/** Used for xpath, tree pattern compilation */
+	/**
+	 * Get a map from token names to token types.
+	 * <p/>
+	 * Used for XPath and tree pattern compilation.
+	 */
+	@NotNull
 	public Map<String, Integer> getTokenTypeMap() {
-		throw new UnsupportedOperationException("recognizer implementation must implement this");
+		String[] tokenNames = getTokenNames();
+		if (tokenNames == null) {
+			throw new UnsupportedOperationException("The current recognizer does not provide a list of token names.");
+		}
+
+		synchronized (tokenTypeMapCache) {
+			Map<String, Integer> result = tokenTypeMapCache.get(tokenNames);
+			if (result == null) {
+				result = Utils.toMap(tokenNames);
+				result.put("EOF", Token.EOF);
+				result = Collections.unmodifiableMap(result);
+				tokenTypeMapCache.put(tokenNames, result);
+			}
+
+			return result;
+		}
 	}
 
-	/** Used for xpath, tree pattern compilation */
+	/**
+	 * Get a map from rule names to rule indexes.
+	 * <p/>
+	 * Used for XPath and tree pattern compilation.
+	 */
+	@NotNull
 	public Map<String, Integer> getRuleIndexMap() {
-		throw new UnsupportedOperationException("recognizer implementation must implement this");
+		String[] ruleNames = getRuleNames();
+		if (ruleNames == null) {
+			throw new UnsupportedOperationException("The current recognizer does not provide a list of rule names.");
+		}
+
+		synchronized (ruleIndexMapCache) {
+			Map<String, Integer> result = ruleIndexMapCache.get(ruleNames);
+			if (result == null) {
+				result = Collections.unmodifiableMap(Utils.toMap(ruleNames));
+				ruleIndexMapCache.put(ruleNames, result);
+			}
+
+			return result;
+		}
 	}
 
 	public int getTokenType(String tokenName) {
@@ -76,12 +122,14 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
 		return Token.INVALID_TYPE;
 	}
 
-	/** If this recognizer was generated, it will have a serialized ATN
+	/**
-	 *  representation of the grammar.
+	 * If this recognizer was generated, it will have a serialized ATN
-	 *
+	 * representation of the grammar.
-	 *  For interpreters, we don't know their serialized ATN despite having
+	 * <p/>
-	 *  created the interpreter from it.
+	 * For interpreters, we don't know their serialized ATN despite having
+	 * created the interpreter from it.
 	 */
+	@NotNull
 	public String getSerializedATN() {
 		throw new UnsupportedOperationException("there is no serialized ATN");
 	}
@@ -91,18 +139,37 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
 	 */
 	public abstract String getGrammarFileName();
 
+	/**
+	 * Get the {@link ATN} used by the recognizer for prediction.
+	 *
+	 * @return The {@link ATN} used by the recognizer for prediction.
+	 */
+	@NotNull
 	public abstract ATN getATN();
 
+	/**
+	 * Get the ATN interpreter used by the recognizer for prediction.
+	 *
+	 * @return The ATN interpreter used by the recognizer for prediction.
+	 */
+	@NotNull
 	public ATNInterpreter getInterpreter() {
 		return _interp;
 	}
 
-	public void setInterpreter(ATNInterpreter interpreter) {
+	/**
+	 * Set the ATN interpreter used by the recognizer for prediction.
+	 *
+	 * @param interpreter The ATN interpreter used by the recognizer for
+	 * prediction.
+	 */
+	public void setInterpreter(@NotNull ATNInterpreter interpreter) {
 		_interp = interpreter;
 	}
 
 	/** What is the error header, normally line/character position information? */
-	public String getErrorHeader(RecognitionException e) {
+	@NotNull
+	public String getErrorHeader(@NotNull RecognitionException e) {
 		int line = e.getOffendingToken().getLine();
 		int charPositionInLine = e.getOffendingToken().getCharPositionInLine();
 		return "line "+line+":"+charPositionInLine;
@@ -134,7 +201,7 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
 	}
 
 	/**
-	 * @throws NullPointerException if {@code listener} is {@code null}.
+	 * @exception NullPointerException if {@code listener} is {@code null}.
 	 */
 	public void addErrorListener(@NotNull ANTLRErrorListener listener) {
 		if (listener == null) {

runtime/Java/src/org/antlr/v4/runtime/Token.java

@@ -57,7 +57,9 @@ public interface Token {
 	 */
 	public static final int HIDDEN_CHANNEL = 1;
 
-	/** Get the text of the token */
+	/**
+	 * Get the text of the token.
+	 */
 	String getText();
 
 	/** Get the token type of the token */

runtime/Java/src/org/antlr/v4/runtime/TokenSource.java

@@ -30,46 +30,83 @@
 package org.antlr.v4.runtime;
 
 import org.antlr.v4.runtime.misc.NotNull;
+import org.antlr.v4.runtime.misc.Nullable;
 
-/** A source of tokens must provide a sequence of tokens via nextToken()
+/**
- *  and also must reveal it's source of characters; CommonToken's text is
+ * A source of tokens must provide a sequence of tokens via {@link #nextToken()}
- *  computed from a CharStream; it only store indices into the char stream.
+ * and also must reveal it's source of characters; {@link CommonToken}'s text is
- *
+ * computed from a {@link CharStream}; it only store indices into the char
- *  Errors from the lexer are never passed to the parser.  Either you want
+ * stream.
- *  to keep going or you do not upon token recognition error.  If you do not
+ * <p/>
- *  want to continue lexing then you do not want to continue parsing.  Just
+ * Errors from the lexer are never passed to the parser. Either you want to keep
- *  throw an exception not under RecognitionException and Java will naturally
+ * going or you do not upon token recognition error. If you do not want to
- *  toss you all the way out of the recognizers.  If you want to continue
+ * continue lexing then you do not want to continue parsing. Just throw an
- *  lexing then you should not throw an exception to the parser--it has already
+ * exception not under {@link RecognitionException} and Java will naturally toss
- *  requested a token.  Keep lexing until you get a valid one.  Just report
+ * you all the way out of the recognizers. If you want to continue lexing then
- *  errors and keep going, looking for a valid token.
+ * you should not throw an exception to the parser--it has already requested a
+ * token. Keep lexing until you get a valid one. Just report errors and keep
+ * going, looking for a valid token.
  */
 public interface TokenSource {
-	/** Return a Token object from your input stream (usually a CharStream).
+	/**
-	 *  Do not fail/return upon lexing error; keep chewing on the characters
+	 * Return a {@link Token} object from your input stream (usually a
-	 *  until you get a good one; errors are not passed through to the parser.
+	 * {@link CharStream}). Do not fail/return upon lexing error; keep chewing
+	 * on the characters until you get a good one; errors are not passed through
+	 * to the parser.
 	 */
+	@NotNull
 	public Token nextToken();
 
+	/**
+	 * Get the line number for the current position in the input stream. The
+	 * first line in the input is line 1.
+	 *
+	 * @return The line number for the current position in the input stream, or
+	 * 0 if the current token source does not track line numbers.
+	 */
 	public int getLine();
 
+	/**
+	 * Get the index into the current line for the current position in the input
+	 * stream. The first character on a line has position 0.
+	 *
+	 * @return The line number for the current position in the input stream, or
+	 * -1 if the current token source does not track character positions.
+	 */
 	public int getCharPositionInLine();
 
-	/** From what character stream was this token created?  You don't have to
+	/**
-	 *  implement but it's nice to know where a Token comes from if you have
+	 * Get the {@link CharStream} from which this token source is currently
-	 *  include files etc... on the input.
+	 * providing tokens.
+	 *
+	 * @return The {@link CharStream} associated with the current position in
+	 * the input, or {@code null} if no input stream is available for the token
+	 * source.
 	 */
+	@Nullable
 	public CharStream getInputStream();
 
-	/** Where are you getting tokens from? normally the implication will simply
+	/**
-	 *  ask lexers input stream.
+	 * Gets the name of the underlying input source. This method returns a
+	 * non-null, non-empty string. If such a name is not known, this method
+	 * returns {@link IntStream#UNKNOWN_SOURCE_NAME}.
 	 */
 	public String getSourceName();
 
-	/** Optional method that lets users set factory in lexer or other source */
+	/**
+	 * Set the {@link TokenFactory} this token source should use for creating
+	 * {@link Token} objects from the input.
+	 *
+	 * @param factory The {@link TokenFactory} to use for creating tokens.
+	 */
 	public void setTokenFactory(@NotNull TokenFactory<?> factory);
 
-	/** Gets the factory used for constructing tokens. */
+	/**
+	 * Gets the {@link TokenFactory} this token source is currently using for
+	 * creating {@link Token} objects from the input.
+	 *
+	 * @return The {@link TokenFactory} currently used by this token source.
+	 */
 	@NotNull
 	public TokenFactory<?> getTokenFactory();
 }

runtime/Java/src/org/antlr/v4/runtime/tree/pattern/Chunk.java

@@ -30,12 +30,16 @@
 
 package org.antlr.v4.runtime.tree.pattern;
 
-/** A chunk is either a token reference, a rule reference, or some plaintext
+/**
- *  within a tree pattern.  Function split() in the pattern matcher returns
+ * A chunk is either a token tag, a rule tag, or a span of literal text within a
- *  a list of chunks in preparation for creating a token stream by tokenize().
+ * tree pattern.
- *  From there, we get a parse tree from with compile(). These chunks are
+ * <p/>
- *  converted to RuleTagToken, TokenTagToken, or the regular tokens
+ * The method {@link ParseTreePatternMatcher#split(String)} returns a list of
- *  of the text surrounding the tags.
+ * chunks in preparation for creating a token stream by
+ * {@link ParseTreePatternMatcher#tokenize(String)}. From there, we get a parse
+ * tree from with {@link ParseTreePatternMatcher#compile(String, int)}. These
+ * chunks are converted to {@link RuleTagToken}, {@link TokenTagToken}, or the
+ * regular tokens of the text surrounding the tags.
  */
 abstract class Chunk {
 }

runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreeMatch.java

@@ -31,103 +31,194 @@
 package org.antlr.v4.runtime.tree.pattern;
 
 import org.antlr.v4.runtime.misc.MultiMap;
+import org.antlr.v4.runtime.misc.NotNull;
+import org.antlr.v4.runtime.misc.Nullable;
 import org.antlr.v4.runtime.tree.ParseTree;
 
 import java.util.Collections;
 import java.util.List;
 
-/** The result of matching a tree against a tree pattern.
+/**
- *  If there is no match, field mismatchedNode is set to the offending
+ * Represents the result of matching a {@link ParseTree} against a tree pattern.
- *  treenode.
  */
 public class ParseTreeMatch {
-	/** Tree we tried to match */
+	/**
-	protected ParseTree tree;
+	 * This is the backing field for {@link #getTree()}.
+	 */
+	private final ParseTree tree;
 
-	/** To what pattern? */
+	/**
-	protected ParseTreePattern pattern;
+	 * This is the backing field for {@link #getPattern()}.
+	 */
+	private final ParseTreePattern pattern;
 
-	/** Map a label or token/rule name to List of nodes in tree we match */
+	/**
-	protected MultiMap<String, ParseTree> labels;
+	 * This is the backing field for {@link #getLabels()}.
+	 */
+	private final MultiMap<String, ParseTree> labels;
 
-	/** Where in actual parse tree we failed if we can't match pattern */
+	/**
-	protected ParseTree mismatchedNode;
+	 * This is the backing field for {@link #getMismatchedNode()}.
+	 */
+	private final ParseTree mismatchedNode;
+
+	/**
+	 * Constructs a new instance of {@link ParseTreeMatch} from the specified
+	 * parse tree and pattern.
+	 *
+	 * @param tree The parse tree to match against the pattern.
+	 * @param pattern The parse tree pattern.
+	 * @param labels A mapping from label names to collections of
+	 * {@link ParseTree} objects located by the tree pattern matching process.
+	 * @param mismatchedNode The first node which failed to match the tree
+	 * pattern during the matching process.
+	 *
+	 * @exception IllegalArgumentException if {@code tree} is {@code null}
+	 * @exception IllegalArgumentException if {@code pattern} is {@code null}
+	 * @exception IllegalArgumentException if {@code labels} is {@code null}
+	 */
+	public ParseTreeMatch(@NotNull ParseTree tree, @NotNull ParseTreePattern pattern, @NotNull MultiMap<String, ParseTree> labels, @Nullable ParseTree mismatchedNode) {
+		if (tree == null) {
+			throw new IllegalArgumentException("tree cannot be null");
+		}
+
+		if (pattern == null) {
+			throw new IllegalArgumentException("pattern cannot be null");
+		}
+
+		if (labels == null) {
+			throw new IllegalArgumentException("labels cannot be null");
+		}
 
-	public ParseTreeMatch(ParseTree tree, ParseTreePattern pattern) {
 		this.tree = tree;
 		this.pattern = pattern;
-		this.labels = new MultiMap<String, ParseTree>();
+		this.labels = labels;
+		this.mismatchedNode = mismatchedNode;
 	}
 
-	/** Get the node associated with label. E.g., for  pattern <id:ID>,
+	/**
-	 *  get("id") returns the node matched for that ID. If there are more than
+	 * Get the first node associated with a specific {@code label}.
-	 *  one nodes matched with that label, this returns the first one matched.
+	 * <p/>
-	 *  If there is no node associated with the label, this returns null.
+	 * For example, for pattern {@code <id:ID>}, {@code get("id")} returns the
+	 * node matched for that {@code ID}. If more than one node
+	 * matched the specified label, only the first is returned. If there is
+	 * no node associated with the label, this returns {@code null}.
+	 * <p/>
+	 * Pattern tags like {@code <ID>} and {@code <expr>} without labels are
+	 * considered to be labeled with {@code ID} and {@code expr}, respectively.
 	 *
-	 *  Pattern tags like <ID> and <expr> without labels are considered to be
+	 * @param label The label to check.
-	 *  labeled with ID and expr, respectively.
+	 *
+	 * @return The first {@link ParseTree} to match a tag with the specified
+	 * label, or {@code null} if no parse tree matched a tag with the label.
 	 */
+	@Nullable
 	public ParseTree get(String label) {
 		List<ParseTree> parseTrees = labels.get(label);
-		if ( parseTrees==null ) return null;
+		if ( parseTrees==null ) {
+			return null;
+		}
+
 		return parseTrees.get(0); // return first if multiple
 	}
 
-	/** Return all nodes matched that are labeled with parameter label.
+	/**
-	 *  If there is only one such node, return a list with one element.
+	 * Return all nodes matching a rule or token tag with the specified label.
-	 *  If there are no nodes matched with that label, return an empty list.
+	 * <p/>
+	 * If the {@code label} is the name of a parser rule or token in the
+	 * grammar, the resulting list will contain both the parse trees matching
+	 * rule or tags explicitly labeled with the label and the complete set of
+	 * parse trees matching the labeled and unlabeled tags in the pattern for
+	 * the parser rule or token. For example, if {@code label} is {@code "foo"},
+	 * the result will contain <em>all</em> of the following.
 	 *
-	 *  Pattern tags like <ID> and <expr> without labels are considered to be
+	 * <ul>
-	 *  labeled with ID and expr, respectively.
+	 * <li>Parse tree nodes matching tags of the form {@code <foo:anyRuleName>} and
+	 * {@code <foo:AnyTokenName>}.</li>
+	 * <li>Parse tree nodes matching tags of the form {@code <anyLabel:foo>}.</li>
+	 * <li>Parse tree nodes matching tags of the form {@code <foo>}.</li>
+	 * </ul>
+	 *
+	 * @param label The label.
+	 *
+	 * @return A collection of all {@link ParseTree} nodes matching tags with
+	 * the specified {@code label}. If no nodes matched the label, an empty list
+	 * is returned.
 	 */
-	public List<ParseTree> getAll(String label) {
+	@NotNull
+	public List<ParseTree> getAll(@NotNull String label) {
 		List<ParseTree> nodes = labels.get(label);
-		if ( nodes==null ) return Collections.emptyList();
+		if ( nodes==null ) {
+			return Collections.emptyList();
+		}
+
 		return nodes;
 	}
 
-	/** Return a mapping from label->[list of nodes]. This includes
+	/**
-	 *  token and rule references such as <ID> and <expr>
+	 * Return a mapping from label &rarr; [list of nodes].
+	 * <p/>
+	 * The map includes special entries corresponding to the names of rules and
+	 * tokens referenced in tags in the original pattern. For additional
+	 * information, see the description of {@link #getAll(String)}.
+	 *
+	 * @return A mapping from labels to parse tree nodes. If the parse tree
+	 * pattern did not contain any rule or token tags, this map will be empty.
 	 */
+	@NotNull
 	public MultiMap<String, ParseTree> getLabels() {
 		return labels;
 	}
 
-	/** Return the node at which we first detected a mismatch. Return
+	/**
-	 *  null if we have not found an error.
+	 * Get the node at which we first detected a mismatch.
+	 *
+	 * @return the node at which we first detected a mismatch, or {@code null}
+	 * if the match was successful.
 	 */
+	@Nullable
 	public ParseTree getMismatchedNode() {
 		return mismatchedNode;
 	}
 
-	/** Return the text of the entire subtree matched. It does not include
+	/**
-	 *  whitespace stripped by the lexer.
+	 * Gets a value indicating whether the match operation succeeded.
+	 *
+	 * @return {@code true} if the match operation succeeded; otherwise,
+	 * {@code false}.
 	 */
-	public String getText() {
-		return tree.getText();
-	}
-
-	/** Did the tree vs pattern fail to match? */
-	public boolean failed() {
-		return mismatchedNode!=null;
-	}
-
-	/** Did the tree vs pattern match? */
 	public boolean succeeded() {
-		return mismatchedNode==null;
+		return mismatchedNode == null;
 	}
 
-	/** Return the tree pattern we are matching against */
+	/**
+	 * Get the tree pattern we are matching against.
+	 *
+	 * @return The tree pattern we are matching against.
+	 */
+	@NotNull
 	public ParseTreePattern getPattern() {
 		return pattern;
 	}
 
-	/** Return the parse tree we are trying to match to a pattern */
+	/**
+	 * Get the parse tree we are trying to match to a pattern.
+	 *
+	 * @return The {@link ParseTree} we are trying to match to a pattern.
+	 */
+	@NotNull
 	public ParseTree getTree() {
 		return tree;
 	}
 
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
 	public String toString() {
-		return tree.getText();
+		return String.format(
+			"Match %s; found %d labels",
+			succeeded() ? "succeeded" : "failed",
+			getLabels().size());
 	}
 }

runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePattern.java

@@ -30,6 +30,7 @@
 
 package org.antlr.v4.runtime.tree.pattern;
 
+import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.xpath.XPath;
 
@@ -37,17 +38,46 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 
-/** A pattern like "<ID> = <expr>;" converted to a ParseTree by
+/**
- *  ParseTreePatternMatcher.compile().
+ * A pattern like {@code <ID> = <expr>;} converted to a {@link ParseTree} by
+ * {@link ParseTreePatternMatcher#compile(String, int)}.
  */
 public class ParseTreePattern {
-	protected int patternRuleIndex;
+	/**
-	protected String pattern;
+	 * This is the backing field for {@link #getPatternRuleIndex()}.
-	protected ParseTree patternTree;
+	 */
-	public ParseTreePatternMatcher matcher;
+	private final int patternRuleIndex;
 
-	public ParseTreePattern(ParseTreePatternMatcher matcher,
+	/**
-							String pattern, int patternRuleIndex, ParseTree patternTree)
+	 * This is the backing field for {@link #getPattern()}.
+	 */
+	@NotNull
+	private final String pattern;
+
+	/**
+	 * This is the backing field for {@link #getPatternTree()}.
+	 */
+	@NotNull
+	private final ParseTree patternTree;
+
+	/**
+	 * This is the backing field for {@link #getMatcher()}.
+	 */
+	@NotNull
+	private final ParseTreePatternMatcher matcher;
+
+	/**
+	 * Construct a new instance of the {@link ParseTreePattern} class.
+	 *
+	 * @param matcher The {@link ParseTreePatternMatcher} which created this
+	 * tree pattern.
+	 * @param pattern The tree pattern in concrete syntax form.
+	 * @param patternRuleIndex The parser rule which serves as the root of the
+	 * tree pattern.
+	 * @param patternTree The tree pattern in {@link ParseTree} form.
+	 */
+	public ParseTreePattern(@NotNull ParseTreePatternMatcher matcher,
+							@NotNull String pattern, int patternRuleIndex, @NotNull ParseTree patternTree)
 	{
 		this.matcher = matcher;
 		this.patternRuleIndex = patternRuleIndex;
@@ -55,18 +85,43 @@ public class ParseTreePattern {
 		this.patternTree = patternTree;
 	}
 
-	public ParseTreeMatch match(ParseTree tree) {
+	/**
+	 * Match a specific parse tree against this tree pattern.
+	 *
+	 * @param tree The parse tree to match against this tree pattern.
+	 * @return A {@link ParseTreeMatch} object describing the result of the
+	 * match operation. The {@link ParseTreeMatch#succeeded()} method can be
+	 * used to determine whether or not the match was successful.
+	 */
+	@NotNull
+	public ParseTreeMatch match(@NotNull ParseTree tree) {
 		return matcher.match(tree, this);
 	}
 
-	public boolean matches(ParseTree tree) {
+	/**
+	 * Determine whether or not a parse tree matches this tree pattern.
+	 *
+	 * @param tree The parse tree to match against this tree pattern.
+	 * @return {@code true} if {@code tree} is a match for the current tree
+	 * pattern; otherwise, {@code false}.
+	 */
+	public boolean matches(@NotNull ParseTree tree) {
 		return matcher.match(tree, this).succeeded();
 	}
 
-	/** Find all nodes using xpath and then try to match those subtrees
+	/**
-	 *  against this tree pattern
+	 * Find all nodes using XPath and then try to match those subtrees against
+	 * this tree pattern.
+	 *
+	 * @param tree The {@link ParseTree} to match against this pattern.
+	 * @param xpath An expression matching the nodes
+	 *
+	 * @return A collection of {@link ParseTreeMatch} objects describing the
+	 * successful matches. Unsuccessful matches are omitted from the result,
+	 * regardless of the reason for the failure.
 	 */
-	public List<ParseTreeMatch> findAll(ParseTree tree, String xpath) {
+	@NotNull
+	public List<ParseTreeMatch> findAll(@NotNull ParseTree tree, @NotNull String xpath) {
 		Collection<ParseTree> subtrees = XPath.findAll(tree, xpath, matcher.getParser());
 		List<ParseTreeMatch> matches = new ArrayList<ParseTreeMatch>();
 		for (ParseTree t : subtrees) {
@@ -78,18 +133,46 @@ public class ParseTreePattern {
 		return matches;
 	}
 
-	public ParseTreePatternMatcher getParseTreePattern() {
+	/**
+	 * Get the {@link ParseTreePatternMatcher} which created this tree pattern.
+	 *
+	 * @return The {@link ParseTreePatternMatcher} which created this tree
+	 * pattern.
+	 */
+	@NotNull
+	public ParseTreePatternMatcher getMatcher() {
 		return matcher;
 	}
 
+	/**
+	 * Get the tree pattern in concrete syntax form.
+	 *
+	 * @return The tree pattern in concrete syntax form.
+	 */
+	@NotNull
 	public String getPattern() {
 		return pattern;
 	}
 
+	/**
+	 * Get the parser rule which serves as the outermost rule for the tree
+	 * pattern.
+	 *
+	 * @return The parser rule which serves as the outermost rule for the tree
+	 * pattern.
+	 */
 	public int getPatternRuleIndex() {
 		return patternRuleIndex;
 	}
 
+	/**
+	 * Get the tree pattern as a {@link ParseTree}. The rule and token tags from
+	 * the pattern are present in the parse tree as terminal nodes with a symbol
+	 * of type {@link RuleTagToken} or {@link TokenTagToken}.
+	 *
+	 * @return The tree pattern as a {@link ParseTree}.
+	 */
+	@NotNull
 	public ParseTree getPatternTree() {
 		return patternTree;
 	}

runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java

@@ -38,69 +38,74 @@ import org.antlr.v4.runtime.Parser;
 import org.antlr.v4.runtime.ParserInterpreter;
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.atn.ATN;
+import org.antlr.v4.runtime.misc.MultiMap;
+import org.antlr.v4.runtime.misc.NotNull;
+import org.antlr.v4.runtime.misc.Nullable;
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.RuleNode;
 import org.antlr.v4.runtime.tree.TerminalNode;
 
-import java.io.IOException;
-import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-/** A tree pattern matching mechanism for ANTLR ParseTrees.
+/**
- *
+ * A tree pattern matching mechanism for ANTLR {@link ParseTree}s.
- *  Patterns are strings of source input text with special tags
+ * <p/>
- *  representing token or rule references such as:
+ * Patterns are strings of source input text with special tags representing
- *
+ * token or rule references such as:
- *  "<ID> = <expr>;"
+ * <p/>
- *
+ * {@code <ID> = <expr>;}
- *  Given a pattern start rule such as statement, this object
+ * <p/>
- *  construct a parse tree with placeholders for the identifier and
+ * Given a pattern start rule such as {@code statement}, this object constructs
- *  expression subtree. Then the match() routines can compare an
+ * a {@link ParseTree} with placeholders for the {@code ID} and {@code expr}
- *  actual parse tree from a parse with this pattern. Tag <ID> matches
+ * subtree. Then the {@link #match} routines can compare an actual
- *  any ID token and tag <expr> references any expression subtree.
+ * {@link ParseTree} from a parse with this pattern. Tag {@code <ID>} matches
- *
+ * any {@code ID} token and tag {@code <expr>} references the result of the
- *  Pattern "x = 0;" is a similar pattern that matches the same
+ * {@code expr} rule (generally an instance of {@code ExprContext}.
- *  pattern except that it requires the identifier to be x and the
+ * <p/>
- *  expression to be 0.
+ * Pattern {@code x = 0;} is a similar pattern that matches the same pattern
- *
+ * except that it requires the identifier to be {@code x} and the expression to
- *  The matches() routines return true or false based upon a match for
+ * be {@code 0}.
- *  the tree rooted at the parameter sent in. The match() routines
+ * <p/>
- *  return a ParseTreeMatch object that contains the parse tree, the
+ * The {@link #matches} routines return {@code true} or {@code false} based
- *  parse tree pattern, and a map from tag name to matched nodes (more
+ * upon a match for the tree rooted at the parameter sent in. The
- *  below). A subtree that fails to match, returns with
+ * {@link #match} routines return a {@link ParseTreeMatch} object that
- *  ParseTreeMatch.mismatchedNode set to the first tree node that did
+ * contains the parse tree, the parse tree pattern, and a map from tag name to
- *  not match.
+ * matched nodes (more below). A subtree that fails to match, returns with
- *
+ * {@link ParseTreeMatch#mismatchedNode} set to the first tree node that did not
- *  For efficiency, you can compile a tree pattern in string form to a
+ * match.
- *  ParseTreePattern object.
+ * <p/>
- *
+ * For efficiency, you can compile a tree pattern in string form to a
- *  See TestParseTreeMatcher for lots of examples. ParseTreePattern
+ * {@link ParseTreePattern} object.
- *  has two static helper methods: findAll() and match() that are easy
+ * <p/>
- *  to use but not superefficient because they create new
+ * See {@code TestParseTreeMatcher} for lots of examples.
- *  ParseTreePatternMatcher objects each time and have to compile the
+ * {@link ParseTreePattern} has two static helper methods:
- *  pattern in string form before using it.
+ * {@link ParseTreePattern#findAll} and {@link ParseTreePattern#match} that
- *
+ * are easy to use but not super efficient because they create new
- *  The lexer and parser that you pass into the
+ * {@link ParseTreePatternMatcher} objects each time and have to compile the
- *  ParseTreePatternMatcher constructor are used to parse the pattern
+ * pattern in string form before using it.
- *  in string form. The lexer converts the "<ID> = <expr>;" into a
+ * <p/>
- *  sequence of four tokens (assuming lexer throws out whitespace or
+ * The lexer and parser that you pass into the {@link ParseTreePatternMatcher}
- *  puts it on a hidden channel). Be aware that the input stream is
+ * constructor are used to parse the pattern in string form. The lexer converts
- *  reset for the lexer (but not the parser; a ParserInterpreter is
+ * the {@code <ID> = <expr>;} into a sequence of four tokens (assuming lexer
- *  created to parse the input.). Any user-defined fields you have put
+ * throws out whitespace or puts it on a hidden channel). Be aware that the
- *  into the lexer might get changed when this mechanism asks it to
+ * input stream is reset for the lexer (but not the parser; a
- *  scan the pattern string.
+ * {@link ParserInterpreter} is created to parse the input.). Any user-defined
- *
+ * fields you have put into the lexer might get changed when this mechanism asks
- *  Normally a parser does not accept token "<expr>" as a valid expr
+ * it to scan the pattern string.
- *  but, from the parser passed in, we create a special version of the
+ * <p/>
- *  underlying grammar representation (an ATN) that allows imaginary
+ * Normally a parser does not accept token {@code <expr>} as a valid
- *  tokens representing rules (<expr>) to match entire rules.
+ * {@code expr} but, from the parser passed in, we create a special version of
- *  We call these bypass alternatives.
+ * the underlying grammar representation (an {@link ATN}) that allows imaginary
- *
+ * tokens representing rules ({@code <expr>}) to match entire rules. We call
- *  Delimiters are < and > with \ as the escape string by default, but
+ * these <em>bypass alternatives</em>.
- *  you can set them to whatever you want using setDelimiters(). You
+ * <p/>
- *  must escape both start and stop strings \< and \>.
+ * Delimiters are {@code <} and {@code >}, with {@code \} as the escape string
+ * by default, but you can set them to whatever you want using
+ * {@link #setDelimiters}. You must escape both start and stop strings
+ * {@code \<} and {@code \>}.
  */
 public class ParseTreePatternMatcher {
 	public static class CannotInvokeStartRule extends RuntimeException {
@@ -109,77 +114,97 @@ public class ParseTreePatternMatcher {
 		}
 	}
 
-	/** Used to convert the tree pattern string into a series of tokens.
+	/**
-	 *  The input stream is reset.
+	 * This is the backing field for {@link #getLexer()}.
 	 */
-	protected Lexer lexer;
+	private final Lexer lexer;
 
-	/** Used to collect to the grammar file name, token names, rule names
+	/**
-	 *  for used to parse the pattern into a parse tree.
+	 * This is the backing field for {@link #getParser()}.
 	 */
-	protected Parser parser;
+	private final Parser parser;
 
-	protected String start = "<", stop=">";
+	protected String start = "<";
+	protected String stop = ">";
 	protected String escape = "\\"; // e.g., \< and \> must escape BOTH!
 
-	public ParseTreePatternMatcher() { } // used for testing only
+	/**
-
+	 * Constructs a {@link ParseTreePatternMatcher} or from a {@link Lexer} and
-	/** Constructs a pattern match or from a lecture and parser object.
+	 * {@link Parser} object. The lexer input stream is altered for tokenizing
-	 *  The lexer input stream is altered for tokenizing the tree patterns.
+	 * the tree patterns. The parser is used as a convenient mechanism to get
-	 *  The parser is used as a convenient mechanism to get the grammar name,
+	 * the grammar name, plus token, rule names.
-	 *  plus token, rule names.
 	 */
 	public ParseTreePatternMatcher(Lexer lexer, Parser parser) {
 		this.lexer = lexer;
 		this.parser = parser;
-		if ( parser!=null ) {
-			parser.createATNWithBypassAlts();
-		}
 	}
 
+	/**
+	 * Set the delimiters used for marking rule and token tags within concrete
+	 * syntax used by the tree pattern parser.
+	 *
+	 * @param start The start delimiter.
+	 * @param stop The stop delimiter.
+	 * @param escapeLeft The escape sequence to use for escaping a start or stop delimiter.
+	 *
+	 * @exception IllegalArgumentException if {@code start} is {@code null} or empty.
+	 * @exception IllegalArgumentException if {@code stop} is {@code null} or empty.
+	 */
 	public void setDelimiters(String start, String stop, String escapeLeft) {
+		if (start == null || start.isEmpty()) {
+			throw new IllegalArgumentException("start cannot be null or empty");
+		}
+
+		if (stop == null || stop.isEmpty()) {
+			throw new IllegalArgumentException("stop cannot be null or empty");
+		}
+
 		this.start = start;
 		this.stop = stop;
 		this.escape = escapeLeft;
 	}
 
-	/** Does pattern matched as rule patternRuleIndex match tree? */
+	/** Does {@code pattern} matched as rule {@code patternRuleIndex} match {@code tree}? */
 	public boolean matches(ParseTree tree, String pattern, int patternRuleIndex) {
 		ParseTreePattern p = compile(pattern, patternRuleIndex);
 		return matches(tree, p);
 	}
 
-	/** Does pattern matched as rule patternRuleIndex match tree? Pass in a
+	/** Does {@code pattern} matched as rule patternRuleIndex match tree? Pass in a
 	 *  compiled pattern instead of a string representation of a tree pattern.
 	 */
 	public boolean matches(ParseTree tree, ParseTreePattern pattern) {
-		ParseTreeMatch match = new ParseTreeMatch(tree, pattern);
+		MultiMap<String, ParseTree> labels = new MultiMap<String, ParseTree>();
-		matches_(tree, pattern.patternTree, match);
+		ParseTree mismatchedNode = matchImpl(tree, pattern.getPatternTree(), labels);
-		return match.succeeded();
+		return mismatchedNode == null;
 	}
 
-	/** Compare pattern matched as rule patternRuleIndex against tree and
+	/**
-	 *  return a ParseTreeMatch object that contains the matched elements,
+	 * Compare {@code pattern} matched as rule {@code patternRuleIndex} against
-	 *  or the node at which the match failed.
+	 * {@code tree} and return a {@link ParseTreeMatch} object that contains the
+	 * matched elements, or the node at which the match failed.
 	 */
 	public ParseTreeMatch match(ParseTree tree, String pattern, int patternRuleIndex) {
 		ParseTreePattern p = compile(pattern, patternRuleIndex);
 		return match(tree, p);
 	}
 
-	/** Compare pattern matched against tree and
+	/**
-	 *  return a ParseTreeMatch object that contains the matched elements,
+	 * Compare {@code pattern} matched against {@code tree} and return a
-	 *  or the node at which the match failed. Pass in a compiled pattern
+	 * {@link ParseTreeMatch} object that contains the matched elements, or the
-	 *  instead of a string representation of a tree pattern.
+	 * node at which the match failed. Pass in a compiled pattern instead of a
+	 * string representation of a tree pattern.
 	 */
-	public ParseTreeMatch match(ParseTree tree, ParseTreePattern pattern) {
+	@NotNull
-		ParseTreeMatch match = new ParseTreeMatch(tree, pattern);
+	public ParseTreeMatch match(@NotNull ParseTree tree, @NotNull ParseTreePattern pattern) {
-		matches_(tree, pattern.patternTree, match);
+		MultiMap<String, ParseTree> labels = new MultiMap<String, ParseTree>();
-		return match;
+		ParseTree mismatchedNode = matchImpl(tree, pattern.getPatternTree(), labels);
+		return new ParseTreeMatch(tree, pattern, labels, mismatchedNode);
 	}
 
-	/** For repeated use of a tree pattern, compile it to a ParseTreePattern
+	/**
-	 *  using this method.
+	 * For repeated use of a tree pattern, compile it to a
+	 * {@link ParseTreePattern} using this method.
 	 */
 	public ParseTreePattern compile(String pattern, int patternRuleIndex) {
 		List<? extends Token> tokenList = tokenize(pattern);
@@ -204,88 +229,131 @@ public class ParseTreePatternMatcher {
 		return new ParseTreePattern(this, pattern, patternRuleIndex, tree);
 	}
 
+	/**
+	 * Used to convert the tree pattern string into a series of tokens. The
+	 * input stream is reset.
+	 */
+	@NotNull
 	public Lexer getLexer() {
 		return lexer;
 	}
 
+	/**
+	 * Used to collect to the grammar file name, token names, rule names for
+	 * used to parse the pattern into a parse tree.
+	 */
+	@NotNull
 	public Parser getParser() {
 		return parser;
 	}
 
 	// ---- SUPPORT CODE ----
 
-	/** Recursively walk tree against patternTree, filling match.labels */
+	/**
-	protected boolean matches_(ParseTree tree,
+	 * Recursively walk {@code tree} against {@code patternTree}, filling
-							   ParseTree patternTree,
+	 * {@code match.}{@link ParseTreeMatch#labels labels}.
-							   ParseTreeMatch match)
+	 *
+	 * @return the first node encountered in {@code tree} which does not match
+	 * a corresponding node in {@code patternTree}, or {@code null} if the match
+	 * was successful. The specific node returned depends on the matching
+	 * algorithm used by the implementation, and may be overridden.
+	 */
+	@Nullable
+	protected ParseTree matchImpl(@NotNull ParseTree tree,
+								  @NotNull ParseTree patternTree,
+								  @NotNull MultiMap<String, ParseTree> labels)
 	{
-		if ( tree==null || patternTree==null ) {
+		if (tree == null) {
-			return false;
+			throw new IllegalArgumentException("tree cannot be null");
 		}
+
+		if (patternTree == null) {
+			throw new IllegalArgumentException("patternTree cannot be null");
+		}
+
 		// x and <ID>, x and y, or x and x; or could be mismatched types
 		if ( tree instanceof TerminalNode && patternTree instanceof TerminalNode ) {
 			TerminalNode t1 = (TerminalNode)tree;
 			TerminalNode t2 = (TerminalNode)patternTree;
-			ParseTreeMatch m = null;
+			ParseTree mismatchedNode = null;
 			// both are tokens and they have same type
 			if ( t1.getSymbol().getType() == t2.getSymbol().getType() ) {
 				if ( t2.getSymbol() instanceof TokenTagToken ) { // x and <ID>
 					TokenTagToken tokenTagToken = (TokenTagToken)t2.getSymbol();
 					// track label->list-of-nodes for both token name and label (if any)
-					match.labels.map(tokenTagToken.tokenName, tree);
+					labels.map(tokenTagToken.getTokenName(), tree);
-					if ( tokenTagToken.label!=null ) {
+					if ( tokenTagToken.getLabel()!=null ) {
-						match.labels.map(tokenTagToken.label, tree);
+						labels.map(tokenTagToken.getLabel(), tree);
 					}
 				}
-				else if ( t1.getText().equals(t2.getText()) ) { // x and x
+				else if ( t1.getText().equals(t2.getText()) ) {
+					// x and x
 				}
-				else { // x and y
+				else {
-					match.mismatchedNode = t1;
+					// x and y
+					if (mismatchedNode == null) {
+						mismatchedNode = t1;
+					}
 				}
 			}
 			else {
-				match.mismatchedNode = t1;
+				if (mismatchedNode == null) {
+					mismatchedNode = t1;
+				}
 			}
-			return match.succeeded();
+
+			return mismatchedNode;
 		}
+
 		if ( tree instanceof ParserRuleContext && patternTree instanceof ParserRuleContext ) {
 			ParserRuleContext r1 = (ParserRuleContext)tree;
 			ParserRuleContext r2 = (ParserRuleContext)patternTree;
+			ParseTree mismatchedNode = null;
 			// (expr ...) and <expr>
 			RuleTagToken ruleTagToken = getRuleTagToken(r2);
 			if ( ruleTagToken!=null ) {
 				ParseTreeMatch m = null;
 				if ( r1.getRuleContext().getRuleIndex() == r2.getRuleContext().getRuleIndex() ) {
 					// track label->list-of-nodes for both rule name and label (if any)
-					match.labels.map(ruleTagToken.ruleName, tree);
+					labels.map(ruleTagToken.getRuleName(), tree);
-					if ( ruleTagToken.label!=null ) {
+					if ( ruleTagToken.getLabel()!=null ) {
-						match.labels.map(ruleTagToken.label, tree);
+						labels.map(ruleTagToken.getLabel(), tree);
 					}
 				}
 				else {
-					match.mismatchedNode = r1;
+					if (mismatchedNode == null) {
+						mismatchedNode = r1;
+					}
 				}
-				return match.succeeded();
+
+				return mismatchedNode;
 			}
+
 			// (expr ...) and (expr ...)
 			if ( r1.getChildCount()!=r2.getChildCount() ) {
-				match.mismatchedNode = r1;
+				if (mismatchedNode == null) {
-				return false;
+					mismatchedNode = r1;
+				}
+
+				return mismatchedNode;
 			}
+
 			int n = r1.getChildCount();
 			for (int i = 0; i<n; i++) {
-				boolean childMatch =
+				ParseTree childMatch = matchImpl(r1.getChild(i), patternTree.getChild(i), labels);
-					matches_(r1.getChild(i), patternTree.getChild(i), match);
+				if ( childMatch != null ) {
-				if ( !childMatch ) return false;
+					return childMatch;
+				}
 			}
-			return true;
+
+			return mismatchedNode;
 		}
+
 		// if nodes aren't both tokens or both rule nodes, can't match
-		match.mismatchedNode = tree;
+		return tree;
-		return false;
 	}
 
-	/** Is t (expr <expr>) subtree? */
+	/** Is {@code t} {@code (expr <expr>)} subtree? */
 	protected RuleTagToken getRuleTagToken(ParseTree t) {
 		if ( t instanceof RuleNode ) {
 			RuleNode r = (RuleNode)t;
@@ -310,40 +378,34 @@ public class ParseTreePatternMatcher {
 			if ( chunk instanceof TagChunk ) {
 				TagChunk tagChunk = (TagChunk)chunk;
 				// add special rule token or conjure up new token from name
-				if ( Character.isUpperCase(tagChunk.tag.charAt(0)) ) {
+				if ( Character.isUpperCase(tagChunk.getTag().charAt(0)) ) {
-					Integer ttype = parser.getTokenType(tagChunk.tag);
+					Integer ttype = parser.getTokenType(tagChunk.getTag());
 					if ( ttype==Token.INVALID_TYPE ) {
-						throw new IllegalArgumentException("Unknown token "+tagChunk.tag+" in pattern: "+pattern);
+						throw new IllegalArgumentException("Unknown token "+tagChunk.getTag()+" in pattern: "+pattern);
 					}
-					TokenTagToken t = new TokenTagToken(tagChunk.tag, ttype, tagChunk.label);
+					TokenTagToken t = new TokenTagToken(tagChunk.getTag(), ttype, tagChunk.getLabel());
 					tokens.add(t);
 				}
-				else if ( Character.isLowerCase(tagChunk.tag.charAt(0)) ) {
+				else if ( Character.isLowerCase(tagChunk.getTag().charAt(0)) ) {
-					int ruleIndex = parser.getRuleIndex(tagChunk.tag);
+					int ruleIndex = parser.getRuleIndex(tagChunk.getTag());
 					if ( ruleIndex==-1 ) {
-						throw new IllegalArgumentException("Unknown rule "+tagChunk.tag+" in pattern: "+pattern);
+						throw new IllegalArgumentException("Unknown rule "+tagChunk.getTag()+" in pattern: "+pattern);
 					}
 					int ruleImaginaryTokenType = parser.getATNWithBypassAlts().ruleToTokenType[ruleIndex];
-					tokens.add(new RuleTagToken(tagChunk.tag, ruleImaginaryTokenType, tagChunk.label));
+					tokens.add(new RuleTagToken(tagChunk.getTag(), ruleImaginaryTokenType, tagChunk.getLabel()));
 				}
 				else {
-					throw new IllegalArgumentException("invalid tag: "+tagChunk.tag+" in pattern: "+pattern);
+					throw new IllegalArgumentException("invalid tag: "+tagChunk.getTag()+" in pattern: "+pattern);
 				}
 			}
 			else {
 				TextChunk textChunk = (TextChunk)chunk;
-				try {
+				ANTLRInputStream in = new ANTLRInputStream(textChunk.getText());
-					ANTLRInputStream in = new ANTLRInputStream(new StringReader(textChunk.text));
+				lexer.setInputStream(in);
-					lexer.setInputStream(in);
+				Token t = lexer.nextToken();
-					Token t = lexer.nextToken();
+				while ( t.getType()!=Token.EOF ) {
-					while ( t.getType()!=Token.EOF ) {
+					tokens.add(t);
-						tokens.add(t);
+					t = lexer.nextToken();
-						t = lexer.nextToken();
-					}
-				}
-				catch (IOException ioe) {
-					// -----------------
-					throw new IllegalArgumentException("IOException lexing pattern: "+pattern, ioe);
 				}
 			}
 		}
@@ -352,12 +414,12 @@ public class ParseTreePatternMatcher {
 		return tokens;
 	}
 
-	/** Split "<ID> = <e:expr> ;" into 4 chunks for tokenizing by tokenize() */
+	/** Split {@code <ID> = <e:expr> ;} into 4 chunks for tokenizing by {@link #tokenize}. */
 	public List<Chunk> split(String pattern) {
 		int p = 0;
 		int n = pattern.length();
 		List<Chunk> chunks = new ArrayList<Chunk>();
-		StringBuffer buf = new StringBuffer();
+		StringBuilder buf = new StringBuilder();
 		// find all start and stop indexes first, then collect
 		List<Integer> starts = new ArrayList<Integer>();
 		List<Integer> stops = new ArrayList<Integer>();
@@ -435,10 +497,14 @@ public class ParseTreePatternMatcher {
 		}
 
 		// strip out the escape sequences from text chunks but not tags
-		for (Chunk c : chunks) {
+		for (int i = 0; i < chunks.size(); i++) {
+			Chunk c = chunks.get(i);
 			if ( c instanceof TextChunk ) {
 				TextChunk tc = (TextChunk)c;
-				tc.text = tc.text.replace(escape, "");
+				String unescaped = tc.getText().replace(escape, "");
+				if (unescaped.length() < tc.getText().length()) {
+					chunks.set(i, new TextChunk(unescaped));
+				}
 			}
 		}
 

runtime/Java/src/org/antlr/v4/runtime/tree/pattern/RuleTagToken.java

@@ -33,75 +33,200 @@ package org.antlr.v4.runtime.tree.pattern;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.TokenSource;
+import org.antlr.v4.runtime.misc.NotNull;
+import org.antlr.v4.runtime.misc.Nullable;
 
-/** A token object representing an entire subtree matched by a rule; e.g., <expr> */
+/**
+ * A {@link Token} object representing an entire subtree matched by a parser
+ * rule; e.g., {@code <expr>}. These tokens are created for {@link TagChunk}
+ * chunks where the tag corresponds to a parser rule.
+ */
 public class RuleTagToken implements Token {
-	protected String ruleName;
+	/**
-	protected int ruleImaginaryTokenType;
+	 * This is the backing field for {@link #getRuleName}.
-	protected String label;
+	 */
+	private final String ruleName;
+	/**
+	 * The token type for the current token. This is the token type assigned to
+	 * the bypass alternative for the rule during ATN deserialization.
+	 */
+	private final int bypassTokenType;
+	/**
+	 * This is the backing field for {@link #getLabel}.
+	 */
+	private final String label;
 
-	public RuleTagToken(String ruleName, int ruleImaginaryTokenType) {
+	/**
-		this.ruleName = ruleName;
+	 * Constructs a new instance of {@link RuleTagToken} with the specified rule
-		this.ruleImaginaryTokenType = ruleImaginaryTokenType;
+	 * name and bypass token type and no label.
+	 *
+	 * @param ruleName The name of the parser rule this rule tag matches.
+	 * @param bypassTokenType The bypass token type assigned to the parser rule.
+	 *
+	 * @exception IllegalArgumentException if {@code ruleName} is {@code null}
+	 * or empty.
+	 */
+	public RuleTagToken(@NotNull String ruleName, int bypassTokenType) {
+		this(ruleName, bypassTokenType, null);
 	}
 
-	public RuleTagToken(String ruleName, int ruleImaginaryTokenType, String label) {
+	/**
-		this(ruleName, ruleImaginaryTokenType);
+	 * Constructs a new instance of {@link RuleTagToken} with the specified rule
+	 * name, bypass token type, and label.
+	 *
+	 * @param ruleName The name of the parser rule this rule tag matches.
+	 * @param bypassTokenType The bypass token type assigned to the parser rule.
+	 * @param label The label associated with the rule tag, or {@code null} if
+	 * the rule tag is unlabeled.
+	 *
+	 * @exception IllegalArgumentException if {@code ruleName} is {@code null}
+	 * or empty.
+	 */
+	public RuleTagToken(@NotNull String ruleName, int bypassTokenType, @Nullable String label) {
+		if (ruleName == null || ruleName.isEmpty()) {
+			throw new IllegalArgumentException("ruleName cannot be null or empty.");
+		}
+
+		this.ruleName = ruleName;
+		this.bypassTokenType = bypassTokenType;
 		this.label = label;
 	}
 
+	/**
+	 * Gets the name of the rule associated with this rule tag.
+	 *
+	 * @return The name of the parser rule associated with this rule tag.
+	 */
+	@NotNull
+	public final String getRuleName() {
+		return ruleName;
+	}
+
+	/**
+	 * Gets the label associated with the rule tag.
+	 *
+	 * @return The name of the label associated with the rule tag, or
+	 * {@code null} if this is an unlabeled rule tag.
+	 */
+	@Nullable
+	public final String getLabel() {
+		return label;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * <p/>
+	 * Rule tag tokens are always placed on the {@link #DEFAULT_CHANNEL}.
+	 */
 	@Override
 	public int getChannel() {
-		return 0;
+		return DEFAULT_CHANNEL;
 	}
 
+	/**
+	 * {@inheritDoc}
+	 * <p/>
+	 * This method returns the rule tag formatted with {@code <} and {@code >}
+	 * delimiters.
+	 */
 	@Override
 	public String getText() {
-		return "<"+ruleName+">";
+		if (label != null) {
+			return "<" + label + ":" + ruleName + ">";
+		}
+
+		return "<" + ruleName + ">";
 	}
 
+	/**
+	 * {@inheritDoc}
+	 * <p/>
+	 * Rule tag tokens have types assigned according to the rule bypass
+	 * transitions created during ATN deserialization.
+	 */
 	@Override
 	public int getType() {
-		return ruleImaginaryTokenType;
+		return bypassTokenType;
 	}
 
+	/**
+	 * {@inheritDoc}
+	 * <p/>
+	 * The implementation for {@link RuleTagToken} always returns 0.
+	 */
 	@Override
 	public int getLine() {
 		return 0;
 	}
 
+	/**
+	 * {@inheritDoc}
+	 * <p/>
+	 * The implementation for {@link RuleTagToken} always returns -1.
+	 */
 	@Override
 	public int getCharPositionInLine() {
-		return 0;
+		return -1;
 	}
 
+	/**
+	 * {@inheritDoc}
+	 * <p/>
+	 * The implementation for {@link RuleTagToken} always returns -1.
+	 */
 	@Override
 	public int getTokenIndex() {
-		return 0;
+		return -1;
 	}
 
+	/**
+	 * {@inheritDoc}
+	 * <p/>
+	 * The implementation for {@link RuleTagToken} always returns -1.
+	 */
 	@Override
 	public int getStartIndex() {
-		return 0;
+		return -1;
 	}
 
+	/**
+	 * {@inheritDoc}
+	 * <p/>
+	 * The implementation for {@link RuleTagToken} always returns -1.
+	 */
 	@Override
 	public int getStopIndex() {
-		return 0;
+		return -1;
 	}
 
+	/**
+	 * {@inheritDoc}
+	 * <p/>
+	 * The implementation for {@link RuleTagToken} always returns {@code null}.
+	 */
 	@Override
 	public TokenSource getTokenSource() {
 		return null;
 	}
 
+	/**
+	 * {@inheritDoc}
+	 * <p/>
+	 * The implementation for {@link RuleTagToken} always returns {@code null}.
+	 */
 	@Override
 	public CharStream getInputStream() {
 		return null;
 	}
 
+	/**
+	 * {@inheritDoc}
+	 * <p/>
+	 * The implementation for {@link RuleTagToken} returns a string of the form
+	 * {@code ruleName:bypassTokenType}.
+	 */
 	@Override
 	public String toString() {
-		return ruleName+":"+ ruleImaginaryTokenType;
+		return ruleName + ":" + bypassTokenType;
 	}
 }

runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TagChunk.java

@@ -30,22 +30,100 @@
 
 package org.antlr.v4.runtime.tree.pattern;
 
-/** A tag in a tree pattern string such as <expr>, <ID>, <id:ID> etc... */
+import org.antlr.v4.runtime.misc.NotNull;
-class TagChunk extends Chunk { // <e:expr> or <ID>
+import org.antlr.v4.runtime.misc.Nullable;
-	public String tag;
-	public String label;
 
+/**
+ * Represents a placeholder tag in a tree pattern. A tag can have any of the
+ * following forms.
+ *
+ * <ul>
+ * <li>{@code expr}: An unlabeled placeholder for a parser rule {@code expr}.</li>
+ * <li>{@code ID}: An unlabeled placeholder for a token of type {@code ID}.</li>
+ * <li>{@code e:expr}: A labeled placeholder for a parser rule {@code expr}.</li>
+ * <li>{@code id:ID}: A labeled placeholder for a token of type {@code ID}.</li>
+ * </ul>
+ *
+ * This class does not perform any validation on the tag or label names aside
+ * from ensuring that the tag is a non-null, non-empty string.
+ */
+class TagChunk extends Chunk {
+	/**
+	 * This is the backing field for {@link #getTag}.
+	 */
+	private final String tag;
+	/**
+	 * This is the backing field for {@link #getLabel}.
+	 */
+	private final String label;
+
+	/**
+	 * Construct a new instance of {@link TagChunk} using the specified tag and
+	 * no label.
+	 *
+	 * @param tag The tag, which should be the name of a parser rule or token
+	 * type.
+	 *
+	 * @exception IllegalArgumentException if {@code tag} is {@code null} or
+	 * empty.
+	 */
 	public TagChunk(String tag) {
-		this.tag = tag;
+		this(null, tag);
 	}
 
+	/**
+	 * Construct a new instance of {@link TagChunk} using the specified label
+	 * and tag.
+	 *
+	 * @param label The label for the tag. If this is {@code null}, the
+	 * {@link TagChunk} represents an unlabeled tag.
+	 * @param tag The tag, which should be the name of a parser rule or token
+	 * type.
+	 *
+	 * @exception IllegalArgumentException if {@code tag} is {@code null} or
+	 * empty.
+	 */
 	public TagChunk(String label, String tag) {
+		if (tag == null || tag.isEmpty()) {
+			throw new IllegalArgumentException("tag cannot be null or empty");
+		}
+
 		this.label = label;
 		this.tag = tag;
 	}
+
+	/**
+	 * Get the tag for this chunk.
+	 *
+	 * @return The tag for the chunk.
+	 */
+	@NotNull
+	public final String getTag() {
+		return tag;
+	}
+
+	/**
+	 * Get the label, if any, assigned to this chunk.
+	 *
+	 * @return The label assigned to this chunk, or {@code null} if no label is
+	 * assigned to the chunk.
+	 */
+	@Nullable
+	public final String getLabel() {
+		return label;
+	}
+
+	/**
+	 * This method returns a text representation of the tag chunk. Labeled tags
+	 * are returned in the form {@code label:tag}, and unlabeled tags are
+	 * returned as just the tag name.
+	 */
 	@Override
 	public String toString() {
-		if ( label!=null ) return label+":"+tag;
+		if (label != null) {
+			return label + ":" + tag;
+		}
+
 		return tag;
 	}
 }

runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TextChunk.java

@@ -30,13 +30,49 @@
 
 package org.antlr.v4.runtime.tree.pattern;
 
-/** The "sea" of text surrounding tags in a tree pattern string */
+import org.antlr.v4.runtime.misc.NotNull;
+
+/**
+ * Represents a span of raw text (concrete syntax) between tags in a tree
+ * pattern string.
+ */
 class TextChunk extends Chunk {
-	public String text;
+	/**
-	public TextChunk(String text) {
+	 * This is the backing field for {@link #getText}.
+	 */
+	@NotNull
+	private final String text;
+
+	/**
+	 * Constructs a new instance of {@link TextChunk} with the specified text.
+	 *
+	 * @param text The text of this chunk.
+	 * @exception IllegalArgumentException if {@code text} is {@code null}.
+	 */
+	public TextChunk(@NotNull String text) {
+		if (text == null) {
+			throw new IllegalArgumentException("text cannot be null");
+		}
+
 		this.text = text;
 	}
 
+	/**
+	 * Gets the raw text of this chunk.
+	 *
+	 * @return The text of the chunk.
+	 */
+	@NotNull
+	public final String getText() {
+		return text;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * <p/>
+	 * The implementation for {@link TextChunk} returns the result of
+	 * {@link #getText()} in single quotes.
+	 */
 	@Override
 	public String toString() {
 		return "'"+text+"'";

runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TokenTagToken.java

@@ -31,29 +31,96 @@
 package org.antlr.v4.runtime.tree.pattern;
 
 import org.antlr.v4.runtime.CommonToken;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.misc.NotNull;
+import org.antlr.v4.runtime.misc.Nullable;
 
-/** A Token in a tree pattern string representing a token reference; e.g., <ID> */
+/**
+ * A {@link Token} object representing a token of a particular type; e.g.,
+ * {@code <ID>}. These tokens are created for {@link TagChunk} chunks where the
+ * tag corresponds to a lexer rule or token type.
+ */
 public class TokenTagToken extends CommonToken {
-	protected String tokenName;
+	/**
-	protected String label;
+	 * This is the backing field for {@link #getTokenName}.
+	 */
+	@NotNull
+	private final String tokenName;
+	/**
+	 * This is the backing field for {@link #getLabel}.
+	 */
+	@Nullable
+	private final String label;
 
-	public TokenTagToken(String tokenName, int type) {
+	/**
-		super(type);
+	 * Constructs a new instance of {@link TokenTagToken} for an unlabeled tag
-		this.tokenName = tokenName;
+	 * with the specified token name and type.
+	 *
+	 * @param tokenName The token name.
+	 * @param type The token type.
+	 */
+	public TokenTagToken(@NotNull String tokenName, int type) {
+		this(tokenName, type, null);
 	}
 
-	public TokenTagToken(String tokenName, int type, String label) {
+	/**
-		this(tokenName, type);
+	 * Constructs a new instance of {@link TokenTagToken} with the specified
+	 * token name, type, and label.
+	 *
+	 * @param tokenName The token name.
+	 * @param type The token type.
+	 * @param label The label associated with the token tag, or {@code null} if
+	 * the token tag is unlabeled.
+	 */
+	public TokenTagToken(@NotNull String tokenName, int type, @Nullable String label) {
+		super(type);
+		this.tokenName = tokenName;
 		this.label = label;
 	}
 
-	@Override
+	/**
-	public String getText() {
+	 * Gets the token name.
-		return "<"+tokenName+">";
+	 * @return The token name.
+	 */
+	@NotNull
+	public final String getTokenName() {
+		return tokenName;
 	}
 
+	/**
+	 * Gets the label associated with the rule tag.
+	 *
+	 * @return The name of the label associated with the rule tag, or
+	 * {@code null} if this is an unlabeled rule tag.
+	 */
+	@Nullable
+	public final String getLabel() {
+		return label;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * <p/>
+	 * The implementation for {@link TokenTagToken} returns the token tag
+	 * formatted with {@code <} and {@code >} delimiters.
+	 */
+	@Override
+	public String getText() {
+		if (label != null) {
+			return "<" + label + ":" + tokenName + ">";
+		}
+
+		return "<" + tokenName + ">";
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * <p/>
+	 * The implementation for {@link TokenTagToken} returns a string of the form
+	 * {@code tokenName:type}.
+	 */
 	@Override
 	public String toString() {
-		return tokenName+":"+type;
+		return tokenName + ":" + type;
 	}
 }

runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPath.java

@@ -6,7 +6,6 @@ import org.antlr.v4.runtime.LexerNoViableAltException;
 import org.antlr.v4.runtime.Parser;
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.Token;
-import org.antlr.v4.runtime.misc.Utils;
 import org.antlr.v4.runtime.tree.ParseTree;
 
 import java.io.IOException;
@@ -14,7 +13,6 @@ import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.Map;
 
 /** Represent a subset of XPath XML path syntax for use in identifying nodes in
  *  parse trees.
@@ -29,7 +27,7 @@ import java.util.Map;
  *  XPath p = new XPath(parser, xpath);
  *  return p.evaluate(this);
  *
- *  See {@link org.antlr.v4.test.TestXPath} for descriptions. In short, this allows
+ *  See {@code org.antlr.v4.test.TestXPath} for descriptions. In short, this allows
  *  operators:
  *
  *      /         root

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -56,7 +56,6 @@ import org.antlr.v4.runtime.tree.*;
 import java.util.List;
 import java.util.Iterator;
 import java.util.ArrayList;
-import java.util.Map;
 
 <parser>
 >>
@@ -225,13 +224,11 @@ public class <parser.name> extends <superClass> {
 	public static final String[] tokenNames = {
 		<parser.tokenNames:{t | <t>}; null="\"\<INVALID>\"", separator=", ", wrap, anchor>
 	};
-	public static final Map\<String, Integer> tokenNameToType = Utils.toMap(tokenNames);
 	public static final int
 		<parser.rules:{r | RULE_<r.name> = <r.index>}; separator=", ", wrap, anchor>;
 	public static final String[] ruleNames = {
 		<parser.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor>
 	};
-	public static final Map\<String, Integer> ruleNameToIndex = Utils.toMap(ruleNames);
 
 	@Override
 	public String getGrammarFileName() { return "<parser.grammarFileName; format="java-escape">"; }
@@ -247,15 +244,6 @@ public class <parser.name> extends <superClass> {
 
 	@Override
 	public ATN getATN() { return _ATN; }
-	@Override
-	public ATN getATNWithBypassAlts() { return _ATNWithBypassAlts; }
-	@Override
-	public void setATNWithBypassAlts(ATN atn) { _ATNWithBypassAlts = atn; }
-
-	@Override
-	public Map\<String, Integer> getTokenTypeMap() { return tokenNameToType; }
-	@Override
-	public Map\<String, Integer> getRuleIndexMap() { return ruleNameToIndex; }
 
 	<namedActions.members>
 	<parser:(ctor)()>
@@ -853,7 +841,6 @@ import org.antlr.v4.runtime.*;
 import org.antlr.v4.runtime.atn.*;
 import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.runtime.misc.*;
-import java.util.Map;
 
 <lexer>
 >>
@@ -875,16 +862,9 @@ public class <lexer.name> extends <superClass> {
 		"\<INVALID>",
 		<lexer.tokenNames:{t | <t>}; separator=", ", wrap, anchor>
 	};
-	public static final Map\<String, Integer> tokenNameToType = Utils.toMap(tokenNames);
 	public static final String[] ruleNames = {
 		<lexer.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor>
 	};
-	public static final Map\<String, Integer> ruleNameToIndex = Utils.toMap(ruleNames);
-
-	@Override
-	public Map\<String, Integer> getTokenTypeMap() { return tokenNameToType; }
-	@Override
-	public Map\<String, Integer> getRuleIndexMap() { return ruleNameToIndex; }
 
 	<namedActions.members>
 
@@ -935,7 +915,6 @@ public static final String _serializedATN =
 <endif>
 public static final ATN _ATN =
 	new ATNDeserializer().deserialize(_serializedATN.toCharArray());
-public static ATN _ATNWithBypassAlts;
 static {
 	_decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
 	for (int i = 0; i \< _ATN.getNumberOfDecisions(); i++) {

tool/test/org/antlr/v4/test/TestParseTreeMatcher.java

@@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue;
 
 public class TestParseTreeMatcher extends BaseTest {
 	@Test public void testChunking() throws Exception {
-		ParseTreePatternMatcher m = new ParseTreePatternMatcher();
+		ParseTreePatternMatcher m = new ParseTreePatternMatcher(null, null);
 		assertEquals("[ID, ' = ', expr, ' ;']", m.split("<ID> = <expr> ;").toString());
 		assertEquals("[' ', ID, ' = ', expr]", m.split(" <ID> = <expr>").toString());
 		assertEquals("[ID, ' = ', expr]", m.split("<ID> = <expr>").toString());
@@ -33,14 +33,14 @@ public class TestParseTreeMatcher extends BaseTest {
 	}
 
 	@Test public void testDelimiters() throws Exception {
-		ParseTreePatternMatcher m = new ParseTreePatternMatcher();
+		ParseTreePatternMatcher m = new ParseTreePatternMatcher(null, null);
 		m.setDelimiters("<<", ">>", "$");
 		String result = m.split("<<ID>> = <<expr>> ;$<< ick $>>").toString();
 		assertEquals("[ID, ' = ', expr, ' ;<< ick >>']", result);
 	}
 
 	@Test public void testInvertedTags() throws Exception {
-		ParseTreePatternMatcher m= new ParseTreePatternMatcher();
+		ParseTreePatternMatcher m= new ParseTreePatternMatcher(null, null);
 		String result = null;
 		try {
 			m.split(">expr<");
@@ -53,7 +53,7 @@ public class TestParseTreeMatcher extends BaseTest {
 	}
 
 	@Test public void testUnclosedTag() throws Exception {
-		ParseTreePatternMatcher m = new ParseTreePatternMatcher();
+		ParseTreePatternMatcher m = new ParseTreePatternMatcher(null, null);
 		String result = null;
 		try {
 			m.split("<expr hi mom");
@@ -66,7 +66,7 @@ public class TestParseTreeMatcher extends BaseTest {
 	}
 
 	@Test public void testExtraClose() throws Exception {
-		ParseTreePatternMatcher m = new ParseTreePatternMatcher();
+		ParseTreePatternMatcher m = new ParseTreePatternMatcher(null, null);
 		String result = null;
 		try {
 			m.split("<expr> >");
@@ -214,7 +214,7 @@ public class TestParseTreeMatcher extends BaseTest {
 		assertEquals("[y]", m.getAll("b").toString());
 		assertEquals("[x, y, z]", m.getAll("ID").toString()); // ordered
 
-		assertEquals("xyz;", m.getText()); // whitespace stripped by lexer
+		assertEquals("xyz;", m.getTree().getText()); // whitespace stripped by lexer
 
 		assertNull(m.get("undefined"));
 		assertEquals("[]", m.getAll("undefined").toString());