diff --git a/runtime/Java/src/org/antlr/v4/runtime/Token.java b/runtime/Java/src/org/antlr/v4/runtime/Token.java index d1915812e..0ea2deb63 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Token.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Token.java @@ -57,7 +57,9 @@ public interface Token { */ public static final int HIDDEN_CHANNEL = 1; - /** Get the text of the token */ + /** + * Get the text of the token. + */ String getText(); /** Get the token type of the token */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java b/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java index 3ffc631f2..4f8190b19 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java +++ b/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java @@ -30,46 +30,83 @@ package org.antlr.v4.runtime; import org.antlr.v4.runtime.misc.NotNull; +import org.antlr.v4.runtime.misc.Nullable; -/** A source of tokens must provide a sequence of tokens via nextToken() - * and also must reveal it's source of characters; CommonToken's text is - * computed from a CharStream; it only store indices into the char stream. - * - * Errors from the lexer are never passed to the parser. Either you want - * to keep going or you do not upon token recognition error. If you do not - * want to continue lexing then you do not want to continue parsing. Just - * throw an exception not under RecognitionException and Java will naturally - * toss you all the way out of the recognizers. If you want to continue - * lexing then you should not throw an exception to the parser--it has already - * requested a token. Keep lexing until you get a valid one. Just report - * errors and keep going, looking for a valid token. +/** + * A source of tokens must provide a sequence of tokens via {@link #nextToken()} + * and also must reveal it's source of characters; {@link CommonToken}'s text is + * computed from a {@link CharStream}; it only store indices into the char + * stream. + *

+ * Errors from the lexer are never passed to the parser. Either you want to keep + * going or you do not upon token recognition error. If you do not want to + * continue lexing then you do not want to continue parsing. Just throw an + * exception not under {@link RecognitionException} and Java will naturally toss + * you all the way out of the recognizers. If you want to continue lexing then + * you should not throw an exception to the parser--it has already requested a + * token. Keep lexing until you get a valid one. Just report errors and keep + * going, looking for a valid token. */ public interface TokenSource { - /** Return a Token object from your input stream (usually a CharStream). - * Do not fail/return upon lexing error; keep chewing on the characters - * until you get a good one; errors are not passed through to the parser. + /** + * Return a {@link Token} object from your input stream (usually a + * {@link CharStream}). Do not fail/return upon lexing error; keep chewing + * on the characters until you get a good one; errors are not passed through + * to the parser. */ + @NotNull public Token nextToken(); + /** + * Get the line number for the current position in the input stream. The + * first line in the input is line 1. + * + * @return The line number for the current position in the input stream, or + * 0 if the current token source does not track line numbers. + */ public int getLine(); + /** + * Get the index into the current line for the current position in the input + * stream. The first character on a line has position 0. + * + * @return The line number for the current position in the input stream, or + * -1 if the current token source does not track character positions. + */ public int getCharPositionInLine(); - /** From what character stream was this token created? You don't have to - * implement but it's nice to know where a Token comes from if you have - * include files etc... on the input. + /** + * Get the {@link CharStream} from which this token source is currently + * providing tokens. + * + * @return The {@link CharStream} associated with the current position in + * the input, or {@code null} if no input stream is available for the token + * source. */ + @Nullable public CharStream getInputStream(); - /** Where are you getting tokens from? normally the implication will simply - * ask lexers input stream. + /** + * Gets the name of the underlying input source. This method returns a + * non-null, non-empty string. If such a name is not known, this method + * returns {@link IntStream#UNKNOWN_SOURCE_NAME}. */ public String getSourceName(); - /** Optional method that lets users set factory in lexer or other source */ + /** + * Set the {@link TokenFactory} this token source should use for creating + * {@link Token} objects from the input. + * + * @param factory The {@link TokenFactory} to use for creating tokens. + */ public void setTokenFactory(@NotNull TokenFactory factory); - /** Gets the factory used for constructing tokens. */ + /** + * Gets the {@link TokenFactory} this token source is currently using for + * creating {@link Token} objects from the input. + * + * @return The {@link TokenFactory} currently used by this token source. + */ @NotNull public TokenFactory getTokenFactory(); }