Updated documentation for Token and TokenSource

runtime/Java/src/org/antlr/v4/runtime/Token.java

@@ -57,7 +57,9 @@ public interface Token {
 	 */
 	public static final int HIDDEN_CHANNEL = 1;
 
-	/** Get the text of the token */
+	/**
+	 * Get the text of the token.
+	 */
 	String getText();
 
 	/** Get the token type of the token */

runtime/Java/src/org/antlr/v4/runtime/TokenSource.java

@@ -30,46 +30,83 @@
 package org.antlr.v4.runtime;
 
 import org.antlr.v4.runtime.misc.NotNull;
+import org.antlr.v4.runtime.misc.Nullable;
 
-/** A source of tokens must provide a sequence of tokens via nextToken()
- *  and also must reveal it's source of characters; CommonToken's text is
- *  computed from a CharStream; it only store indices into the char stream.
- *
- *  Errors from the lexer are never passed to the parser.  Either you want
- *  to keep going or you do not upon token recognition error.  If you do not
- *  want to continue lexing then you do not want to continue parsing.  Just
- *  throw an exception not under RecognitionException and Java will naturally
- *  toss you all the way out of the recognizers.  If you want to continue
- *  lexing then you should not throw an exception to the parser--it has already
- *  requested a token.  Keep lexing until you get a valid one.  Just report
- *  errors and keep going, looking for a valid token.
+/**
+ * A source of tokens must provide a sequence of tokens via {@link #nextToken()}
+ * and also must reveal it's source of characters; {@link CommonToken}'s text is
+ * computed from a {@link CharStream}; it only store indices into the char
+ * stream.
+ * <p/>
+ * Errors from the lexer are never passed to the parser. Either you want to keep
+ * going or you do not upon token recognition error. If you do not want to
+ * continue lexing then you do not want to continue parsing. Just throw an
+ * exception not under {@link RecognitionException} and Java will naturally toss
+ * you all the way out of the recognizers. If you want to continue lexing then
+ * you should not throw an exception to the parser--it has already requested a
+ * token. Keep lexing until you get a valid one. Just report errors and keep
+ * going, looking for a valid token.
  */
 public interface TokenSource {
-	/** Return a Token object from your input stream (usually a CharStream).
-	 *  Do not fail/return upon lexing error; keep chewing on the characters
-	 *  until you get a good one; errors are not passed through to the parser.
+	/**
+	 * Return a {@link Token} object from your input stream (usually a
+	 * {@link CharStream}). Do not fail/return upon lexing error; keep chewing
+	 * on the characters until you get a good one; errors are not passed through
+	 * to the parser.
 	 */
+	@NotNull
 	public Token nextToken();
 
+	/**
+	 * Get the line number for the current position in the input stream. The
+	 * first line in the input is line 1.
+	 *
+	 * @return The line number for the current position in the input stream, or
+	 * 0 if the current token source does not track line numbers.
+	 */
 	public int getLine();
 
+	/**
+	 * Get the index into the current line for the current position in the input
+	 * stream. The first character on a line has position 0.
+	 *
+	 * @return The line number for the current position in the input stream, or
+	 * -1 if the current token source does not track character positions.
+	 */
 	public int getCharPositionInLine();
 
-	/** From what character stream was this token created?  You don't have to
-	 *  implement but it's nice to know where a Token comes from if you have
-	 *  include files etc... on the input.
+	/**
+	 * Get the {@link CharStream} from which this token source is currently
+	 * providing tokens.
+	 *
+	 * @return The {@link CharStream} associated with the current position in
+	 * the input, or {@code null} if no input stream is available for the token
+	 * source.
 	 */
+	@Nullable
 	public CharStream getInputStream();
 
-	/** Where are you getting tokens from? normally the implication will simply
-	 *  ask lexers input stream.
+	/**
+	 * Gets the name of the underlying input source. This method returns a
+	 * non-null, non-empty string. If such a name is not known, this method
+	 * returns {@link IntStream#UNKNOWN_SOURCE_NAME}.
 	 */
 	public String getSourceName();
 
-	/** Optional method that lets users set factory in lexer or other source */
+	/**
+	 * Set the {@link TokenFactory} this token source should use for creating
+	 * {@link Token} objects from the input.
+	 *
+	 * @param factory The {@link TokenFactory} to use for creating tokens.
+	 */
 	public void setTokenFactory(@NotNull TokenFactory<?> factory);
 
-	/** Gets the factory used for constructing tokens. */
+	/**
+	 * Gets the {@link TokenFactory} this token source is currently using for
+	 * creating {@link Token} objects from the input.
+	 *
+	 * @return The {@link TokenFactory} currently used by this token source.
+	 */
 	@NotNull
 	public TokenFactory<?> getTokenFactory();
 }