diff --git a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java index 9d5d7cc69..ede5c3b73 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java @@ -32,7 +32,9 @@ import org.antlr.v4.runtime.atn.LexerATNSimulator; import org.antlr.v4.runtime.misc.Interval; import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.EmptyStackException; +import java.util.List; /** A lexer is recognizer that draws input symbols from a character stream. * lexer grammars result in a subclass of this object. A Lexer object @@ -337,6 +339,19 @@ public abstract class Lexer extends Recognizer return null; } + /** Return a list of all Token objects in input char stream. + * Forces load of all tokens. Does not include EOF token. + */ + public List getAllTokens() { + List tokens = new ArrayList(); + Token t = nextToken(); + while ( t.getType()!=Token.EOF ) { + tokens.add(t); + t = nextToken(); + } + return tokens; + } + public void recover(LexerNoViableAltException e) { getInterpreter().consume(_input); // skip a char and try again }