Fix usage of BufferedTokenStream: cannot consume EOF

runtime/Java/src/org/antlr/v4/runtime/Parser.java

@@ -382,7 +382,9 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
 	 */
 	public Token consume() {
 		Token o = getCurrentToken();
-		getInputStream().consume();
+		if (o.getType() != EOF) {
+			getInputStream().consume();
+		}
 		boolean hasListener = _parseListeners != null && !_parseListeners.isEmpty();
 		if (_buildParseTrees || hasListener) {
 			if ( _errHandler.inErrorRecoveryMode(this) ) {

tool/test/org/antlr/v4/test/TestBufferedTokenStream.java

@@ -145,10 +145,6 @@ public class TestBufferedTokenStream extends BaseTest {
             tokens.consume();
             t = tokens.LT(1);
         }
-        tokens.consume();
-        tokens.LT(1); // push it past end
-        tokens.consume();
-        tokens.LT(1);
 
         String result = tokens.getText();
         String expecting = "x = 3 * 0 + 2 * 0;";

tool/test/org/antlr/v4/test/TestCommonTokenStream.java

@@ -255,12 +255,50 @@ public class TestCommonTokenStream extends TestBufferedTokenStream {
 		assertEquals(Token.EOF, tokens.LA(1));
 		assertEquals(0, tokens.index());
 		assertEquals(1, tokens.size());
-		tokens.consume();
+	}
 
-		assertEquals(Token.EOF, tokens.LA(1));
-		assertEquals(0, tokens.index());
-		assertEquals(1, tokens.size());
-		tokens.consume();
+	@Test(expected = IllegalStateException.class)
+	public void testCannotConsumeEOF() throws Exception {
+		TokenSource lexer = new TokenSource() {
+
+			@Override
+			public Token nextToken() {
+				return new CommonToken(Token.EOF);
+			}
+
+			@Override
+			public int getLine() {
+				return 0;
+			}
+
+			@Override
+			public int getCharPositionInLine() {
+				return 0;
+			}
+
+			@Override
+			public CharStream getInputStream() {
+				return null;
+			}
+
+			@Override
+			public String getSourceName() {
+				return null;
+			}
+
+			@Override
+			public TokenFactory<?> getTokenFactory() {
+				throw new UnsupportedOperationException("Not supported yet.");
+			}
+
+			@Override
+			public void setTokenFactory(TokenFactory<?> factory) {
+				throw new UnsupportedOperationException("Not supported yet.");
+			}
+		};
+
+		CommonTokenStream tokens = new CommonTokenStream(lexer);
+		tokens.fill();
 
 		assertEquals(Token.EOF, tokens.LA(1));
 		assertEquals(0, tokens.index());