Merge branch 'token-stream-bugs' of git://github.com/sharwell/antlr4

runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java

@@ -52,6 +52,7 @@ import java.util.Set;
  *  to confuse small moving window of tokens it uses for the full buffer.
  */
 public class BufferedTokenStream implements TokenStream {
+	@NotNull
     protected TokenSource tokenSource;
 
     /** Record every single token pulled from the source so we can reproduce
@@ -61,9 +62,6 @@ public class BufferedTokenStream implements TokenStream {
      */
     protected List<Token> tokens = new ArrayList<Token>(100);
 
-    /** Track the last mark() call result value for use in rewind(). */
-    protected int lastMarker;
-
     /** The index into the tokens list of the current token (next token
      *  to consume).  tokens[p] should be LT(1).  p=-1 indicates need
      *  to initialize with first token.  The ctor doesn't get a token.
@@ -71,9 +69,19 @@ public class BufferedTokenStream implements TokenStream {
      */
     protected int p = -1;
 
-    public BufferedTokenStream() { }
+	/**
+	 * Set to {@code true} when the EOF token is fetched. Do not continue fetching
+	 * tokens after that point, or multiple EOF tokens could end up in the
+	 * {@link #tokens} array.
+	 *
+	 * @see #fetch
+	 */
+	protected boolean fetchedEOF;
 
     public BufferedTokenStream(TokenSource tokenSource) {
+		if (tokenSource == null) {
+			throw new NullPointerException("tokenSource cannot be null");
+		}
         this.tokenSource = tokenSource;
     }
 
@@ -96,17 +104,13 @@ public class BufferedTokenStream implements TokenStream {
 	}
 
     public void reset() {
-        p = 0;
-        lastMarker = 0;
+        seek(0);
     }
 
     @Override
     public void seek(int index) {
-        if (p == -1) {
-            setup();
-        }
-
-        p = index;
+        lazyInit();
+        p = adjustSeekIndex(index);
     }
 
     @Override
@@ -115,34 +119,56 @@ public class BufferedTokenStream implements TokenStream {
     /** Move the input pointer to the next incoming token.  The stream
      *  must become active with LT(1) available.  consume() simply
      *  moves the input pointer so that LT(1) points at the next
-     *  input symbol. Consume at least one token.
-     *
-     *  Walk past any token not on the channel the parser is listening to.
+     *  input symbol. Consume at least one token, unless EOF has been reached.
      */
     @Override
     public void consume() {
-        if ( p == -1 ) setup();
-        p++;
-        sync(p);
+        lazyInit();
+		if (sync(p + 1)) {
+			p = adjustSeekIndex(p + 1);
+		}
     }
 
-    /** Make sure index i in tokens has a token. */
-    protected void sync(int i) {
+    /** Make sure index {@code i} in tokens has a token.
+	 *
+	 * @return {@code true} if a token is located at index {@code i}, otherwise
+	 *    {@code false}.
+	 * @see #get(int i)
+	 */
+    protected boolean sync(int i) {
+		assert i >= 0;
         int n = i - tokens.size() + 1; // how many more elements we need?
         //System.out.println("sync("+i+") needs "+n);
-        if ( n > 0 ) fetch(n);
+        if ( n > 0 ) {
+			int fetched = fetch(n);
+			return fetched >= n;
+		}
+
+		return true;
     }
 
-    /** add n elements to buffer */
-    protected void fetch(int n) {
-        for (int i=1; i<=n; i++) {
+    /** Add {@code n} elements to buffer.
+	 *
+	 * @return The actual number of elements added to the buffer.
+	 */
+    protected int fetch(int n) {
+		if (fetchedEOF) {
+			return 0;
+		}
+
+        for (int i = 0; i < n; i++) {
             Token t = tokenSource.nextToken();
             if ( t instanceof WritableToken ) {
                 ((WritableToken)t).setTokenIndex(tokens.size());
             }
             tokens.add(t);
-            if ( t.getType()==Token.EOF ) break;
+            if ( t.getType()==Token.EOF ) {
+				fetchedEOF = true;
+				return i + 1;
+			}
         }
+
+		return n;
     }
 
     @Override
@@ -156,7 +182,7 @@ public class BufferedTokenStream implements TokenStream {
 	/** Get all tokens from start..stop inclusively */
 	public List<Token> get(int start, int stop) {
 		if ( start<0 || stop<0 ) return null;
-		if ( p == -1 ) setup();
+		lazyInit();
 		List<Token> subset = new ArrayList<Token>();
 		if ( stop>=tokens.size() ) stop = tokens.size()-1;
 		for (int i = start; i <= stop; i++) {
@@ -177,7 +203,7 @@ public class BufferedTokenStream implements TokenStream {
 
     @Override
     public Token LT(int k) {
-        if ( p == -1 ) setup();
+        lazyInit();
         if ( k==0 ) return null;
         if ( k < 0 ) return LB(-k);
 
@@ -191,7 +217,33 @@ public class BufferedTokenStream implements TokenStream {
         return tokens.get(i);
     }
 
-    protected void setup() { sync(0); p = 0; }
+	/**
+	 * Allowed derived classes to modify the behavior of operations which change
+	 * the current stream position by adjusting the target token index of a seek
+	 * operation. The default implementation simply returns {@code i}. If an
+	 * exception is thrown in this method, the current stream index should not be
+	 * changed.
+	 * <p>
+	 * For example, {@link CommonTokenStream} overrides this method to ensure that
+	 * the seek target is always an on-channel token.
+	 *
+	 * @param i The target token index.
+	 * @return The adjusted target token index.
+	 */
+	protected int adjustSeekIndex(int i) {
+		return i;
+	}
+
+	protected final void lazyInit() {
+		if (p == -1) {
+			setup();
+		}
+	}
+
+    protected void setup() {
+		sync(0);
+		p = adjustSeekIndex(0);
+	}
 
     /** Reset this token stream by setting its token source. */
     public void setTokenSource(TokenSource tokenSource) {
@@ -211,7 +263,7 @@ public class BufferedTokenStream implements TokenStream {
      *  method looks at both on and off channel tokens.
      */
     public List<Token> getTokens(int start, int stop, Set<Integer> types) {
-        if ( p == -1 ) setup();
+        lazyInit();
 		if ( start<0 || stop>=tokens.size() ||
 			 stop<0  || start>=tokens.size() )
 		{
@@ -273,7 +325,7 @@ public class BufferedTokenStream implements TokenStream {
 	 *  EOF. If channel is -1, find any non default channel token.
 	 */
 	public List<Token> getHiddenTokensToRight(int tokenIndex, int channel) {
-		if ( p == -1 ) setup();
+		lazyInit();
 		if ( tokenIndex<0 || tokenIndex>=tokens.size() ) {
 			throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1));
 		}
@@ -302,7 +354,7 @@ public class BufferedTokenStream implements TokenStream {
 	 *  If channel is -1, find any non default channel token.
 	 */
 	public List<Token> getHiddenTokensToLeft(int tokenIndex, int channel) {
-		if ( p == -1 ) setup();
+		lazyInit();
 		if ( tokenIndex<0 || tokenIndex>=tokens.size() ) {
 			throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1));
 		}
@@ -346,7 +398,7 @@ public class BufferedTokenStream implements TokenStream {
 	@NotNull
 	@Override
 	public String getText() {
-		if ( p == -1 ) setup();
+        lazyInit();
 		fill();
 		return getText(Interval.of(0,size()-1));
 	}
@@ -357,7 +409,7 @@ public class BufferedTokenStream implements TokenStream {
 		int start = interval.a;
 		int stop = interval.b;
         if ( start<0 || stop<0 ) return "";
-        if ( p == -1 ) setup();
+        lazyInit();
         if ( stop>=tokens.size() ) stop = tokens.size()-1;
 
 		StringBuilder buf = new StringBuilder();
@@ -387,14 +439,13 @@ public class BufferedTokenStream implements TokenStream {
 
     /** Get all tokens from lexer until EOF */
     public void fill() {
-        if ( p == -1 ) setup();
-        if ( tokens.get(p).getType()==Token.EOF ) return;
-
-        int i = p+1;
-        sync(i);
-        while ( tokens.get(i).getType()!=Token.EOF ) {
-            i++;
-            sync(i);
-        }
+        lazyInit();
+		final int blockSize = 1000;
+		while (true) {
+			int fetched = fetch(blockSize);
+			if (fetched < blockSize) {
+				return;
+			}
+		}
     }
 }

runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java

@@ -50,8 +50,6 @@ public class CommonTokenStream extends BufferedTokenStream {
     /** Skip tokens on any channel but this one; this is how we skip whitespace... */
     protected int channel = Token.DEFAULT_CHANNEL;
 
-    public CommonTokenStream() { ; }
-
     public CommonTokenStream(TokenSource tokenSource) {
         super(tokenSource);
     }
@@ -61,32 +59,9 @@ public class CommonTokenStream extends BufferedTokenStream {
         this.channel = channel;
     }
 
-    /** Always leave p on an on-channel token. */
-    @Override
-    public void consume() {
-        if ( p == -1 ) setup();
-        p++;
-        sync(p);
-		Token t = tokens.get(p);
-		while ( t.getType()!=Token.EOF && t.getChannel()!=channel ) {
-            p++;
-            sync(p);
-			t = tokens.get(p);
-        }
-    }
-
-    @Override
-    public void seek(int index) {
-        super.seek(index);
-        while (p < index) {
-            consume();
-        }
-    }
-
 	@Override
-	public void reset() {
-		super.reset();
-		p = nextTokenOnChannel(p, channel);
+	protected int adjustSeekIndex(int i) {
+		return nextTokenOnChannel(i, channel);
 	}
 
     @Override
@@ -108,35 +83,23 @@ public class CommonTokenStream extends BufferedTokenStream {
     @Override
     public Token LT(int k) {
         //System.out.println("enter LT("+k+")");
-        if ( p == -1 ) setup();
+        lazyInit();
         if ( k == 0 ) return null;
         if ( k < 0 ) return LB(-k);
         int i = p;
         int n = 1; // we know tokens[p] is a good one
         // find k good tokens
         while ( n<k ) {
-            // skip off-channel tokens
-            i = nextTokenOnChannel(i + 1, channel);
+            // skip off-channel tokens, but make sure to not look past EOF
+			if (sync(i + 1)) {
+				i = nextTokenOnChannel(i + 1, channel);
+			}
             n++;
         }
 //		if ( i>range ) range = i;
         return tokens.get(i);
     }
 
-    @Override
-    protected void setup() {
-        p = 0;
-        sync(0);
-        int i = 0;
-        Token token = tokens.get(i);
-        while ( token.getType()!=Token.EOF && token.getChannel()!=channel ) {
-            i++;
-            sync(i);
-            token = tokens.get(i);
-        }
-        p = i;
-    }
-
 	/** Count EOF just once. */
 	public int getNumberOfOnChannelTokens() {
 		int n = 0;
@@ -148,11 +111,4 @@ public class CommonTokenStream extends BufferedTokenStream {
 		}
 		return n;
 	}
-
-    /** Reset this token stream by setting its token source. */
-    @Override
-    public void setTokenSource(TokenSource tokenSource) {
-        super.setTokenSource(tokenSource);
-        channel = Token.DEFAULT_CHANNEL;
-    }
 }

tool/test/org/antlr/v4/test/BaseTest.java

@@ -892,12 +892,17 @@ public abstract class BaseTest {
         public FilteringTokenStream(TokenSource src) { super(src); }
         Set<Integer> hide = new HashSet<Integer>();
         @Override
-        protected void sync(int i) {
-            super.sync(i);
+        protected boolean sync(int i) {
+            if (!super.sync(i)) {
+				return false;
+			}
+
 			Token t = get(i);
 			if ( hide.contains(t.getType()) ) {
 				((WritableToken)t).setChannel(Token.HIDDEN_CHANNEL);
 			}
+
+			return true;
         }
         public void setTokenTypeChannel(int ttype, int channel) {
             hide.add(ttype);

tool/test/org/antlr/v4/test/TestCommonTokenStream.java

@@ -176,4 +176,63 @@ public class TestCommonTokenStream extends TestBufferedTokenStream {
 					 tokens.getHiddenTokensToLeft(9).toString());
 		assertEquals(null, tokens.getHiddenTokensToRight(9));
 	}
+
+	@Test
+	public void testSingleEOF() throws Exception {
+		TokenSource lexer = new TokenSource() {
+
+			@Override
+			public Token nextToken() {
+				return new CommonToken(Token.EOF);
+			}
+
+			@Override
+			public int getLine() {
+				return 0;
+			}
+
+			@Override
+			public int getCharPositionInLine() {
+				return 0;
+			}
+
+			@Override
+			public CharStream getInputStream() {
+				return null;
+			}
+
+			@Override
+			public String getSourceName() {
+				return null;
+			}
+
+			@Override
+			public TokenFactory<?> getTokenFactory() {
+				throw new UnsupportedOperationException("Not supported yet.");
+			}
+
+			@Override
+			public void setTokenFactory(TokenFactory<?> factory) {
+				throw new UnsupportedOperationException("Not supported yet.");
+			}
+		};
+
+		CommonTokenStream tokens = new CommonTokenStream(lexer);
+		tokens.fill();
+
+		assertEquals(Token.EOF, tokens.LA(1));
+		assertEquals(0, tokens.index());
+		assertEquals(1, tokens.size());
+		tokens.consume();
+
+		assertEquals(Token.EOF, tokens.LA(1));
+		assertEquals(0, tokens.index());
+		assertEquals(1, tokens.size());
+		tokens.consume();
+
+		assertEquals(Token.EOF, tokens.LA(1));
+		assertEquals(0, tokens.index());
+		assertEquals(1, tokens.size());
+		tokens.consume();
+	}
 }