From 7e910e90a957fa9b76c9cf8f729d4cd44be6fd2a Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Fri, 16 Mar 2012 20:05:49 -0500 Subject: [PATCH 01/12] Don't allow BufferedTokenStream to be constructed without a non-null TokenSource --- .../src/org/antlr/v4/runtime/BufferedTokenStream.java | 8 ++++++-- .../Java/src/org/antlr/v4/runtime/CommonTokenStream.java | 2 -- .../Java/src/org/antlr/v4/runtime/TokenRewriteStream.java | 4 ---- tool/test/org/antlr/v4/test/TestPerformance.java | 4 ++-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index e038357c8..422ecf382 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -29,6 +29,8 @@ package org.antlr.v4.runtime; +import org.antlr.v4.runtime.misc.NotNull; + import java.util.*; /** Buffer all input tokens but do on-demand fetching of new tokens from @@ -46,6 +48,7 @@ import java.util.*; * to confuse small moving window of tokens it uses for the full buffer. */ public class BufferedTokenStream implements TokenStream { + @NotNull protected TokenSource tokenSource; /** Record every single token pulled from the source so we can reproduce @@ -65,9 +68,10 @@ public class BufferedTokenStream implements TokenStream { */ protected int p = -1; - public BufferedTokenStream() { } - public BufferedTokenStream(TokenSource tokenSource) { + if (tokenSource == null) { + throw new NullPointerException("tokenSource cannot be null"); + } this.tokenSource = tokenSource; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java index 2b9f38bac..376704069 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java @@ -50,8 +50,6 @@ public class CommonTokenStream extends BufferedTokenStream { /** Skip tokens on any channel but this one; this is how we skip whitespace... */ protected int channel = Token.DEFAULT_CHANNEL; - public CommonTokenStream() { ; } - public CommonTokenStream(TokenSource tokenSource) { super(tokenSource); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenRewriteStream.java b/runtime/Java/src/org/antlr/v4/runtime/TokenRewriteStream.java index c48db68f0..3278f7b1c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/TokenRewriteStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/TokenRewriteStream.java @@ -173,10 +173,6 @@ public class TokenRewriteStream extends CommonTokenStream { /** Map String (program name) -> Integer index */ protected Map lastRewriteTokenIndexes = null; - public TokenRewriteStream() { - init(); - } - protected void init() { programs = new HashMap>(); programs.put(DEFAULT_PROGRAM_NAME, new ArrayList(PROGRAM_INIT_SIZE)); diff --git a/tool/test/org/antlr/v4/test/TestPerformance.java b/tool/test/org/antlr/v4/test/TestPerformance.java index 8b37d2e5e..99e05ab5f 100644 --- a/tool/test/org/antlr/v4/test/TestPerformance.java +++ b/tool/test/org/antlr/v4/test/TestPerformance.java @@ -451,8 +451,8 @@ public class TestPerformance extends BaseTest { final Constructor parserCtor = parserClass.getConstructor(TokenStream.class); // construct initial instances of the lexer and parser to deserialize their ATNs - lexerCtor.newInstance(new ANTLRInputStream("")); - parserCtor.newInstance(new CommonTokenStream()); + TokenSource tokenSource = lexerCtor.newInstance(new ANTLRInputStream("")); + parserCtor.newInstance(new CommonTokenStream(tokenSource)); return new ParserFactory() { @SuppressWarnings({"PointlessBooleanExpression"}) From 525b90fe9a03982dd9c78a21b059eab839180918 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Fri, 16 Mar 2012 20:06:30 -0500 Subject: [PATCH 02/12] Add tests for CommonTokenStream handling of consume() at EOF --- .../antlr/v4/test/TestCommonTokenStream.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tool/test/org/antlr/v4/test/TestCommonTokenStream.java b/tool/test/org/antlr/v4/test/TestCommonTokenStream.java index 6701c544f..8853b5fcf 100644 --- a/tool/test/org/antlr/v4/test/TestCommonTokenStream.java +++ b/tool/test/org/antlr/v4/test/TestCommonTokenStream.java @@ -236,4 +236,59 @@ public class TestCommonTokenStream extends BaseTest { assertEquals("=", tokens.LT(-3).getText()); assertEquals("x", tokens.LT(-4).getText()); } + + @Test + public void testSingleEOF() throws Exception { + TokenSource lexer = new TokenSource() { + + @Override + public Token nextToken() { + return new CommonToken(Token.EOF); + } + + @Override + public int getLine() { + return 0; + } + + @Override + public int getCharPositionInLine() { + return 0; + } + + @Override + public CharStream getInputStream() { + return null; + } + + @Override + public String getSourceName() { + return null; + } + + @Override + public void setTokenFactory(TokenFactory factory) { + throw new UnsupportedOperationException("Not supported yet."); + } + }; + + CommonTokenStream tokens = new CommonTokenStream(lexer); + tokens.fill(); + + assertEquals(Token.EOF, tokens.LA(1)); + assertEquals(0, tokens.index()); + assertEquals(1, tokens.size()); + tokens.consume(); + + assertEquals(Token.EOF, tokens.LA(1)); + assertEquals(0, tokens.index()); + assertEquals(1, tokens.size()); + tokens.consume(); + + assertEquals(Token.EOF, tokens.LA(1)); + assertEquals(0, tokens.index()); + assertEquals(1, tokens.size()); + tokens.consume(); + } + } From 95d3b69f1f3ce3016573e1a0265823819f1ec8ce Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 17 Mar 2012 12:34:22 -0500 Subject: [PATCH 03/12] Remove unused field --- .../Java/src/org/antlr/v4/runtime/BufferedTokenStream.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index 422ecf382..4857ba5ff 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -58,9 +58,6 @@ public class BufferedTokenStream implements TokenStream { */ protected List tokens = new ArrayList(100); - /** Track the last mark() call result value for use in rewind(). */ - protected int lastMarker; - /** The index into the tokens list of the current token (next token * to consume). tokens[p] should be LT(1). p=-1 indicates need * to initialize with first token. The ctor doesn't get a token. @@ -95,7 +92,6 @@ public class BufferedTokenStream implements TokenStream { public void reset() { p = 0; - lastMarker = 0; } @Override From 26a7a811df3acb26403b015a510254a8193ad4b2 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 17 Mar 2012 12:47:33 -0500 Subject: [PATCH 04/12] Remove unnecessary duplication of code --- .../antlr/v4/runtime/CommonTokenStream.java | 27 ++++--------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java index 376704069..6c60eea54 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java @@ -62,23 +62,14 @@ public class CommonTokenStream extends BufferedTokenStream { /** Always leave p on an on-channel token. */ @Override public void consume() { - if ( p == -1 ) setup(); - p++; - sync(p); - Token t = tokens.get(p); - while ( t.getType()!=Token.EOF && t.getChannel()!=channel ) { - p++; - sync(p); - t = tokens.get(p); - } + super.consume(); + p = skipOffTokenChannels(p); } @Override public void seek(int index) { super.seek(index); - while (p < index) { - consume(); - } + p = skipOffTokenChannels(p); } @Override @@ -144,16 +135,8 @@ public class CommonTokenStream extends BufferedTokenStream { @Override protected void setup() { - p = 0; - sync(0); - int i = 0; - Token token = tokens.get(i); - while ( token.getType()!=Token.EOF && token.getChannel()!=channel ) { - i++; - sync(i); - token = tokens.get(i); - } - p = i; + super.setup(); + p = skipOffTokenChannels(p); } /** Count EOF just once. */ From 595f92fe5ee58c7d108d54c5e58234b1d6fe41b8 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 17 Mar 2012 12:53:33 -0500 Subject: [PATCH 05/12] Add lazyInit() method to take the place of "if (p==-1) setup();" --- .../antlr/v4/runtime/BufferedTokenStream.java | 25 +++++++++++-------- .../antlr/v4/runtime/CommonTokenStream.java | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index 4857ba5ff..1ca7e12a5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -96,10 +96,7 @@ public class BufferedTokenStream implements TokenStream { @Override public void seek(int index) { - if (p == -1) { - setup(); - } - + lazyInit(); p = index; } @@ -115,7 +112,7 @@ public class BufferedTokenStream implements TokenStream { */ @Override public void consume() { - if ( p == -1 ) setup(); + lazyInit(); p++; sync(p); } @@ -150,7 +147,7 @@ public class BufferedTokenStream implements TokenStream { /** Get all tokens from start..stop inclusively */ public List get(int start, int stop) { if ( start<0 || stop<0 ) return null; - if ( p == -1 ) setup(); + lazyInit(); List subset = new ArrayList(); if ( stop>=tokens.size() ) stop = tokens.size()-1; for (int i = start; i <= stop; i++) { @@ -171,7 +168,7 @@ public class BufferedTokenStream implements TokenStream { @Override public T LT(int k) { - if ( p == -1 ) setup(); + lazyInit(); if ( k==0 ) return null; if ( k < 0 ) return LB(-k); @@ -185,6 +182,12 @@ public class BufferedTokenStream implements TokenStream { return tokens.get(i); } + protected final void lazyInit() { + if (p == -1) { + setup(); + } + } + protected void setup() { sync(0); p = 0; } /** Reset this token stream by setting its token source. */ @@ -205,7 +208,7 @@ public class BufferedTokenStream implements TokenStream { * method looks at both on and off channel tokens. */ public List getTokens(int start, int stop, Set types) { - if ( p == -1 ) setup(); + lazyInit(); if ( stop>=tokens.size() ) stop=tokens.size()-1; if ( start<0 ) start=0; if ( start>stop ) return null; @@ -236,7 +239,7 @@ public class BufferedTokenStream implements TokenStream { /** Grab *all* tokens from stream and return string */ @Override public String toString() { - if ( p == -1 ) setup(); + lazyInit(); fill(); return toString(0, tokens.size()-1); } @@ -244,7 +247,7 @@ public class BufferedTokenStream implements TokenStream { @Override public String toString(int start, int stop) { if ( start<0 || stop<0 ) return ""; - if ( p == -1 ) setup(); + lazyInit(); if ( stop>=tokens.size() ) stop = tokens.size()-1; StringBuilder buf = new StringBuilder(); for (int i = start; i <= stop; i++) { @@ -265,7 +268,7 @@ public class BufferedTokenStream implements TokenStream { /** Get all tokens from lexer until EOF */ public void fill() { - if ( p == -1 ) setup(); + lazyInit(); if ( tokens.get(p).getType()==Token.EOF ) return; int i = p+1; diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java index 6c60eea54..0b7c144ca 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java @@ -97,7 +97,7 @@ public class CommonTokenStream extends BufferedTokenStream { @Override public Token LT(int k) { //System.out.println("enter LT("+k+")"); - if ( p == -1 ) setup(); + lazyInit(); if ( k == 0 ) return null; if ( k < 0 ) return LB(-k); int i = p; From 5404ea4c159974afde31efc7d19af43769205012 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 17 Mar 2012 12:54:46 -0500 Subject: [PATCH 06/12] Standard 0-based indexing in for loop --- runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index 1ca7e12a5..f90b86ce0 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -126,7 +126,7 @@ public class BufferedTokenStream implements TokenStream { /** add n elements to buffer */ protected void fetch(int n) { - for (int i=1; i<=n; i++) { + for (int i = 0; i < n; i++) { T t = (T)tokenSource.nextToken(); if ( t instanceof WritableToken ) { ((WritableToken)t).setTokenIndex(tokens.size()); From e8830ae51a2a14e14ef12b990a365fa2045e19e6 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 17 Mar 2012 15:42:17 -0500 Subject: [PATCH 07/12] Fix handling of consume() at EOF in BufferedTokenStream and derived --- .../antlr/v4/runtime/BufferedTokenStream.java | 53 +++++++++++++++---- .../antlr/v4/runtime/CommonTokenStream.java | 6 ++- tool/test/org/antlr/v4/test/BaseTest.java | 9 +++- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index f90b86ce0..fa997f4df 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -65,6 +65,15 @@ public class BufferedTokenStream implements TokenStream { */ protected int p = -1; + /** + * Set to {@code true} when the EOF token is fetched. Do not continue fetching + * tokens after that point, or multiple EOF tokens could end up in the + * {@link #tokens} array. + * + * @see #fetch + */ + protected boolean fetchedEOF; + public BufferedTokenStream(TokenSource tokenSource) { if (tokenSource == null) { throw new NullPointerException("tokenSource cannot be null"); @@ -106,34 +115,56 @@ public class BufferedTokenStream implements TokenStream { /** Move the input pointer to the next incoming token. The stream * must become active with LT(1) available. consume() simply * moves the input pointer so that LT(1) points at the next - * input symbol. Consume at least one token. - * - * Walk past any token not on the channel the parser is listening to. + * input symbol. Consume at least one token, unless EOF has been reached. */ @Override public void consume() { lazyInit(); - p++; - sync(p); + if (sync(p + 1)) { + p++; + } } - /** Make sure index i in tokens has a token. */ - protected void sync(int i) { + /** Make sure index {@code i} in tokens has a token. + * + * @return {@code true} if a token is located at index {@code i}, otherwise + * {@code false}. + * @see #get(int i) + */ + protected boolean sync(int i) { + assert i >= 0; int n = i - tokens.size() + 1; // how many more elements we need? //System.out.println("sync("+i+") needs "+n); - if ( n > 0 ) fetch(n); + if ( n > 0 ) { + int fetched = fetch(n); + return fetched >= n; + } + + return true; } - /** add n elements to buffer */ - protected void fetch(int n) { + /** Add {@code n} elements to buffer. + * + * @return The actual number of elements added to the buffer. + */ + protected int fetch(int n) { + if (fetchedEOF) { + return 0; + } + for (int i = 0; i < n; i++) { T t = (T)tokenSource.nextToken(); if ( t instanceof WritableToken ) { ((WritableToken)t).setTokenIndex(tokens.size()); } tokens.add(t); - if ( t.getType()==Token.EOF ) break; + if ( t.getType()==Token.EOF ) { + fetchedEOF = true; + return i + 1; + } } + + return n; } @Override diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java index 0b7c144ca..52ef15afa 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java @@ -104,8 +104,10 @@ public class CommonTokenStream extends BufferedTokenStream { int n = 1; // we know tokens[p] is a good one // find k good tokens while ( nrange ) range = i; diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java index 2261b40ea..e8d372113 100644 --- a/tool/test/org/antlr/v4/test/BaseTest.java +++ b/tool/test/org/antlr/v4/test/BaseTest.java @@ -838,12 +838,17 @@ public abstract class BaseTest { public FilteringTokenStream(TokenSource src) { super(src); } Set hide = new HashSet(); @Override - protected void sync(int i) { - super.sync(i); + protected boolean sync(int i) { + if (!super.sync(i)) { + return false; + } + Token t = get(i); if ( hide.contains(t.getType()) ) { ((WritableToken)t).setChannel(Token.HIDDEN_CHANNEL); } + + return true; } public void setTokenTypeChannel(int ttype, int channel) { hide.add(ttype); From c970a2e7ee9532dacbf661af83be955bd087d53a Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 17 Mar 2012 15:42:45 -0500 Subject: [PATCH 08/12] Updated doc comment --- .../Java/src/org/antlr/v4/runtime/CommonTokenStream.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java index 52ef15afa..ef22e9941 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java @@ -59,7 +59,11 @@ public class CommonTokenStream extends BufferedTokenStream { this.channel = channel; } - /** Always leave p on an on-channel token. */ + /** + * {@inheritDoc} + *

+ * Always leave {@code p} on an on-channel token. + */ @Override public void consume() { super.consume(); From 4778383e1d2f2fff1ab88a1da7e1d65f7d3704af Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 17 Mar 2012 15:53:05 -0500 Subject: [PATCH 09/12] Fix BufferedTokenStream.reset() putting the stream in an invalid state when called before lazyInit --- runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index fa997f4df..ac13d43c8 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -100,7 +100,7 @@ public class BufferedTokenStream implements TokenStream { } public void reset() { - p = 0; + seek(0); } @Override From 78eb5b0119fb9563b8d391190090bb5e84844fb6 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 17 Mar 2012 15:59:19 -0500 Subject: [PATCH 10/12] Simplify maintenance of CommonTokenStream by providing a hook in BufferedTokenStream for manipulating the target of seek operations (including a simple consume operation). --- .../antlr/v4/runtime/BufferedTokenStream.java | 26 +++++++++++++++-- .../antlr/v4/runtime/CommonTokenStream.java | 28 ++----------------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index ac13d43c8..fb8081f2a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -106,7 +106,7 @@ public class BufferedTokenStream implements TokenStream { @Override public void seek(int index) { lazyInit(); - p = index; + p = adjustSeekIndex(index); } @Override @@ -121,7 +121,7 @@ public class BufferedTokenStream implements TokenStream { public void consume() { lazyInit(); if (sync(p + 1)) { - p++; + p = adjustSeekIndex(p + 1); } } @@ -213,13 +213,33 @@ public class BufferedTokenStream implements TokenStream { return tokens.get(i); } + /** + * Allowed derived classes to modify the behavior of operations which change + * the current stream position by adjusting the target token index of a seek + * operation. The default implementation simply returns {@code i}. If an + * exception is thrown in this method, the current stream index should not be + * changed. + *

+ * For example, {@link CommonTokenStream} overrides this method to ensure that + * the seek target is always an on-channel token. + * + * @param i The target token index. + * @return The adjusted target token index. + */ + protected int adjustSeekIndex(int i) { + return i; + } + protected final void lazyInit() { if (p == -1) { setup(); } } - protected void setup() { sync(0); p = 0; } + protected void setup() { + sync(0); + p = adjustSeekIndex(0); + } /** Reset this token stream by setting its token source. */ public void setTokenSource(TokenSource tokenSource) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java index ef22e9941..320dfa492 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java @@ -59,27 +59,9 @@ public class CommonTokenStream extends BufferedTokenStream { this.channel = channel; } - /** - * {@inheritDoc} - *

- * Always leave {@code p} on an on-channel token. - */ - @Override - public void consume() { - super.consume(); - p = skipOffTokenChannels(p); - } - - @Override - public void seek(int index) { - super.seek(index); - p = skipOffTokenChannels(p); - } - @Override - public void reset() { - super.reset(); - p = skipOffTokenChannels(p); + protected int adjustSeekIndex(int i) { + return skipOffTokenChannels(i); } @Override @@ -139,12 +121,6 @@ public class CommonTokenStream extends BufferedTokenStream { return i; } - @Override - protected void setup() { - super.setup(); - p = skipOffTokenChannels(p); - } - /** Count EOF just once. */ public int getNumberOfOnChannelTokens() { int n = 0; From 24d7bed5ec975daee396b014e868b44f6459df89 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 17 Mar 2012 16:00:24 -0500 Subject: [PATCH 11/12] Don't reset the channel of a CommonTokenStream during a call to setTokenSource --- .../Java/src/org/antlr/v4/runtime/CommonTokenStream.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java index 320dfa492..abd160d44 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java @@ -132,11 +132,4 @@ public class CommonTokenStream extends BufferedTokenStream { } return n; } - - /** Reset this token stream by setting its token source. */ - @Override - public void setTokenSource(TokenSource tokenSource) { - super.setTokenSource(tokenSource); - channel = Token.DEFAULT_CHANNEL; - } } From dd0944b9c469608bb13d16620ff79a78728d1c1f Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 17 Mar 2012 17:25:58 -0500 Subject: [PATCH 12/12] Simplify and improve performance of BufferedTokenStream.fill --- .../org/antlr/v4/runtime/BufferedTokenStream.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index fb8081f2a..4e3b19154 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -320,13 +320,12 @@ public class BufferedTokenStream implements TokenStream { /** Get all tokens from lexer until EOF */ public void fill() { lazyInit(); - if ( tokens.get(p).getType()==Token.EOF ) return; - - int i = p+1; - sync(i); - while ( tokens.get(i).getType()!=Token.EOF ) { - i++; - sync(i); - } + final int blockSize = 1000; + while (true) { + int fetched = fetch(blockSize); + if (fetched < blockSize) { + return; + } + } } }