diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index 0bf2dfc2d..2ac2b5c0b 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -40,12 +40,12 @@ import java.util.*; * * You can't use this stream if you pass whitespace or other off-channel * tokens to the parser. The stream can't ignore off-channel tokens. - * (UnbufferedTokenStream is the same way.) + * (UnbufferedTokenStream is the same way.) Use CommonTokenStream. * * This is not a subclass of UnbufferedTokenStream because I don't want * to confuse small moving window of tokens it uses for the full buffer. */ -public class BufferedTokenStream implements TokenStream { +public class BufferedTokenStream implements TokenStream { protected TokenSource tokenSource; /** Record every single token pulled from the source so we can reproduce @@ -53,7 +53,7 @@ public class BufferedTokenStream implements TokenStream { * as its moving window moves through the input. This list captures * everything so we can access complete input text. */ - protected List tokens = new ArrayList(100); + protected List tokens = new ArrayList(100); /** Track the last mark() call result value for use in rewind(). */ protected int lastMarker; @@ -65,9 +65,7 @@ public class BufferedTokenStream implements TokenStream { */ protected int p = -1; -// protected int range = -1; // how deep have we gone? - - public BufferedTokenStream() {;} + public BufferedTokenStream() { } public BufferedTokenStream(TokenSource tokenSource) { this.tokenSource = tokenSource; @@ -136,16 +134,17 @@ public class BufferedTokenStream implements TokenStream { /** add n elements to buffer */ protected void fetch(int n) { for (int i=1; i<=n; i++) { - WritableToken t = (WritableToken)tokenSource.nextToken(); -// System.out.println("adding "+t+" at index "+tokens.size()); - t.setTokenIndex(tokens.size()); + T t = (T)tokenSource.nextToken(); + if ( t instanceof WritableToken ) { + ((WritableToken)t).setTokenIndex(tokens.size()); + } tokens.add(t); if ( t.getType()==Token.EOF ) break; } } @Override - public Token get(int i) { + public T get(int i) { if ( i < 0 || i >= tokens.size() ) { throw new NoSuchElementException("token index "+i+" out of range 0.."+(tokens.size()-1)); } @@ -153,13 +152,13 @@ public class BufferedTokenStream implements TokenStream { } /** Get all tokens from start..stop inclusively */ - public List get(int start, int stop) { + public List get(int start, int stop) { if ( start<0 || stop<0 ) return null; if ( p == -1 ) setup(); - List subset = new ArrayList(); + List subset = new ArrayList(); if ( stop>=tokens.size() ) stop = tokens.size()-1; for (int i = start; i <= stop; i++) { - Token t = tokens.get(i); + T t = tokens.get(i); if ( t.getType()==Token.EOF ) break; subset.add(t); } @@ -169,13 +168,13 @@ public class BufferedTokenStream implements TokenStream { @Override public int LA(int i) { return LT(i).getType(); } - protected Token LB(int k) { + protected T LB(int k) { if ( (p-k)<0 ) return null; return tokens.get(p-k); } @Override - public Token LT(int k) { + public T LT(int k) { if ( p == -1 ) setup(); if ( k==0 ) return null; if ( k < 0 ) return LB(-k); @@ -199,9 +198,9 @@ public class BufferedTokenStream implements TokenStream { p = -1; } - public List getTokens() { return tokens; } + public List getTokens() { return tokens; } - public List getTokens(int start, int stop) { + public List getTokens(int start, int stop) { return getTokens(start, stop, null); } @@ -209,16 +208,16 @@ public class BufferedTokenStream implements TokenStream { * the token type BitSet. Return null if no tokens were found. This * method looks at both on and off channel tokens. */ - public List getTokens(int start, int stop, Set types) { + public List getTokens(int start, int stop, Set types) { if ( p == -1 ) setup(); if ( stop>=tokens.size() ) stop=tokens.size()-1; if ( start<0 ) start=0; if ( start>stop ) return null; - // list = tokens[start:stop]:{Token t, t.getType() in types} - List filteredTokens = new ArrayList(); + // list = tokens[start:stop]:{T t, t.getType() in types} + List filteredTokens = new ArrayList(); for (int i=start; i<=stop; i++) { - Token t = tokens.get(i); + T t = tokens.get(i); if ( types==null || types.contains(t.getType()) ) { filteredTokens.add(t); } @@ -229,7 +228,7 @@ public class BufferedTokenStream implements TokenStream { return filteredTokens; } - public List getTokens(int start, int stop, int ttype) { + public List getTokens(int start, int stop, int ttype) { HashSet s = new HashSet(ttype); s.add(ttype); return getTokens(start,stop, s); @@ -250,9 +249,9 @@ public class BufferedTokenStream implements TokenStream { if ( start<0 || stop<0 ) return ""; if ( p == -1 ) setup(); if ( stop>=tokens.size() ) stop = tokens.size()-1; - StringBuffer buf = new StringBuffer(); + StringBuilder buf = new StringBuilder(); for (int i = start; i <= stop; i++) { - Token t = tokens.get(i); + T t = tokens.get(i); if ( t.getType()==Token.EOF ) break; buf.append(t.getText()); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java index c99696a70..2b341fbf2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java @@ -43,10 +43,10 @@ package org.antlr.v4.runtime; * whitespace and comments etc. to the parser on a hidden channel (i.e., * you set $channel instead of calling skip() in lexer rules.) * - * @see org.antlr.runtime.UnbufferedTokenStream - * @see org.antlr.runtime.BufferedTokenStream + * @see UnbufferedTokenStream + * @see BufferedTokenStream */ -public class CommonTokenStream extends BufferedTokenStream { +public class CommonTokenStream extends BufferedTokenStream { /** Skip tokens on any channel but this one; this is how we skip whitespace... */ protected int channel = Token.DEFAULT_CHANNEL; @@ -74,7 +74,7 @@ public class CommonTokenStream extends BufferedTokenStream { } @Override - protected Token LB(int k) { + protected CommonToken LB(int k) { if ( k==0 || (p-k)<0 ) return null; int i = p; @@ -90,7 +90,7 @@ public class CommonTokenStream extends BufferedTokenStream { } @Override - public Token LT(int k) { + public CommonToken LT(int k) { //System.out.println("enter LT("+k+")"); if ( p == -1 ) setup(); if ( k == 0 ) return null; diff --git a/runtime/Java/src/org/antlr/v4/runtime/ANTLRUnbufferedInputStream.java b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java similarity index 94% rename from runtime/Java/src/org/antlr/v4/runtime/ANTLRUnbufferedInputStream.java rename to runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java index c15e7fe9e..128dca261 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ANTLRUnbufferedInputStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java @@ -34,7 +34,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; -public class ANTLRUnbufferedInputStream implements CharStream { +public class UnbufferedCharStream implements CharStream { /** A buffer of the data being scanned */ protected char[] data; @@ -59,20 +59,20 @@ public class ANTLRUnbufferedInputStream implements CharStream { /** What is name or source of this char stream? */ public String name; - public ANTLRUnbufferedInputStream(InputStream input) { + public UnbufferedCharStream(InputStream input) { this(input, 256); } - public ANTLRUnbufferedInputStream(Reader input) { + public UnbufferedCharStream(Reader input) { this(input, 256); } - public ANTLRUnbufferedInputStream(InputStream input, int bufferSize) { + public UnbufferedCharStream(InputStream input, int bufferSize) { this.input = new InputStreamReader(input); data = new char[bufferSize]; } - public ANTLRUnbufferedInputStream(Reader input, int bufferSize) { + public UnbufferedCharStream(Reader input, int bufferSize) { this.input = input; data = new char[bufferSize]; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java new file mode 100644 index 000000000..92d20ac5a --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java @@ -0,0 +1,100 @@ +/* + [The "BSD license"] + Copyright (c) 2011 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.runtime; + +import org.antlr.v4.runtime.misc.LookaheadStream; + +/** A token stream that pulls tokens from the source on-demand and + * without tracking a complete buffer of the tokens. This stream buffers + * the minimum number of tokens possible. + * + * You can't use this stream if you pass whitespace or other off-channel + * tokens to the parser. The stream can't ignore off-channel tokens. + * + * You can only look backwards 1 token: LT(-1). + * + * Use this when you need to read from a socket or other infinite stream. + * + * @see BufferedTokenStream + * @see CommonTokenStream + */ +public class UnbufferedTokenStream + extends LookaheadStream + implements TokenStream +{ + protected TokenSource tokenSource; + protected int tokenIndex = 0; // simple counter to set token index in tokens + + /** Skip tokens on any channel but this one; this is how we skip whitespace... */ + protected int channel = Token.DEFAULT_CHANNEL; + + public UnbufferedTokenStream(TokenSource tokenSource) { + this.tokenSource = tokenSource; + } + + @Override + public T nextElement() { + T t = (T)tokenSource.nextToken(); + if ( t instanceof WritableToken ) { + ((WritableToken)t).setTokenIndex(tokenIndex); + } + tokenIndex++; + return t; + } + + @Override + public boolean isEOF(Token o) { + return false; + } + + @Override + public TokenSource getTokenSource() { return tokenSource; } + + @Override + public String toString(int start, int stop) { + throw new UnsupportedOperationException("unbuffered stream can't give strings"); + } + + @Override + public String toString(Token start, Token stop) { + throw new UnsupportedOperationException("unbuffered stream can't give strings"); + } + + @Override + public int LA(int i) { return LT(i).getType(); } + + @Override + public T get(int i) { + throw new UnsupportedOperationException("Absolute token indexes are meaningless in an unbuffered stream"); + } + + @Override + public String getSourceName() { return tokenSource.getSourceName(); } +} diff --git a/tool/test/org/antlr/v4/test/TestUnbufferedInputStream.java b/tool/test/org/antlr/v4/test/TestUnbufferedInputStream.java index 406e77e55..d63830b46 100644 --- a/tool/test/org/antlr/v4/test/TestUnbufferedInputStream.java +++ b/tool/test/org/antlr/v4/test/TestUnbufferedInputStream.java @@ -29,15 +29,15 @@ package org.antlr.v4.test; -import org.antlr.v4.runtime.ANTLRUnbufferedInputStream; import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.UnbufferedCharStream; import org.junit.Test; import java.io.StringReader; public class TestUnbufferedInputStream extends BaseTest { @Test public void testNoChar() throws Exception { - CharStream input = new ANTLRUnbufferedInputStream( + CharStream input = new UnbufferedCharStream( new StringReader("") ); assertEquals(CharStream.EOF, input.LA(1)); @@ -48,7 +48,7 @@ public class TestUnbufferedInputStream extends BaseTest { } @Test public void test1Char() throws Exception { - CharStream input = new ANTLRUnbufferedInputStream( + CharStream input = new UnbufferedCharStream( new StringReader("x") ); assertEquals('x', input.LA(1)); @@ -57,7 +57,7 @@ public class TestUnbufferedInputStream extends BaseTest { } @Test public void test2Char() throws Exception { - CharStream input = new ANTLRUnbufferedInputStream( + CharStream input = new UnbufferedCharStream( new StringReader("xy") ); assertEquals('x', input.LA(1)); @@ -68,7 +68,7 @@ public class TestUnbufferedInputStream extends BaseTest { } @Test public void test2CharAhead() throws Exception { - CharStream input = new ANTLRUnbufferedInputStream( + CharStream input = new UnbufferedCharStream( new StringReader("xy") ); assertEquals('x', input.LA(1)); @@ -77,7 +77,7 @@ public class TestUnbufferedInputStream extends BaseTest { } @Test public void testBufferExpand() throws Exception { - CharStream input = new ANTLRUnbufferedInputStream( + CharStream input = new UnbufferedCharStream( new StringReader("01234"), 2 // buff size 2 ); @@ -90,7 +90,7 @@ public class TestUnbufferedInputStream extends BaseTest { } @Test public void testBufferWrapSize1() throws Exception { - CharStream input = new ANTLRUnbufferedInputStream( + CharStream input = new UnbufferedCharStream( new StringReader("01234"), 1 // buff size 1 ); @@ -108,7 +108,7 @@ public class TestUnbufferedInputStream extends BaseTest { } @Test public void testBufferWrapSize2() throws Exception { - CharStream input = new ANTLRUnbufferedInputStream( + CharStream input = new UnbufferedCharStream( new StringReader("01234"), 2 // buff size 2 ); @@ -126,7 +126,7 @@ public class TestUnbufferedInputStream extends BaseTest { } @Test public void test1Mark() throws Exception { - CharStream input = new ANTLRUnbufferedInputStream( + CharStream input = new UnbufferedCharStream( new StringReader("xyz") ); int m = input.mark(); @@ -138,7 +138,7 @@ public class TestUnbufferedInputStream extends BaseTest { } @Test public void test2Mark() throws Exception { - CharStream input = new ANTLRUnbufferedInputStream( + CharStream input = new UnbufferedCharStream( new StringReader("xyz"), 2 );