diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index e189205f7..766ee9ba8 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -129,7 +129,7 @@ public class BufferedTokenStream implements TokenStream { /** add n elements to buffer */ protected void fetch(int n) { for (int i=1; i<=n; i++) { - Token t = tokenSource.nextToken(); + WritableToken t = (WritableToken)tokenSource.nextToken(); // System.out.println("adding "+t+" at index "+tokens.size()); t.setTokenIndex(tokens.size()); tokens.add(t); diff --git a/runtime/Java/src/org/antlr/v4/runtime/CharStream.java b/runtime/Java/src/org/antlr/v4/runtime/CharStream.java index 079fd03b0..15822d409 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CharStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CharStream.java @@ -37,22 +37,4 @@ public interface CharStream extends IntStream { * use this on streams that don't support it. */ public String substring(int start, int stop); - - /** Get the ith character of lookahead. This is the same usually as - * LA(i). This will be used for labels in the generated - * lexer code. I'd prefer to return a char here type-wise, but it's - * probably better to be 32-bit clean and be consistent with LA. - */ - public int LT(int i); - - /** ANTLR tracks the line information automatically */ - int getLine(); - - /** Because this stream can rewind, we need to be able to reset the line */ - void setLine(int line); - - void setCharPositionInLine(int pos); - - /** The index of the character relative to the beginning of the line 0..n-1 */ - int getCharPositionInLine(); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java b/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java index d7e5480de..197fe83a4 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java @@ -30,17 +30,19 @@ package org.antlr.v4.runtime; import java.io.Serializable; -public class CommonToken implements Token, Serializable { +public class CommonToken implements WritableToken, Serializable { protected int type; protected int line; protected int charPositionInLine = -1; // set to invalid position protected int channel=DEFAULT_CHANNEL; - protected transient CharStream input; + protected TokenSource source; + // TODO: rm protected transient CharStream input; /** We need to be able to change the text once in a while. If * this is non-null, then getText should return this. Note that * start/stop are not affected by changing this. */ + // TODO: can store these in map in token stream rather than as field here protected String text; /** What token number is this from 0..n-1 tokens; < 0 implies invalid index */ @@ -56,14 +58,14 @@ public class CommonToken implements Token, Serializable { this.type = type; } - public CommonToken(CharStream input, int type, int channel, int start, int stop) { - this.input = input; + public CommonToken(TokenSource source, int type, int channel, int start, int stop) { + this.source = source; this.type = type; this.channel = channel; this.start = start; this.stop = stop; - this.line = input.getLine(); - this.charPositionInLine = input.getCharPositionInLine(); + this.line = source.getLine(); + this.charPositionInLine = source.getCharPositionInLine(); } public CommonToken(int type, String text) { @@ -79,7 +81,7 @@ public class CommonToken implements Token, Serializable { index = oldToken.getTokenIndex(); charPositionInLine = oldToken.getCharPositionInLine(); channel = oldToken.getChannel(); - input = oldToken.getInputStream(); + source = oldToken.getTokenSource(); if ( oldToken instanceof CommonToken ) { start = ((CommonToken)oldToken).start; stop = ((CommonToken)oldToken).stop; @@ -98,6 +100,7 @@ public class CommonToken implements Token, Serializable { if ( text!=null ) { return text; } + CharStream input = getTokenSource().getInputStream(); if ( input==null ) { return null; } @@ -167,12 +170,12 @@ public class CommonToken implements Token, Serializable { this.index = index; } - public CharStream getInputStream() { - return input; + public TokenSource getTokenSource() { + return source; } - public void setInputStream(CharStream input) { - this.input = input; + public CharStream getInputStream() { + return source.getInputStream(); } public String toString() { diff --git a/runtime/Java/src/org/antlr/v4/runtime/IntStream.java b/runtime/Java/src/org/antlr/v4/runtime/IntStream.java index 319257e65..a2e038977 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/IntStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/IntStream.java @@ -31,9 +31,8 @@ package org.antlr.v4.runtime; /** A simple stream of integers used when all I care about is the char * or token type sequence (such as interpretation). * - * TODO: Notes from Oli: index, seek, LA, LT, consume, getSourceName needed * Do like Java IO and have new BufferedStream(new TokenStream) rather than - * using inheritance. line/col go into lexer + * using inheritance? */ public interface IntStream { void consume(); @@ -46,13 +45,17 @@ public interface IntStream { int LA(int i); /** Tell the stream to start buffering if it hasn't already. Return - * current input position, index(). seek(mark()) should not - * affect the input cursor. + * a marker, usually current input position, index(). + * Calling release(mark()) should not affect the input cursor. + * Can seek to any index between where we were when mark() was called + * and current index() until we release this marker. */ int mark(); - /** Reset the stream so that next call to index() would return index arg. */ - void rewind(int index); + /** Release requirement that stream holds tokens from marked location + * to current index(). + */ + void release(int marker); /** Return the current input symbol index 0..n where n indicates the * last symbol has been read. The index is the symbol about to be @@ -61,21 +64,16 @@ public interface IntStream { int index(); /** Set the input cursor to the position indicated by index. This is - * normally used to seek ahead in the input stream. No buffering is - * required to do this unless you know your stream will use seek to - * move backwards such as when backtracking. - * - * This is different from rewind in its multi-directional - * requirement and in that its argument is strictly an input cursor (index). + * normally used to rewind the input stream but can move forward as well. + * It's up to the stream implementation to make sure that tokens are + * buffered as necessary to make seek land on a valid token. + * Or, they should avoid moving the input cursor. * * For char streams, seeking forward must update the stream state such * as line number. For seeking backwards, you will be presumably * backtracking using the mark/rewind mechanism that restores state and * so this method does not need to update state when seeking backwards. * - * Currently, this method is only used for efficient backtracking using - * memoization, but in the future it may be used for incremental parsing. - * * The index is 0..n-1. A seek to position i means that LA(1) will * return the ith symbol. So, seeking to 0 means LA(1) will return the * first element in the stream. diff --git a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java index 47df0be1e..a3850f907 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java @@ -115,13 +115,13 @@ public abstract class Lexer extends Recognizer token = null; channel = Token.DEFAULT_CHANNEL; tokenStartCharIndex = input.index(); - tokenStartCharPositionInLine = input.getCharPositionInLine(); - tokenStartLine = input.getLine(); + tokenStartCharPositionInLine = _interp.getCharPositionInLine(); + tokenStartLine = _interp.getLine(); text = null; do { type = Token.INVALID_TYPE; if ( input.LA(1)==CharStream.EOF ) { - Token eof = new CommonToken(input,Token.EOF, + WritableToken eof = new CommonToken(this,Token.EOF, Token.DEFAULT_CHANNEL, input.index(),input.index()); eof.setLine(getLine()); @@ -182,14 +182,14 @@ public abstract class Lexer extends Recognizer this.input = input; } - public CharStream getCharStream() { - return ((CharStream)input); - } - public String getSourceName() { return input.getSourceName(); } + public CharStream getInputStream() { + return input; + } + /** Currently does not support multiple emits per nextToken invocation * for efficiency reasons. Subclass and override this method and * nextToken (to push tokens into a list and pull from that list rather @@ -210,9 +210,9 @@ public abstract class Lexer extends Recognizer * Parser or TreeParser.getMissingSymbol(). */ public Token emit() { - Token t = new CommonToken(((CharStream)input), type, - channel, tokenStartCharIndex, - getCharIndex()-1); + WritableToken t = new CommonToken(this, type, + channel, tokenStartCharIndex, + getCharIndex()-1); t.setLine(tokenStartLine); t.setText(text); t.setCharPositionInLine(tokenStartCharPositionInLine); @@ -221,11 +221,11 @@ public abstract class Lexer extends Recognizer } public int getLine() { - return ((CharStream)input).getLine(); + return _interp.getLine(); } public int getCharPositionInLine() { - return ((CharStream)input).getCharPositionInLine(); + return _interp.getCharPositionInLine(); } /** What is the index of the current character of lookahead? */ @@ -338,6 +338,7 @@ public abstract class Lexer extends Recognizer public void recover(RecognitionException re) { //System.out.println("consuming char "+(char)input.LA(1)+" during recovery"); //re.printStackTrace(); + // TODO: Do we lose character or line position information? input.consume(); } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/LexerRecognitionExeption.java b/runtime/Java/src/org/antlr/v4/runtime/LexerRecognitionExeption.java index 3af297333..04890cff1 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/LexerRecognitionExeption.java +++ b/runtime/Java/src/org/antlr/v4/runtime/LexerRecognitionExeption.java @@ -55,8 +55,8 @@ public class LexerRecognitionExeption extends RuntimeException { this.lexer = lexer; this.index = input.index(); this.c = input.LA(1); - this.line = input.getLine(); - this.charPositionInLine = input.getCharPositionInLine(); + this.line = lexer.getLine(); + this.charPositionInLine = lexer.getCharPositionInLine(); } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/Parser.java b/runtime/Java/src/org/antlr/v4/runtime/Parser.java index ce55c93c1..debbeb13d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Parser.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Parser.java @@ -67,7 +67,7 @@ public class Parser extends BaseRecognizer { t.line = current.getLine(); t.charPositionInLine = current.getCharPositionInLine(); t.channel = Token.DEFAULT_CHANNEL; - t.input = current.getInputStream(); + t.source = current.getTokenSource(); return t; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java b/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java index 4cf03a722..93d2f349c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java @@ -121,7 +121,7 @@ public class RecognitionException extends RuntimeException { if ( recognizer!=null ) expecting = recognizer._interp.atn.nextTokens(ctx); this.index = input.index(); - if ( input instanceof TokenStream) { + if ( input instanceof TokenStream ) { this.token = ((TokenStream)input).LT(1); this.line = token.getLine(); this.charPositionInLine = token.getCharPositionInLine(); @@ -129,11 +129,6 @@ public class RecognitionException extends RuntimeException { if ( input instanceof TreeNodeStream) { //extractInformationFromTreeNodeStream(input); } - else if ( input instanceof CharStream) { - this.c = input.LA(1); - this.line = ((CharStream)input).getLine(); - this.charPositionInLine = ((CharStream)input).getCharPositionInLine(); - } else { this.c = input.LA(1); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/Token.java b/runtime/Java/src/org/antlr/v4/runtime/Token.java index 1a528bdf3..d28abdc7d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Token.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Token.java @@ -29,6 +29,10 @@ package org.antlr.v4.runtime; +/** A token has properties: text, type, line, character position in the line + * (so we can ignore tabs), token channel, index, and source from which + * we obtained this token. + */ public interface Token { public static final int INVALID_TYPE = 0; public static final Token INVALID_TOKEN = new CommonToken(INVALID_TYPE); @@ -36,6 +40,7 @@ public interface Token { /** imaginary tree navigation type; traverse "get child" link */ public static final int DOWN = 1; + /** imaginary tree navigation type; finish with a child list */ public static final int UP = 2; @@ -55,32 +60,34 @@ public interface Token { public static final int HIDDEN_CHANNEL = 99; /** Get the text of the token */ - public String getText(); - public void setText(String text); + String getText(); - public int getType(); - public void setType(int ttype); - /** The line number on which this token was matched; line=1..n */ - public int getLine(); - public void setLine(int line); + /** Get the token type of the token */ + int getType(); - /** The index of the first character relative to the beginning of the line 0..n-1 */ - public int getCharPositionInLine(); - public void setCharPositionInLine(int pos); + /** The line number on which the 1st character of this token was matched, + * line=1..n + */ + int getLine(); - public int getChannel(); - public void setChannel(int channel); + /** The index of the first character of this token relative to the + * beginning of the line at which it occurs, 0..n-1 + */ + int getCharPositionInLine(); + + /** Return the channel this token. Each token can arrive at the parser + * on a different channel, but the parser only "tunes" to a single channel. + * The parser ignores everything not on DEFAULT_CHANNEL. + */ + int getChannel(); /** An index from 0..n-1 of the token object in the input stream. * This must be valid in order to use the ANTLRWorks debugger. */ - public int getTokenIndex(); - public void setTokenIndex(int index); + int getTokenIndex(); - /** From what character stream was this token created? You don't have to - * implement but it's nice to know where a Token comes from if you have - * include files etc... on the input. + /** Where does this token come from? You can get the + * character input stream from the token source. */ - public CharStream getInputStream(); - public void setInputStream(CharStream input); + TokenSource getTokenSource(); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java b/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java index c4d28b0be..b1117de67 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java +++ b/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java @@ -48,6 +48,16 @@ public interface TokenSource { */ public Token nextToken(); + public int getLine(); + + public int getCharPositionInLine(); + + /** From what character stream was this token created? You don't have to + * implement but it's nice to know where a Token comes from if you have + * include files etc... on the input. + */ + public CharStream getInputStream(); + /** Where are you getting tokens from? normally the implication will simply * ask lexers input stream. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index 27141bb85..c1ea3ecc8 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -41,6 +41,12 @@ public class LexerATNSimulator extends ATNSimulator { protected Lexer recog; + /** line number 1..n within the input */ + protected int line = 1; + + /** The index of the character relative to the beginning of the line 0..n-1 */ + protected int charPositionInLine = 0; + protected DFA[] dfa; protected int mode = Lexer.DEFAULT_MODE; @@ -87,6 +93,9 @@ public class LexerATNSimulator extends ATNSimulator { (char)input.LA(1)); //System.out.println("DFA start of execDFA: "+dfa[mode].toLexerString()); int prevAcceptMarker = -1; + int prevAcceptIndex = -1; + int prevAcceptLine = 0; + int prevAcceptCharPos = -1; DFAState prevAcceptState = null; DFAState s = s0; int startIndex = input.index(); @@ -99,7 +108,10 @@ public class LexerATNSimulator extends ATNSimulator { if ( dfa_debug ) System.out.println("accept; predict "+s.prediction+ " in state "+s.stateNumber); prevAcceptState = s; - prevAcceptMarker = input.index(); + prevAcceptMarker = input.mark(); + prevAcceptIndex = input.index(); + prevAcceptLine = line; + prevAcceptCharPos = charPositionInLine; // keep going unless we're at EOF; check if something else could match if ( t==CharStream.EOF ) break; } @@ -134,7 +146,7 @@ public class LexerATNSimulator extends ATNSimulator { DFAState target = s.edges[t]; if ( target == ERROR ) break; s = target; - input.consume(); + consume(input, t); t = input.LA(1); } if ( prevAcceptState==null ) { @@ -150,7 +162,10 @@ public class LexerATNSimulator extends ATNSimulator { } if ( actionIndex>=0 ) recog.action(null, prevAcceptState.ruleIndex, actionIndex); } - input.seek(prevAcceptMarker); + input.release(prevAcceptMarker); + input.seek(prevAcceptIndex); + line = prevAcceptLine; + charPositionInLine = prevAcceptCharPos; return prevAcceptState.prediction; } @@ -223,7 +238,7 @@ public class LexerATNSimulator extends ATNSimulator { break; } - input.consume(); + consume(input, t); addDFAEdge(closure, t, reach); t = input.LA(1); @@ -464,6 +479,28 @@ public class LexerATNSimulator extends ATNSimulator { return dfa[mode]; } + public int getLine() { + return line; + } + + public int getCharPositionInLine() { + return charPositionInLine; + } + + public void consume(CharStream input, int curChar) { + //System.out.println("prev p="+p+", c="+(char)data[p]); + charPositionInLine++; + if ( curChar=='\n' ) { + /* + System.out.println("newline char found on line: "+line+ + "@ pos="+charPositionInLine); + */ + line++; + charPositionInLine=0; + } + input.consume(); + } + public String getTokenName(int t) { if ( t==-1 ) return "EOF"; //if ( atn.g!=null ) return atn.g.getTokenDisplayName(t); diff --git a/runtime/Java/src/org/antlr/v4/runtime/debug/BlankDebugEventListener.java b/runtime/Java/src/org/antlr/v4/runtime/debug/BlankDebugEventListener.java index 11fa25fb0..5f9c77427 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/debug/BlankDebugEventListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/debug/BlankDebugEventListener.java @@ -28,8 +28,7 @@ package org.antlr.v4.runtime.debug; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.*; /** A blank listener that does nothing; useful for real classes so * they don't have to have lots of blank methods and are less @@ -48,7 +47,7 @@ public class BlankDebugEventListener implements DebugEventListener { public void consumeHiddenToken(Token token) {} public void LT(int i, Token t) {} public void mark(int i) {} - public void rewind(int i) {} + public void release(int i) {} public void seek(int i) {} public void recognitionException(RecognitionException e) {} public void beginResync() {} diff --git a/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventHub.java b/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventHub.java index 780cd98ad..f007c0cdf 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventHub.java +++ b/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventHub.java @@ -27,11 +27,9 @@ */ package org.antlr.v4.runtime.debug; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.*; -import java.util.ArrayList; -import java.util.List; +import java.util.*; /** Broadcast debug events to multiple listeners. Lets you debug and still * use the event mechanism to build parse trees etc... Not thread-safe. @@ -158,10 +156,10 @@ public class DebugEventHub implements DebugEventListener { } } - public void rewind(int index) { + public void release(int marker) { for (int i = 0; i < listeners.size(); i++) { DebugEventListener listener = (DebugEventListener)listeners.get(i); - listener.rewind(index); + listener.release(marker); } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventListener.java b/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventListener.java index fad1001a5..20fa2c6bb 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventListener.java @@ -28,8 +28,7 @@ package org.antlr.v4.runtime.debug; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.*; /** All debugging events that a recognizer can trigger. * @@ -91,7 +90,7 @@ public interface DebugEventListener { /** After an arbitrarily long lookahead, this informs the * debugger that stream should be rewound to the position index. */ - public void rewind(int index); + public void release(int marker); /** Set the input cursor to the position indicated by index. */ public void seek(int index); diff --git a/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventRepeater.java b/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventRepeater.java index d7b084152..bd4864669 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventRepeater.java +++ b/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventRepeater.java @@ -27,8 +27,7 @@ */ package org.antlr.v4.runtime.debug; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.*; /** A simple event repeater (proxy) that delegates all functionality to the * listener sent into the ctor. Useful if you want to listen in on a few @@ -58,7 +57,7 @@ public class DebugEventRepeater implements DebugEventListener { public void consumeHiddenToken(Token token) { listener.consumeHiddenToken(token); } public void LT(int i, Token t) { listener.LT(i, t); } public void mark(int i) { listener.mark(i); } - public void rewind(int i) { listener.rewind(i); } + public void release(int i) { listener.release(i); } public void seek(int i) { listener.seek(i); } public void recognitionException(RecognitionException e) { listener.recognitionException(e); } public void beginResync() { listener.beginResync(); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventSocketProxy.java b/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventSocketProxy.java index 1fdec5a3a..a6d7611da 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventSocketProxy.java +++ b/runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventSocketProxy.java @@ -27,14 +27,11 @@ */ package org.antlr.v4.runtime.debug; -import org.antlr.v4.runtime.BaseRecognizer; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.*; import org.antlr.v4.runtime.tree.TreeAdaptor; import java.io.*; -import java.net.ServerSocket; -import java.net.Socket; +import java.net.*; /** A proxy debug event listener that forwards events over a socket to * a debugger (or any other listener) using a simple text-based protocol; @@ -165,7 +162,7 @@ public class DebugEventSocketProxy extends BlankDebugEventListener { transmit("mark\t"+i); } - public void rewind(int i) { + public void release(int i) { transmit("rewind\t"+i); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/debug/DebugTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/debug/DebugTokenStream.java index acd6714b8..5e62ef6bc 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/debug/DebugTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/debug/DebugTokenStream.java @@ -27,9 +27,7 @@ */ package org.antlr.v4.runtime.debug; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenSource; -import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.*; public class DebugTokenStream implements TokenStream { protected DebugEventListener dbg; @@ -111,9 +109,9 @@ public class DebugTokenStream implements TokenStream { return input.range(); } - public void rewind(int index) { - dbg.rewind(index); - input.seek(index); + public void release(int index) { + dbg.release(index); + input.release(index); } public void seek(int index) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/debug/DebugTreeNodeStream.java b/runtime/Java/src/org/antlr/v4/runtime/debug/DebugTreeNodeStream.java index 3dc56cfc8..c63ba6416 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/debug/DebugTreeNodeStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/debug/DebugTreeNodeStream.java @@ -28,8 +28,7 @@ package org.antlr.v4.runtime.debug; import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.tree.TreeAdaptor; -import org.antlr.v4.runtime.tree.TreeNodeStream; +import org.antlr.v4.runtime.tree.*; /** Debug any tree node stream. The constructor accepts the stream * and a debug listener. As node stream calls come in, debug events @@ -95,9 +94,9 @@ public class DebugTreeNodeStream implements TreeNodeStream { return lastMarker; } - public void rewind(int marker) { - dbg.rewind(marker); - input.rewind(marker); + public void release(int marker) { + dbg.release(marker); + input.release(marker); } public int index() { diff --git a/runtime/Java/src/org/antlr/v4/runtime/debug/RemoteDebugEventSocketListener.java b/runtime/Java/src/org/antlr/v4/runtime/debug/RemoteDebugEventSocketListener.java index 753909864..bd9338a3f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/debug/RemoteDebugEventSocketListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/debug/RemoteDebugEventSocketListener.java @@ -27,15 +27,11 @@ */ package org.antlr.v4.runtime.debug; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.tree.BaseTree; -import org.antlr.v4.runtime.tree.Tree; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.tree.*; import java.io.*; -import java.net.ConnectException; -import java.net.Socket; +import java.net.*; import java.util.StringTokenizer; public class RemoteDebugEventSocketListener implements Runnable { @@ -110,11 +106,9 @@ public class RemoteDebugEventSocketListener implements Runnable { public void setTokenIndex(int index) { this.index = index; } - public CharStream getInputStream() { + public TokenSource getTokenSource() { return null; } - public void setInputStream(CharStream input) { - } public String toString() { String channelStr = ""; if ( channel!= Token.DEFAULT_CHANNEL ) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java b/runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java index 93a5f80a8..53cb3d0c1 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java @@ -195,19 +195,19 @@ public abstract class BaseTreeAdaptor implements TreeAdaptor { } public Object create(int tokenType, Token fromToken) { - fromToken = createToken(fromToken); + WritableToken tok = createToken(fromToken); //((ClassicToken)fromToken).setType(tokenType); - fromToken.setType(tokenType); - Tree t = (Tree)create(fromToken); + tok.setType(tokenType); + Tree t = (Tree)create(tok); return t; } public Object create(int tokenType, Token fromToken, String text) { if (fromToken == null) return create(tokenType, text); - fromToken = createToken(fromToken); - fromToken.setType(tokenType); - fromToken.setText(text); - Tree t = (Tree)create(fromToken); + WritableToken tok = createToken(fromToken); + tok.setType(tokenType); + tok.setText(text); + Tree t = (Tree)create(tok); return t; } @@ -273,7 +273,7 @@ public abstract class BaseTreeAdaptor implements TreeAdaptor { * If you care what the token payload objects' type is, you should * override this method and any other createToken variant. */ - public abstract Token createToken(int tokenType, String text); + public abstract WritableToken createToken(int tokenType, String text); /** Tell me how to create a token for use with imaginary token nodes. * For example, there is probably no input symbol associated with imaginary @@ -289,6 +289,6 @@ public abstract class BaseTreeAdaptor implements TreeAdaptor { * If you care what the token payload objects' type is, you should * override this method and any other createToken variant. */ - public abstract Token createToken(Token fromToken); + public abstract WritableToken createToken(Token fromToken); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/CommonTreeAdaptor.java b/runtime/Java/src/org/antlr/v4/runtime/tree/CommonTreeAdaptor.java index 14c0f30d7..452ed47d1 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/CommonTreeAdaptor.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/CommonTreeAdaptor.java @@ -66,7 +66,7 @@ public class CommonTreeAdaptor extends BaseTreeAdaptor { * override this method and any other createToken variant. */ @Override - public Token createToken(int tokenType, String text) { + public WritableToken createToken(int tokenType, String text) { return new CommonToken(tokenType, text); } @@ -85,7 +85,7 @@ public class CommonTreeAdaptor extends BaseTreeAdaptor { * override this method and any other createToken variant. */ @Override - public Token createToken(Token fromToken) { + public WritableToken createToken(Token fromToken) { return new CommonToken(fromToken); } diff --git a/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java b/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java index b42f27fea..fbaddaf2a 100644 --- a/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java +++ b/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java @@ -55,13 +55,26 @@ public class LexerInterpreter implements TokenSource { public String getSourceName() { return g.name; } + public int getCharPositionInLine() { + return 0; + } + + public int getLine() { + return 0; + } + + public CharStream getInputStream() { + return null; + } + public Token nextToken() { + // TODO: Deal with off channel tokens int start = input.index(); + int tokenStartCharPositionInLine = interp.getCharPositionInLine(); + int tokenStartLine = interp.getLine(); int ttype = interp.match(input, Lexer.DEFAULT_MODE); int stop = input.index()-1; - int tokenStartCharPositionInLine = input.getCharPositionInLine(); - int tokenStartLine = input.getLine(); - Token t = new CommonToken(input, ttype, Token.DEFAULT_CHANNEL, start, stop); + WritableToken t = new CommonToken(this, ttype, Token.DEFAULT_CHANNEL, start, stop); t.setLine(tokenStartLine); t.setCharPositionInLine(tokenStartCharPositionInLine); return t; diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java index 48c64b17f..cd243250d 100644 --- a/tool/test/org/antlr/v4/test/BaseTest.java +++ b/tool/test/org/antlr/v4/test/BaseTest.java @@ -837,7 +837,10 @@ public abstract class BaseTest { Set hide = new HashSet(); protected void sync(int i) { super.sync(i); - if ( hide.contains(get(i).getType()) ) get(i).setChannel(Token.HIDDEN_CHANNEL); + Token t = get(i); + if ( hide.contains(t.getType()) ) { + ((WritableToken)t).setChannel(Token.HIDDEN_CHANNEL); + } } public void setTokenTypeChannel(int ttype, int channel) { hide.add(ttype); diff --git a/tool/test/org/antlr/v4/test/TestCommonTokenStream.java b/tool/test/org/antlr/v4/test/TestCommonTokenStream.java index c11366817..b6d5a7724 100644 --- a/tool/test/org/antlr/v4/test/TestCommonTokenStream.java +++ b/tool/test/org/antlr/v4/test/TestCommonTokenStream.java @@ -177,7 +177,7 @@ public class TestCommonTokenStream extends BaseTest { TokenSource lexer = // simulate input " x =34 ;\n" new TokenSource() { int i = 0; - Token[] tokens = { + WritableToken[] tokens = { new CommonToken(1," "), new CommonToken(1,"x"), new CommonToken(1," "), @@ -200,7 +200,16 @@ public class TestCommonTokenStream extends BaseTest { return tokens[i++]; } public String getSourceName() { return "test"; } - }; + public int getCharPositionInLine() { + return 0; + } + public int getLine() { + return 0; + } + public CharStream getInputStream() { + return null; + } + }; CommonTokenStream tokens = new CommonTokenStream(lexer);