Split token into read-only and writable portions. added release method. moved the methods around that get input streams and token streams between interfaces. simplify the character stream interface by moving the gas line and get characterization methods to the lexer.

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9039]

runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java

@@ -129,7 +129,7 @@ public class BufferedTokenStream implements TokenStream {
     /** add n elements to buffer */
     protected void fetch(int n) {
         for (int i=1; i<=n; i++) {
-            Token t = tokenSource.nextToken();
+            WritableToken t = (WritableToken)tokenSource.nextToken();
 //            System.out.println("adding "+t+" at index "+tokens.size());
             t.setTokenIndex(tokens.size());
             tokens.add(t);

runtime/Java/src/org/antlr/v4/runtime/CharStream.java

@@ -37,22 +37,4 @@ public interface CharStream extends IntStream {
 	 *  use this on streams that don't support it.
 	 */
 	public String substring(int start, int stop);
-
-	/** Get the ith character of lookahead.  This is the same usually as
-	 *  LA(i).  This will be used for labels in the generated
-	 *  lexer code.  I'd prefer to return a char here type-wise, but it's
-	 *  probably better to be 32-bit clean and be consistent with LA.
-	 */
-	public int LT(int i);
-
-	/** ANTLR tracks the line information automatically */
-	int getLine();
-
-	/** Because this stream can rewind, we need to be able to reset the line */
-	void setLine(int line);
-
-	void setCharPositionInLine(int pos);
-
-	/** The index of the character relative to the beginning of the line 0..n-1 */
-	int getCharPositionInLine();
 }

runtime/Java/src/org/antlr/v4/runtime/CommonToken.java

@@ -30,17 +30,19 @@ package org.antlr.v4.runtime;
 
 import java.io.Serializable;
 
-public class CommonToken implements Token, Serializable {
+public class CommonToken implements WritableToken, Serializable {
 	protected int type;
 	protected int line;
 	protected int charPositionInLine = -1; // set to invalid position
 	protected int channel=DEFAULT_CHANNEL;
-	protected transient CharStream input;
+	protected TokenSource source;
+	// TODO: rm protected transient CharStream input;
 
 	/** We need to be able to change the text once in a while.  If
 	 *  this is non-null, then getText should return this.  Note that
 	 *  start/stop are not affected by changing this.
 	  */
+	// TODO: can store these in map in token stream rather than as field here
 	protected String text;
 
 	/** What token number is this from 0..n-1 tokens; < 0 implies invalid index */
@@ -56,14 +58,14 @@ public class CommonToken implements Token, Serializable {
 		this.type = type;
 	}
 
-	public CommonToken(CharStream input, int type, int channel, int start, int stop) {
-		this.input = input;
+	public CommonToken(TokenSource source, int type, int channel, int start, int stop) {
+		this.source = source;
 		this.type = type;
 		this.channel = channel;
 		this.start = start;
 		this.stop = stop;
-        this.line = input.getLine();
-        this.charPositionInLine = input.getCharPositionInLine();
+        this.line = source.getLine();
+        this.charPositionInLine = source.getCharPositionInLine();
 	}
 
 	public CommonToken(int type, String text) {
@@ -79,7 +81,7 @@ public class CommonToken implements Token, Serializable {
 		index = oldToken.getTokenIndex();
 		charPositionInLine = oldToken.getCharPositionInLine();
 		channel = oldToken.getChannel();
-        input = oldToken.getInputStream();
+        source = oldToken.getTokenSource();
 		if ( oldToken instanceof CommonToken ) {
 			start = ((CommonToken)oldToken).start;
 			stop = ((CommonToken)oldToken).stop;
@@ -98,6 +100,7 @@ public class CommonToken implements Token, Serializable {
 		if ( text!=null ) {
 			return text;
 		}
+		CharStream input = getTokenSource().getInputStream();
 		if ( input==null ) {
 			return null;
 		}
@@ -167,12 +170,12 @@ public class CommonToken implements Token, Serializable {
 		this.index = index;
 	}
 
-	public CharStream getInputStream() {
-		return input;
+	public TokenSource getTokenSource() {
+		return source;
 	}
 
-	public void setInputStream(CharStream input) {
-		this.input = input;
+	public CharStream getInputStream() {
+		return source.getInputStream();
 	}
 
 	public String toString() {

runtime/Java/src/org/antlr/v4/runtime/IntStream.java

@@ -31,9 +31,8 @@ package org.antlr.v4.runtime;
 /** A simple stream of integers used when all I care about is the char
  *  or token type sequence (such as interpretation).
  *
- *  TODO: Notes from Oli: index, seek, LA, LT, consume, getSourceName needed
  *  Do like Java IO and have new BufferedStream(new TokenStream) rather than
- *  using inheritance. line/col go into lexer
+ *  using inheritance?
  */
 public interface IntStream {
 	void consume();
@@ -46,13 +45,17 @@ public interface IntStream {
 	int LA(int i);
 
 	/** Tell the stream to start buffering if it hasn't already.  Return
-     *  current input position, index(). seek(mark()) should not
-	 *  affect the input cursor.
+     *  a marker, usually current input position, index().
+	 *  Calling release(mark()) should not affect the input cursor.
+	 *  Can seek to any index between where we were when mark() was called
+	 *  and current index() until we release this marker.
      */
 	int mark();
 
-	/** Reset the stream so that next call to index() would return index arg. */
-	void rewind(int index);
+	/** Release requirement that stream holds tokens from marked location
+	 *  to current index().
+	 */
+	void release(int marker);
 
 	/** Return the current input symbol index 0..n where n indicates the
      *  last symbol has been read.  The index is the symbol about to be
@@ -61,21 +64,16 @@ public interface IntStream {
 	int index();
 
 	/** Set the input cursor to the position indicated by index.  This is
-	 *  normally used to seek ahead in the input stream.  No buffering is
-	 *  required to do this unless you know your stream will use seek to
-	 *  move backwards such as when backtracking.
-	 *
-	 *  This is different from rewind in its multi-directional
-	 *  requirement and in that its argument is strictly an input cursor (index).
+	 *  normally used to rewind the input stream but can move forward as well.
+	 *  It's up to the stream implementation to make sure that tokens are
+	 *  buffered as necessary to make seek land on a valid token.
+	 *  Or, they should avoid moving the input cursor.
 	 *
 	 *  For char streams, seeking forward must update the stream state such
 	 *  as line number.  For seeking backwards, you will be presumably
 	 *  backtracking using the mark/rewind mechanism that restores state and
 	 *  so this method does not need to update state when seeking backwards.
 	 *
-	 *  Currently, this method is only used for efficient backtracking using
-	 *  memoization, but in the future it may be used for incremental parsing.
-	 *
 	 *  The index is 0..n-1.  A seek to position i means that LA(1) will
 	 *  return the ith symbol.  So, seeking to 0 means LA(1) will return the
 	 *  first element in the stream.

runtime/Java/src/org/antlr/v4/runtime/Lexer.java

@@ -115,13 +115,13 @@ public abstract class Lexer extends Recognizer<LexerATNSimulator>
 			token = null;
 			channel = Token.DEFAULT_CHANNEL;
 			tokenStartCharIndex = input.index();
-			tokenStartCharPositionInLine = input.getCharPositionInLine();
-			tokenStartLine = input.getLine();
+			tokenStartCharPositionInLine = _interp.getCharPositionInLine();
+			tokenStartLine = _interp.getLine();
 			text = null;
 			do {
 				type = Token.INVALID_TYPE;
 				if ( input.LA(1)==CharStream.EOF ) {
-					Token eof = new CommonToken(input,Token.EOF,
+					WritableToken eof = new CommonToken(this,Token.EOF,
 												Token.DEFAULT_CHANNEL,
 												input.index(),input.index());
 					eof.setLine(getLine());
@@ -182,14 +182,14 @@ public abstract class Lexer extends Recognizer<LexerATNSimulator>
 		this.input = input;
 	}
 
-	public CharStream getCharStream() {
-		return ((CharStream)input);
-	}
-
 	public String getSourceName() {
 		return input.getSourceName();
 	}
 
+	public CharStream getInputStream() {
+		return input;
+	}
+
 	/** Currently does not support multiple emits per nextToken invocation
 	 *  for efficiency reasons.  Subclass and override this method and
 	 *  nextToken (to push tokens into a list and pull from that list rather
@@ -210,7 +210,7 @@ public abstract class Lexer extends Recognizer<LexerATNSimulator>
 	 *  Parser or TreeParser.getMissingSymbol().
 	 */
 	public Token emit() {
-		Token t = new CommonToken(((CharStream)input), type,
+		WritableToken t = new CommonToken(this, type,
 										  channel, tokenStartCharIndex,
 										  getCharIndex()-1);
 		t.setLine(tokenStartLine);
@@ -221,11 +221,11 @@ public abstract class Lexer extends Recognizer<LexerATNSimulator>
 	}
 
 	public int getLine() {
-		return ((CharStream)input).getLine();
+		return _interp.getLine();
 	}
 
 	public int getCharPositionInLine() {
-		return ((CharStream)input).getCharPositionInLine();
+		return _interp.getCharPositionInLine();
 	}
 
 	/** What is the index of the current character of lookahead? */
@@ -338,6 +338,7 @@ public abstract class Lexer extends Recognizer<LexerATNSimulator>
 	public void recover(RecognitionException re) {
 		//System.out.println("consuming char "+(char)input.LA(1)+" during recovery");
 		//re.printStackTrace();
+		// TODO: Do we lose character or line position information?
 		input.consume();
 	}
 }

runtime/Java/src/org/antlr/v4/runtime/LexerRecognitionExeption.java

@@ -55,8 +55,8 @@ public class LexerRecognitionExeption extends RuntimeException {
 		this.lexer = lexer;
 		this.index = input.index();
 		this.c = input.LA(1);
-		this.line = input.getLine();
-		this.charPositionInLine = input.getCharPositionInLine();
+		this.line = lexer.getLine();
+		this.charPositionInLine = lexer.getCharPositionInLine();
 	}
 
 }

runtime/Java/src/org/antlr/v4/runtime/Parser.java

@@ -67,7 +67,7 @@ public class Parser extends BaseRecognizer {
 		t.line = current.getLine();
 		t.charPositionInLine = current.getCharPositionInLine();
 		t.channel = Token.DEFAULT_CHANNEL;
-		t.input = current.getInputStream();
+		t.source = current.getTokenSource();
 		return t;
 	}
 

runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java

@@ -121,7 +121,7 @@ public class RecognitionException extends RuntimeException {
 		if ( recognizer!=null ) expecting = recognizer._interp.atn.nextTokens(ctx);
 
 		this.index = input.index();
-		if ( input instanceof TokenStream) {
+		if ( input instanceof TokenStream ) {
 			this.token = ((TokenStream)input).LT(1);
 			this.line = token.getLine();
 			this.charPositionInLine = token.getCharPositionInLine();
@@ -129,11 +129,6 @@ public class RecognitionException extends RuntimeException {
 		if ( input instanceof TreeNodeStream) {
 			//extractInformationFromTreeNodeStream(input);
 		}
-		else if ( input instanceof CharStream) {
-			this.c = input.LA(1);
-			this.line = ((CharStream)input).getLine();
-			this.charPositionInLine = ((CharStream)input).getCharPositionInLine();
-		}
 		else {
 			this.c = input.LA(1);
 		}

runtime/Java/src/org/antlr/v4/runtime/Token.java

@@ -29,6 +29,10 @@
 
 package org.antlr.v4.runtime;
 
+/** A token has properties: text, type, line, character position in the line
+ *  (so we can ignore tabs), token channel, index, and source from which
+ *  we obtained this token.
+ */
 public interface Token {
 	public static final int INVALID_TYPE = 0;
 	public static final Token INVALID_TOKEN = new CommonToken(INVALID_TYPE);
@@ -36,6 +40,7 @@ public interface Token {
 
 	/** imaginary tree navigation type; traverse "get child" link */
 	public static final int DOWN = 1;
+
 	/** imaginary tree navigation type; finish with a child list */
 	public static final int UP = 2;
 
@@ -55,32 +60,34 @@ public interface Token {
 	public static final int HIDDEN_CHANNEL = 99;
 
 	/** Get the text of the token */
-	public String getText();
-	public void setText(String text);
+	String getText();
 
-	public int getType();
-	public void setType(int ttype);
-	/**  The line number on which this token was matched; line=1..n */
-	public int getLine();
-    public void setLine(int line);
+	/** Get the token type of the token */
+	int getType();
 
-	/** The index of the first character relative to the beginning of the line 0..n-1 */
-	public int getCharPositionInLine();
-	public void setCharPositionInLine(int pos);
+	/** The line number on which the 1st character of this token was matched,
+	 *  line=1..n
+	 */
+	int getLine();
 
-	public int getChannel();
-	public void setChannel(int channel);
+	/** The index of the first character of this token relative to the
+	 *  beginning of the line at which it occurs, 0..n-1
+	 */
+	int getCharPositionInLine();
+
+	/** Return the channel this token. Each token can arrive at the parser
+	 *  on a different channel, but the parser only "tunes" to a single channel.
+	 *  The parser ignores everything not on DEFAULT_CHANNEL.
+	 */
+	int getChannel();
 
 	/** An index from 0..n-1 of the token object in the input stream.
 	 *  This must be valid in order to use the ANTLRWorks debugger.
 	 */
-	public int getTokenIndex();
-	public void setTokenIndex(int index);
+	int getTokenIndex();
 
-	/** From what character stream was this token created?  You don't have to
-	 *  implement but it's nice to know where a Token comes from if you have
-	 *  include files etc... on the input.
+	/** Where does this token come from? You can get the
+	 *  character input stream from the token source.
 	 */
-	public CharStream getInputStream();
-	public void setInputStream(CharStream input);
+	TokenSource getTokenSource();
 }

runtime/Java/src/org/antlr/v4/runtime/TokenSource.java

@@ -48,6 +48,16 @@ public interface TokenSource {
 	 */
 	public Token nextToken();
 
+	public int getLine();
+
+	public int getCharPositionInLine();
+
+	/** From what character stream was this token created?  You don't have to
+	 *  implement but it's nice to know where a Token comes from if you have
+	 *  include files etc... on the input.
+	 */
+	public CharStream getInputStream();
+
 	/** Where are you getting tokens from? normally the implication will simply
 	 *  ask lexers input stream.
 	 */

runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java

@@ -41,6 +41,12 @@ public class LexerATNSimulator extends ATNSimulator {
 
 	protected Lexer recog;
 
+	/** line number 1..n within the input */
+	protected int line = 1;
+
+	/** The index of the character relative to the beginning of the line 0..n-1 */
+	protected int charPositionInLine = 0;
+
 	protected DFA[] dfa;
 	protected int mode = Lexer.DEFAULT_MODE;
 
@@ -87,6 +93,9 @@ public class LexerATNSimulator extends ATNSimulator {
 											(char)input.LA(1));
 		//System.out.println("DFA start of execDFA: "+dfa[mode].toLexerString());
 		int prevAcceptMarker = -1;
+		int prevAcceptIndex = -1;
+		int prevAcceptLine = 0;
+		int prevAcceptCharPos = -1;
 		DFAState prevAcceptState = null;
 		DFAState s = s0;
 		int startIndex = input.index();
@@ -99,7 +108,10 @@ public class LexerATNSimulator extends ATNSimulator {
 				if ( dfa_debug ) System.out.println("accept; predict "+s.prediction+
 													" in state "+s.stateNumber);
 				prevAcceptState = s;
-				prevAcceptMarker = input.index();
+				prevAcceptMarker = input.mark();
+				prevAcceptIndex = input.index();
+				prevAcceptLine = line;
+				prevAcceptCharPos = charPositionInLine;
 				// keep going unless we're at EOF; check if something else could match
 				if ( t==CharStream.EOF ) break;
 			}
@@ -134,7 +146,7 @@ public class LexerATNSimulator extends ATNSimulator {
 			DFAState target = s.edges[t];
 			if ( target == ERROR ) break;
 			s = target;
-			input.consume();
+			consume(input, t);
 			t = input.LA(1);
 		}
 		if ( prevAcceptState==null ) {
@@ -150,7 +162,10 @@ public class LexerATNSimulator extends ATNSimulator {
 			}
 			if ( actionIndex>=0 ) recog.action(null, prevAcceptState.ruleIndex, actionIndex);
 		}
-		input.seek(prevAcceptMarker);
+		input.release(prevAcceptMarker);
+		input.seek(prevAcceptIndex);
+		line = prevAcceptLine;
+		charPositionInLine = prevAcceptCharPos;
 		return prevAcceptState.prediction;
 	}
 
@@ -223,7 +238,7 @@ public class LexerATNSimulator extends ATNSimulator {
 				break;
 			}
 
-			input.consume();
+			consume(input, t);
 			addDFAEdge(closure, t, reach);
 			t = input.LA(1);
 
@@ -464,6 +479,28 @@ public class LexerATNSimulator extends ATNSimulator {
 		return dfa[mode];
 	}
 
+	public int getLine() {
+		return line;
+	}
+
+	public int getCharPositionInLine() {
+		return charPositionInLine;
+	}
+
+	public void consume(CharStream input, int curChar) {
+		//System.out.println("prev p="+p+", c="+(char)data[p]);
+		charPositionInLine++;
+		if ( curChar=='\n' ) {
+			/*
+			   System.out.println("newline char found on line: "+line+
+								  "@ pos="+charPositionInLine);
+			   */
+			line++;
+			charPositionInLine=0;
+		}
+		input.consume();
+	}
+
 	public String getTokenName(int t) {
 		if ( t==-1 ) return "EOF";
 		//if ( atn.g!=null ) return atn.g.getTokenDisplayName(t);

runtime/Java/src/org/antlr/v4/runtime/debug/BlankDebugEventListener.java

@@ -28,8 +28,7 @@
 package org.antlr.v4.runtime.debug;
 
 
-import org.antlr.v4.runtime.RecognitionException;
-import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.*;
 
 /** A blank listener that does nothing; useful for real classes so
  *  they don't have to have lots of blank methods and are less
@@ -48,7 +47,7 @@ public class BlankDebugEventListener implements DebugEventListener {
 	public void consumeHiddenToken(Token token) {}
 	public void LT(int i, Token t) {}
 	public void mark(int i) {}
-	public void rewind(int i) {}
+	public void release(int i) {}
 	public void seek(int i) {}
 	public void recognitionException(RecognitionException e) {}
 	public void beginResync() {}

runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventHub.java

@@ -27,11 +27,9 @@
  */
 package org.antlr.v4.runtime.debug;
 
-import org.antlr.v4.runtime.RecognitionException;
-import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.*;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
 
 /** Broadcast debug events to multiple listeners.  Lets you debug and still
  *  use the event mechanism to build parse trees etc...  Not thread-safe.
@@ -158,10 +156,10 @@ public class DebugEventHub implements DebugEventListener {
 		}
 	}
 
-	public void rewind(int index) {
+	public void release(int marker) {
 		for (int i = 0; i < listeners.size(); i++) {
 			DebugEventListener listener = (DebugEventListener)listeners.get(i);
-			listener.rewind(index);
+			listener.release(marker);
 		}
 	}
 

runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventListener.java

@@ -28,8 +28,7 @@
 package org.antlr.v4.runtime.debug;
 
 
-import org.antlr.v4.runtime.RecognitionException;
-import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.*;
 
 /** All debugging events that a recognizer can trigger.
  *
@@ -91,7 +90,7 @@ public interface DebugEventListener {
 	/** After an arbitrarily long lookahead, this informs the
 	 *  debugger that stream should be rewound to the position index.
 	 */
-	public void rewind(int index);
+	public void release(int marker);
 
 	/** Set the input cursor to the position indicated by index. */
 	public void seek(int index);

runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventRepeater.java

@@ -27,8 +27,7 @@
  */
 package org.antlr.v4.runtime.debug;
 
-import org.antlr.v4.runtime.RecognitionException;
-import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.*;
 
 /** A simple event repeater (proxy) that delegates all functionality to the
  *  listener sent into the ctor.  Useful if you want to listen in on a few
@@ -58,7 +57,7 @@ public class DebugEventRepeater implements DebugEventListener {
 	public void consumeHiddenToken(Token token) { listener.consumeHiddenToken(token); }
 	public void LT(int i, Token t) { listener.LT(i, t); }
 	public void mark(int i) { listener.mark(i); }
-	public void rewind(int i) { listener.rewind(i); }
+	public void release(int i) { listener.release(i); }
 	public void seek(int i) { listener.seek(i); }
 	public void recognitionException(RecognitionException e) { listener.recognitionException(e); }
 	public void beginResync() { listener.beginResync(); }

runtime/Java/src/org/antlr/v4/runtime/debug/DebugEventSocketProxy.java

@@ -27,14 +27,11 @@
  */
 package org.antlr.v4.runtime.debug;
 
-import org.antlr.v4.runtime.BaseRecognizer;
-import org.antlr.v4.runtime.RecognitionException;
-import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.*;
 import org.antlr.v4.runtime.tree.TreeAdaptor;
 
 import java.io.*;
-import java.net.ServerSocket;
-import java.net.Socket;
+import java.net.*;
 
 /** A proxy debug event listener that forwards events over a socket to
  *  a debugger (or any other listener) using a simple text-based protocol;
@@ -165,7 +162,7 @@ public class DebugEventSocketProxy extends BlankDebugEventListener {
 		transmit("mark\t"+i);
 	}
 
-	public void rewind(int i) {
+	public void release(int i) {
 		transmit("rewind\t"+i);
 	}
 

runtime/Java/src/org/antlr/v4/runtime/debug/DebugTokenStream.java

@@ -27,9 +27,7 @@
  */
 package org.antlr.v4.runtime.debug;
 
-import org.antlr.v4.runtime.Token;
-import org.antlr.v4.runtime.TokenSource;
-import org.antlr.v4.runtime.TokenStream;
+import org.antlr.v4.runtime.*;
 
 public class DebugTokenStream implements TokenStream {
 	protected DebugEventListener dbg;
@@ -111,9 +109,9 @@ public class DebugTokenStream implements TokenStream {
 		return input.range();
 	}
 
-	public void rewind(int index) {
-		dbg.rewind(index);
-		input.seek(index);
+	public void release(int index) {
+		dbg.release(index);
+		input.release(index);
 	}
 
 	public void seek(int index) {

runtime/Java/src/org/antlr/v4/runtime/debug/DebugTreeNodeStream.java

@@ -28,8 +28,7 @@
 package org.antlr.v4.runtime.debug;
 
 import org.antlr.v4.runtime.TokenStream;
-import org.antlr.v4.runtime.tree.TreeAdaptor;
-import org.antlr.v4.runtime.tree.TreeNodeStream;
+import org.antlr.v4.runtime.tree.*;
 
 /** Debug any tree node stream.  The constructor accepts the stream
  *  and a debug listener.  As node stream calls come in, debug events
@@ -95,9 +94,9 @@ public class DebugTreeNodeStream implements TreeNodeStream {
 		return lastMarker;
 	}
 
-	public void rewind(int marker) {
-		dbg.rewind(marker);
-		input.rewind(marker);
+	public void release(int marker) {
+		dbg.release(marker);
+		input.release(marker);
 	}
 
 	public int index() {

runtime/Java/src/org/antlr/v4/runtime/debug/RemoteDebugEventSocketListener.java

@@ -27,15 +27,11 @@
  */
 package org.antlr.v4.runtime.debug;
 
-import org.antlr.v4.runtime.CharStream;
-import org.antlr.v4.runtime.RecognitionException;
-import org.antlr.v4.runtime.Token;
-import org.antlr.v4.runtime.tree.BaseTree;
-import org.antlr.v4.runtime.tree.Tree;
+import org.antlr.v4.runtime.*;
+import org.antlr.v4.runtime.tree.*;
 
 import java.io.*;
-import java.net.ConnectException;
-import java.net.Socket;
+import java.net.*;
 import java.util.StringTokenizer;
 
 public class RemoteDebugEventSocketListener implements Runnable {
@@ -110,11 +106,9 @@ public class RemoteDebugEventSocketListener implements Runnable {
 		public void setTokenIndex(int index) {
 			this.index = index;
 		}
-		public CharStream getInputStream() {
+		public TokenSource getTokenSource() {
 			return null;
 		}
-		public void setInputStream(CharStream input) {
-		}
 		public String toString() {
 			String channelStr = "";
 			if ( channel!= Token.DEFAULT_CHANNEL ) {

runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java

@@ -195,19 +195,19 @@ public abstract class BaseTreeAdaptor implements TreeAdaptor {
 	}
 
 	public Object create(int tokenType, Token fromToken) {
-		fromToken = createToken(fromToken);
+		WritableToken tok = createToken(fromToken);
 		//((ClassicToken)fromToken).setType(tokenType);
-		fromToken.setType(tokenType);
-		Tree t = (Tree)create(fromToken);
+		tok.setType(tokenType);
+		Tree t = (Tree)create(tok);
 		return t;
 	}
 
 	public Object create(int tokenType, Token fromToken, String text) {
         if (fromToken == null) return create(tokenType, text);
-		fromToken = createToken(fromToken);
-		fromToken.setType(tokenType);
-		fromToken.setText(text);
-		Tree t = (Tree)create(fromToken);
+		WritableToken tok = createToken(fromToken);
+		tok.setType(tokenType);
+		tok.setText(text);
+		Tree t = (Tree)create(tok);
 		return t;
 	}
 
@@ -273,7 +273,7 @@ public abstract class BaseTreeAdaptor implements TreeAdaptor {
 	 *  If you care what the token payload objects' type is, you should
 	 *  override this method and any other createToken variant.
 	 */
-	public abstract Token createToken(int tokenType, String text);
+	public abstract WritableToken createToken(int tokenType, String text);
 
 	/** Tell me how to create a token for use with imaginary token nodes.
 	 *  For example, there is probably no input symbol associated with imaginary
@@ -289,6 +289,6 @@ public abstract class BaseTreeAdaptor implements TreeAdaptor {
 	 *  If you care what the token payload objects' type is, you should
 	 *  override this method and any other createToken variant.
 	 */
-	public abstract Token createToken(Token fromToken);
+	public abstract WritableToken createToken(Token fromToken);
 }
 

runtime/Java/src/org/antlr/v4/runtime/tree/CommonTreeAdaptor.java

@@ -66,7 +66,7 @@ public class CommonTreeAdaptor extends BaseTreeAdaptor {
 	 *  override this method and any other createToken variant.
 	 */
 	@Override
-	public Token createToken(int tokenType, String text) {
+	public WritableToken createToken(int tokenType, String text) {
 		return new CommonToken(tokenType, text);
 	}
 
@@ -85,7 +85,7 @@ public class CommonTreeAdaptor extends BaseTreeAdaptor {
 	 *  override this method and any other createToken variant.
 	 */
 	@Override
-	public Token createToken(Token fromToken) {
+	public WritableToken createToken(Token fromToken) {
 		return new CommonToken(fromToken);
 	}
 

tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java

@@ -55,13 +55,26 @@ public class LexerInterpreter implements TokenSource {
 
 	public String getSourceName() {	return g.name; }
 
+	public int getCharPositionInLine() {
+		return 0;
+	}
+
+	public int getLine() {
+		return 0;
+	}
+
+	public CharStream getInputStream() {
+		return null;
+	}
+
 	public Token nextToken() {
+		// TODO: Deal with off channel tokens
 		int start = input.index();
+		int tokenStartCharPositionInLine = interp.getCharPositionInLine();
+		int tokenStartLine = interp.getLine();
 		int ttype = interp.match(input, Lexer.DEFAULT_MODE);
 		int stop = input.index()-1;
-		int tokenStartCharPositionInLine = input.getCharPositionInLine();
-		int tokenStartLine = input.getLine();
-		Token t = new CommonToken(input, ttype, Token.DEFAULT_CHANNEL, start, stop);
+		WritableToken t = new CommonToken(this, ttype, Token.DEFAULT_CHANNEL, start, stop);
 		t.setLine(tokenStartLine);
 		t.setCharPositionInLine(tokenStartCharPositionInLine);
 		return t;

tool/test/org/antlr/v4/test/BaseTest.java

@@ -837,7 +837,10 @@ public abstract class BaseTest {
         Set<Integer> hide = new HashSet<Integer>();
         protected void sync(int i) {
             super.sync(i);
-            if ( hide.contains(get(i).getType()) ) get(i).setChannel(Token.HIDDEN_CHANNEL);
+			Token t = get(i);
+			if ( hide.contains(t.getType()) ) {
+				((WritableToken)t).setChannel(Token.HIDDEN_CHANNEL);
+			}
         }
         public void setTokenTypeChannel(int ttype, int channel) {
             hide.add(ttype);

tool/test/org/antlr/v4/test/TestCommonTokenStream.java

@@ -177,7 +177,7 @@ public class TestCommonTokenStream extends BaseTest {
         TokenSource lexer = // simulate input " x =34  ;\n"
             new TokenSource() {
                 int i = 0;
-                Token[] tokens = {
+                WritableToken[] tokens = {
                     new CommonToken(1," "),
                     new CommonToken(1,"x"),
                     new CommonToken(1," "),
@@ -200,6 +200,15 @@ public class TestCommonTokenStream extends BaseTest {
                     return tokens[i++];
                 }
                 public String getSourceName() { return "test"; }
+				public int getCharPositionInLine() {
+					return 0;
+				}
+				public int getLine() {
+					return 0;
+				}
+				public CharStream getInputStream() {
+					return null;
+				}
 			};
 
         CommonTokenStream tokens = new CommonTokenStream(lexer);