From ea434982fb9f4aa7d7cd68554b172079375055d8 Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Fri, 16 Mar 2012 08:07:46 -0500
Subject: [PATCH 01/34] Pull error reporting outside of try/finally to ensure
 errors are not hidden

---
 tool/test/org/antlr/v4/test/BaseTest.java | 40 ++++++++++++-----------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java
index 2261b40ea..f811e4568 100644
--- a/tool/test/org/antlr/v4/test/BaseTest.java
+++ b/tool/test/org/antlr/v4/test/BaseTest.java
@@ -358,37 +358,39 @@ public abstract class BaseTest {
 		System.out.println("dir "+tmpdir);
 		mkdir(tmpdir);
 		writeFile(tmpdir, fileName, grammarStr);
+		ErrorQueue equeue = new ErrorQueue();
+		final List<String> options = new ArrayList<String>();
+		Collections.addAll(options, extraOptions);
+		options.add("-o");
+		options.add(tmpdir);
+		options.add("-lib");
+		options.add(tmpdir);
+		options.add(new File(tmpdir,grammarFileName).toString());
 		try {
-			final List<String> options = new ArrayList<String>();
-			Collections.addAll(options, extraOptions);
-			options.add("-o");
-			options.add(tmpdir);
-			options.add("-lib");
-			options.add(tmpdir);
-			options.add(new File(tmpdir,grammarFileName).toString());
 			final String[] optionsA = new String[options.size()];
 			options.toArray(optionsA);
-			ErrorQueue equeue = new ErrorQueue();
 			Tool antlr = newTool(optionsA);
 			antlr.addListener(equeue);
 			antlr.processGrammarsOnCommandLine();
-			if ( equeue.errors.size()>0 ) {
-				allIsWell = false;
-				System.err.println("antlr reports errors from "+options);
-				for (int i = 0; i < equeue.errors.size(); i++) {
-					ANTLRMessage msg = equeue.errors.get(i);
-					System.err.println(msg);
-				}
-				System.out.println("!!!\ngrammar:");
-				System.out.println(grammarStr);
-				System.out.println("###");
-			}
 		}
 		catch (Exception e) {
 			allIsWell = false;
 			System.err.println("problems building grammar: "+e);
 			e.printStackTrace(System.err);
 		}
+
+		if ( equeue.errors.size()>0 ) {
+			allIsWell = false;
+			System.err.println("antlr reports errors from "+options);
+			for (int i = 0; i < equeue.errors.size(); i++) {
+				ANTLRMessage msg = equeue.errors.get(i);
+				System.err.println(msg);
+			}
+			System.out.println("!!!\ngrammar:");
+			System.out.println(grammarStr);
+			System.out.println("###");
+		}
+
 		return allIsWell;
 	}
 

From cc20a52cdd6f04d17f53050c7457aafd1ba51b60 Mon Sep 17 00:00:00 2001
From: Terence Parr <parrt@antlr.org>
Date: Sat, 31 Mar 2012 15:54:00 -0700
Subject: [PATCH 02/34] allow special "tokens" start rule name so we can test
 pure lexer grammars.

---
 .../org/antlr/v4/runtime/misc/TestRig.java    | 25 ++++++++++++-------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/TestRig.java b/runtime/Java/src/org/antlr/v4/runtime/misc/TestRig.java
index f78d090f5..dd600321f 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/misc/TestRig.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/misc/TestRig.java
@@ -57,6 +57,9 @@ import java.lang.reflect.Method;
  *        [input-filename]
  */
 public class TestRig {
+
+	public static final String LEXER_START_RULE_NAME = "tokens";
+
 	public static void main(String[] args) throws Exception {
 		String grammarName;
 		String startRuleName;
@@ -69,10 +72,12 @@ public class TestRig {
 		boolean diagnostics = false;
 		String encoding = null;
 		if ( args.length < 2 ) {
-			System.err.println("java org.antlr.v4.runtime.misc.TestRig GrammarName startRuleName" +
-							   " [-tokens] [-print] [-gui] [-ps file.ps] [-encoding encodingname]" +
-							   " [-trace] [-diagnostics]"+
-							   " [input-filename]");
+			System.err.println("java org.antlr.v4.runtime.misc.TestRig GrammarName startRuleName\n" +
+							   "  [-tokens] [-print] [-gui] [-ps file.ps] [-encoding encodingname]\n" +
+							   "  [-trace] [-diagnostics]\n"+
+							   "  [input-filename]");
+			System.err.println("Use startRuleName='tokens' if GrammarName is a lexer grammar.");
+			System.err.println("Omitting input-filename makes rig read from stdin.");
 			return;
 		}
 		int i=0;
@@ -121,16 +126,11 @@ public class TestRig {
 		}
 //		System.out.println("exec "+grammarName+"."+startRuleName);
 		String lexerName = grammarName+"Lexer";
-		String parserName = grammarName+"Parser";
 		ClassLoader cl = Thread.currentThread().getContextClassLoader();
 		Class lexerClass = cl.loadClass(lexerName);
 		if ( lexerClass==null ) {
 			System.err.println("Can't load "+lexerName);
 		}
-		Class parserClass = cl.loadClass(parserName);
-		if ( parserClass==null ) {
-			System.err.println("Can't load "+parserName);
-		}
 
 		InputStream is = System.in;
 		if ( inputFile!=null ) {
@@ -158,6 +158,13 @@ public class TestRig {
 				}
 			}
 
+			if ( startRuleName.equals(LEXER_START_RULE_NAME) ) return;
+
+			String parserName = grammarName+"Parser";
+			Class parserClass = cl.loadClass(parserName);
+			if ( parserClass==null ) {
+				System.err.println("Can't load "+parserName);
+			}
 			Constructor<Parser> parserCtor = parserClass.getConstructor(TokenStream.class);
 			Parser parser = parserCtor.newInstance(tokens);
 

From f238d7579ef4de47535e16c9d5f98573b252e014 Mon Sep 17 00:00:00 2001
From: Terence Parr <parrt@antlr.org>
Date: Sat, 31 Mar 2012 17:27:11 -0700
Subject: [PATCH 03/34] added getText() to ParseTree. getText(tokens) is still
 really what you want but convenient for ctx.type().getText() when it's just
 one token.

---
 .../antlr/v4/runtime/BufferedTokenStream.java | 17 +++++++------
 .../antlr/v4/runtime/ParserRuleContext.java   | 25 +++++++++++++++++++
 .../src/org/antlr/v4/runtime/RuleContext.java |  6 +++++
 .../org/antlr/v4/runtime/tree/ParseTree.java  |  9 +++++++
 4 files changed, 49 insertions(+), 8 deletions(-)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java
index e038357c8..fc7a2fc41 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java
@@ -29,7 +29,11 @@
 
 package org.antlr.v4.runtime;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Set;
 
 /** Buffer all input tokens but do on-demand fetching of new tokens from
  *  lexer. Useful when the parser or lexer has to set context/mode info before
@@ -246,13 +250,10 @@ public class BufferedTokenStream<T extends Token> implements TokenStream {
         if ( start<0 || stop<0 ) return "";
         if ( p == -1 ) setup();
         if ( stop>=tokens.size() ) stop = tokens.size()-1;
-        StringBuilder buf = new StringBuilder();
-        for (int i = start; i <= stop; i++) {
-            T t = tokens.get(i);
-            if ( t.getType()==Token.EOF ) break;
-            buf.append(t.getText());
-        }
-        return buf.toString();
+
+		int a = tokens.get(start).getStartIndex();
+		int b = tokens.get(stop).getStopIndex();
+		return tokenSource.getInputStream().substring(a, b);
     }
 
     @Override
diff --git a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java
index 906c96b5a..48d748ab1 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java
@@ -297,6 +297,31 @@ public class ParserRuleContext<Symbol extends Token> extends RuleContext {
 		return Interval.of(start.getTokenIndex(), stop.getTokenIndex());
 	}
 
+	/** Return the combined text of all leaf nodes. Does not get any
+	 *  off-channel tokens (if any) so won't return whitespace and
+	 *  comments if they are sent to parser on hidden channel.
+	 *
+	 *  This just recursively collects all leaf nodes and combines text.
+	 */
+	@Override
+	public String getText() {
+		StringBuilder buf = new StringBuilder();
+		getText_(this, buf);
+		return buf.toString();
+	}
+
+	protected void getText_(ParseTree p, StringBuilder buf) {
+		if ( p instanceof TerminalNode ) {
+			buf.append(((TerminalNode)p).getSymbol().getText());
+			return;
+		}
+		int n = p.getChildCount();
+		for (int i=0; i<n; i++) {
+			ParseTree c = p.getChild(i);
+			getText_(c, buf);
+		}
+	}
+
 	/** Return the text matched by this context and below in the parse
 	 *  tree. It includes tokens from this.start .. this.stop inclusive.
 	 *  It includes hidden channel tokens between start, stop.  The
diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java
index 485b369f6..c518291d6 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java
@@ -224,6 +224,12 @@ public class RuleContext implements ParseTree.RuleNode {
 	@Override
 	public RuleContext getPayload() { return this; }
 
+	/** We don't know our children, can't implement */
+	@Override
+	public String getText() {
+		throw new UnsupportedOperationException("RuleContexts don't know their children");
+	}
+
 	public int getRuleIndex() { return -1; }
 
 	@Override
diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java
index 3ff110c97..d8dea305d 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java
@@ -84,6 +84,9 @@ public interface ParseTree extends SyntaxTree {
 			return visitor.visitTerminal(this);
 		}
 
+		@Override
+		public String getText() { return symbol.getText(); }
+
 		@Override
 		public String toStringTree(Parser parser) {
 			return toString();
@@ -135,6 +138,12 @@ public interface ParseTree extends SyntaxTree {
 	/** The ParseTreeVisitor needs a double dispatch method */
 	public <T> T accept(ParseTreeVisitor<? extends T> visitor);
 
+	/** Return the combined text of all leaf nodes. Does not get any
+	 *  off-channel tokens (if any) so won't return whitespace and
+	 *  comments if they are sent to parser on hidden channel.
+	 */
+	public String getText();
+
 	/** Specialize toStringTree so that it can print out more information
 	 * 	based upon the parser.
 	 */

From 588e30bfe17b017ff6fd2128d04b62cd2826610e Mon Sep 17 00:00:00 2001
From: Terence Parr <parrt@antlr.org>
Date: Sun, 1 Apr 2012 12:12:49 -0700
Subject: [PATCH 04/34] getText(...) moved to CharStream and TokenStream;
 cleaned up names. removed getInputString from parser.

---
 .../antlr/v4/runtime/ANTLRInputStream.java    |  6 ++-
 .../antlr/v4/runtime/BufferedTokenStream.java | 33 ++++++++++------
 .../src/org/antlr/v4/runtime/CharStream.java  |  5 ++-
 .../src/org/antlr/v4/runtime/CommonToken.java |  4 +-
 .../v4/runtime/DefaultErrorStrategy.java      |  2 +-
 .../v4/runtime/DiagnosticErrorListener.java   |  7 ++--
 .../Java/src/org/antlr/v4/runtime/Lexer.java  |  4 +-
 .../v4/runtime/LexerNoViableAltException.java |  3 +-
 .../Java/src/org/antlr/v4/runtime/Parser.java | 12 ------
 .../antlr/v4/runtime/ParserRuleContext.java   | 35 -----------------
 .../src/org/antlr/v4/runtime/RuleContext.java | 19 ++++++++-
 .../antlr/v4/runtime/TokenRewriteStream.java  | 39 +++++++++++--------
 .../src/org/antlr/v4/runtime/TokenStream.java | 21 ++++++----
 .../v4/runtime/UnbufferedCharStream.java      |  6 ++-
 .../v4/runtime/UnbufferedTokenStream.java     |  5 ++-
 .../v4/runtime/atn/LexerATNSimulator.java     | 12 ++++--
 .../v4/runtime/atn/ParserATNSimulator.java    | 17 +++++---
 .../v4/tool/templates/codegen/Java/Java.stg   |  6 +--
 tool/test/org/antlr/v4/test/BaseTest.java     | 16 ++++----
 .../antlr/v4/test/TestTokenRewriteStream.java | 19 ++++-----
 20 files changed, 144 insertions(+), 127 deletions(-)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java b/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java
index 88a842cae..f519e81b2 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java
@@ -28,6 +28,8 @@
  */
 package org.antlr.v4.runtime;
 
+import org.antlr.v4.runtime.misc.Interval;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -210,7 +212,9 @@ public class ANTLRInputStream implements CharStream {
 	}
 
 	@Override
-	public String substring(int start, int stop) {
+	public String getText(Interval interval) {
+		int start = interval.a;
+		int stop = interval.b;
 		if ( stop >= n ) stop = n-1;
 		int count = stop - start + 1;
 		if ( start >= n ) return "";
diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java
index fc7a2fc41..d375c1181 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java
@@ -29,6 +29,8 @@
 
 package org.antlr.v4.runtime;
 
+import org.antlr.v4.runtime.misc.Interval;
+
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -239,27 +241,36 @@ public class BufferedTokenStream<T extends Token> implements TokenStream {
 
     /** Grab *all* tokens from stream and return string */
     @Override
-    public String toString() {
-        if ( p == -1 ) setup();
-        fill();
-        return toString(0, tokens.size()-1);
-    }
+    public String toString() { return getText(); }
+
+	/** Get the text of all tokens in this buffer. */
+	public String getText() {
+		if ( p == -1 ) setup();
+		fill();
+		return getText(Interval.of(0,size()-1));
+	}
 
     @Override
-    public String toString(int start, int stop) {
+    public String getText(Interval interval) {
+		int start = interval.a;
+		int stop = interval.b;
         if ( start<0 || stop<0 ) return "";
         if ( p == -1 ) setup();
         if ( stop>=tokens.size() ) stop = tokens.size()-1;
 
-		int a = tokens.get(start).getStartIndex();
-		int b = tokens.get(stop).getStopIndex();
-		return tokenSource.getInputStream().substring(a, b);
+		StringBuilder buf = new StringBuilder();
+		for (int i = start; i <= stop; i++) {
+			T t = tokens.get(i);
+			if ( t.getType()==Token.EOF ) break;
+			buf.append(t.getText());
+		}
+		return buf.toString();
     }
 
     @Override
-    public String toString(Token start, Token stop) {
+    public String getText(Token start, Token stop) {
         if ( start!=null && stop!=null ) {
-            return toString(start.getTokenIndex(), stop.getTokenIndex());
+            return getText(Interval.of(start.getTokenIndex(), stop.getTokenIndex()));
         }
         return null;
     }
diff --git a/runtime/Java/src/org/antlr/v4/runtime/CharStream.java b/runtime/Java/src/org/antlr/v4/runtime/CharStream.java
index da3a490ff..288385731 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/CharStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/CharStream.java
@@ -28,6 +28,8 @@
  */
 package org.antlr.v4.runtime;
 
+import org.antlr.v4.runtime.misc.Interval;
+
 /** A source of characters for an ANTLR lexer */
 public interface CharStream extends IntStream {
 	public static final int EOF = -1;
@@ -37,6 +39,7 @@ public interface CharStream extends IntStream {
 	/** For unbuffered streams, you can't use this; primarily I'm providing
 	 *  a useful interface for action code.  Just make sure actions don't
 	 *  use this on streams that don't support it.
+	 * @param interval
 	 */
-	public String substring(int start, int stop);
+	public String getText(Interval interval);
 }
diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java b/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java
index c474f02b3..26914dbac 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java
@@ -28,6 +28,8 @@
  */
 package org.antlr.v4.runtime;
 
+import org.antlr.v4.runtime.misc.Interval;
+
 import java.io.Serializable;
 
 public class CommonToken implements WritableToken, Serializable {
@@ -109,7 +111,7 @@ public class CommonToken implements WritableToken, Serializable {
 		if ( input==null ) return null;
 		int n = input.size();
 		if ( start<n && stop<n) {
-			return input.substring(start,stop);
+			return input.getText(Interval.of(start,stop));
 		}
 		else {
 			return "<EOF>";
diff --git a/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java b/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java
index af353a47e..40557dc22 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java
@@ -205,7 +205,7 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy {
 		String input;
 		if (tokens instanceof TokenStream) {
 			if ( e.startToken.getType()==Token.EOF ) input = "<EOF>";
-			else input = ((TokenStream)tokens).toString(e.startToken, e.offendingToken);
+			else input = ((TokenStream)tokens).getText(e.startToken, e.offendingToken);
 		}
 		else {
 			input = "<unknown input>";
diff --git a/runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorListener.java b/runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorListener.java
index d79d37acd..46d1b14c5 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorListener.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorListener.java
@@ -31,6 +31,7 @@ package org.antlr.v4.runtime;
 
 import org.antlr.v4.runtime.atn.ATNConfigSet;
 import org.antlr.v4.runtime.dfa.DFA;
+import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.runtime.misc.IntervalSet;
 import org.antlr.v4.runtime.misc.NotNull;
 
@@ -42,7 +43,7 @@ public class DiagnosticErrorListener extends BaseErrorListener<Token> {
     {
         recognizer.notifyErrorListeners("reportAmbiguity d=" + dfa.decision +
 										": ambigAlts=" + ambigAlts + ", input='" +
-										recognizer.getInputString(startIndex, stopIndex) + "'");
+										recognizer.getTokenStream().getText(Interval.of(startIndex, stopIndex)) + "'");
     }
 
 	@Override
@@ -53,7 +54,7 @@ public class DiagnosticErrorListener extends BaseErrorListener<Token> {
 	{
 		recognizer.notifyErrorListeners("reportAttemptingFullContext d=" +
 										dfa.decision + ", input='" +
-										recognizer.getInputString(startIndex, stopIndex) + "'");
+										recognizer.getTokenStream().getText(Interval.of(startIndex, stopIndex)) + "'");
 	}
 
 	@Override
@@ -62,6 +63,6 @@ public class DiagnosticErrorListener extends BaseErrorListener<Token> {
     {
         recognizer.notifyErrorListeners("reportContextSensitivity d=" +
 										dfa.decision + ", input='" +
-										recognizer.getInputString(startIndex, stopIndex) + "'");
+										recognizer.getTokenStream().getText(Interval.of(startIndex, stopIndex)) + "'");
     }
 }
diff --git a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java
index 3bca85c83..3a62a432b 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java
@@ -29,10 +29,10 @@
 package org.antlr.v4.runtime;
 
 import org.antlr.v4.runtime.atn.LexerATNSimulator;
+import org.antlr.v4.runtime.misc.Interval;
 
 import java.util.ArrayDeque;
 import java.util.EmptyStackException;
-import java.util.List;
 
 /** A lexer is recognizer that draws input symbols from a character stream.
  *  lexer grammars result in a subclass of this object. A Lexer object
@@ -308,7 +308,7 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator>
 
 	public void notifyListeners(LexerNoViableAltException e) {
 		String msg = "token recognition error at: '"+
-			_input.substring(_tokenStartCharIndex, _input.index())+"'";
+			_input.getText(Interval.of(_tokenStartCharIndex, _input.index()))+"'";
 
 		ANTLRErrorListener<? super Integer> listener = getErrorListenerDispatch();
 		listener.error(this, null, _tokenStartLine, _tokenStartCharPositionInLine, msg, e);
diff --git a/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java b/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java
index fd43776e3..b3a971ce9 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java
@@ -30,6 +30,7 @@
 package org.antlr.v4.runtime;
 
 import org.antlr.v4.runtime.atn.ATNConfigSet;
+import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.runtime.misc.Utils;
 
 public class LexerNoViableAltException extends RecognitionException {
@@ -57,7 +58,7 @@ public class LexerNoViableAltException extends RecognitionException {
 	public String toString() {
 		String symbol = "";
 		if (startIndex >= 0 && startIndex < input.size()) {
-			symbol = getInputStream().substring(startIndex, startIndex);
+			symbol = getInputStream().getText(Interval.of(startIndex,startIndex));
 			symbol = Utils.escapeWhitespace(symbol, false);
 		}
 
diff --git a/runtime/Java/src/org/antlr/v4/runtime/Parser.java b/runtime/Java/src/org/antlr/v4/runtime/Parser.java
index e0607d3ba..839ab62c3 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/Parser.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/Parser.java
@@ -295,18 +295,6 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
 		this._input = input;
 	}
 
-    public String getInputString(int start) {
-        return getInputString(start, getInputStream().index());
-    }
-
-    public String getInputString(int start, int stop) {
-        SymbolStream<Token> input = getInputStream();
-        if ( input instanceof TokenStream ) {
-            return ((TokenStream)input).toString(start,stop);
-        }
-        return "n/a";
-    }
-
     /** Match needs to return the current input symbol, which gets put
      *  into the label for the associated token ref; e.g., x=ID.
      */
diff --git a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java
index 48d748ab1..0c1fdaa24 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java
@@ -297,41 +297,6 @@ public class ParserRuleContext<Symbol extends Token> extends RuleContext {
 		return Interval.of(start.getTokenIndex(), stop.getTokenIndex());
 	}
 
-	/** Return the combined text of all leaf nodes. Does not get any
-	 *  off-channel tokens (if any) so won't return whitespace and
-	 *  comments if they are sent to parser on hidden channel.
-	 *
-	 *  This just recursively collects all leaf nodes and combines text.
-	 */
-	@Override
-	public String getText() {
-		StringBuilder buf = new StringBuilder();
-		getText_(this, buf);
-		return buf.toString();
-	}
-
-	protected void getText_(ParseTree p, StringBuilder buf) {
-		if ( p instanceof TerminalNode ) {
-			buf.append(((TerminalNode)p).getSymbol().getText());
-			return;
-		}
-		int n = p.getChildCount();
-		for (int i=0; i<n; i++) {
-			ParseTree c = p.getChild(i);
-			getText_(c, buf);
-		}
-	}
-
-	/** Return the text matched by this context and below in the parse
-	 *  tree. It includes tokens from this.start .. this.stop inclusive.
-	 *  It includes hidden channel tokens between start, stop.  The
-	 *  edge tokens are always on-channel tokens.
-	 */
-	public String getText(TokenStream tokens) {
-		Interval range = getSourceInterval();
-		return range==Interval.INVALID ? null : tokens.toString(range.a, range.b);
-	}
-
 	public Symbol getStart() { return start; }
 	public Symbol getStop() { return stop; }
 
diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java
index c518291d6..8d87e85d3 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java
@@ -224,10 +224,25 @@ public class RuleContext implements ParseTree.RuleNode {
 	@Override
 	public RuleContext getPayload() { return this; }
 
-	/** We don't know our children, can't implement */
+	/** Return the combined text of all child nodes. This method only considers
+	 *  tokens which have been added to the parse tree.
+	 *  <p>
+	 *  Since tokens on hidden channels (e.g. whitespace or comments) are not
+	 *  added to the parse trees, they will not appear in the output of this
+	 *  method.
+	 */
 	@Override
 	public String getText() {
-		throw new UnsupportedOperationException("RuleContexts don't know their children");
+		if (getChildCount() == 0) {
+			return "";
+		}
+
+		StringBuilder builder = new StringBuilder();
+		for (int i = 0; i < getChildCount(); i++) {
+			builder.append(getChild(i).getText());
+		}
+
+		return builder.toString();
 	}
 
 	public int getRuleIndex() { return -1; }
diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenRewriteStream.java b/runtime/Java/src/org/antlr/v4/runtime/TokenRewriteStream.java
index c48db68f0..85f09c99f 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/TokenRewriteStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/TokenRewriteStream.java
@@ -28,9 +28,13 @@
  */
 package org.antlr.v4.runtime;
 
+import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.runtime.misc.Nullable;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 /** Useful for dumping out the input stream after doing some
  *  augmentation or other manipulations.
@@ -354,30 +358,35 @@ public class TokenRewriteStream extends CommonTokenStream {
 	}
 
 	@Override
-	public String toString() {
-        fill();
-		return toString(MIN_TOKEN_INDEX, size()-1);
-	}
+	public String toString() { return getText(); }
 
 	public String toString(String programName) {
         fill();
-		return toString(programName, MIN_TOKEN_INDEX, size()-1);
+		return toString(programName, Interval.of(0, size() - 1));
 	}
 
 	@Override
-	public String toString(int start, int end) {
-		return toString(DEFAULT_PROGRAM_NAME, start, end);
+	public String getText() {
+		fill();
+		return getText(Interval.of(0,size()-1));
 	}
 
-	public String toString(String programName, int start, int end) {
+	@Override
+	public String getText(Interval interval) {
+		return toString(DEFAULT_PROGRAM_NAME, interval);
+	}
+
+	public String toString(String programName, Interval interval) {
 		List<RewriteOperation> rewrites = programs.get(programName);
+		int start = interval.a;
+		int stop = interval.b;
 
         // ensure start/end are in range
-        if ( end>tokens.size()-1 ) end = tokens.size()-1;
+        if ( stop>tokens.size()-1 ) stop = tokens.size()-1;
         if ( start<0 ) start = 0;
 
         if ( rewrites==null || rewrites.isEmpty() ) {
-			return toOriginalString(start,end); // no instructions to execute
+			return toOriginalString(start,stop); // no instructions to execute
 		}
 		StringBuilder buf = new StringBuilder();
 
@@ -386,7 +395,7 @@ public class TokenRewriteStream extends CommonTokenStream {
 
         // Walk buffer, executing instructions and emitting tokens
         int i = start;
-        while ( i <= end && i < tokens.size() ) {
+        while ( i <= stop && i < tokens.size() ) {
 			RewriteOperation op = indexToOp.get(i);
 			indexToOp.remove(i); // remove so any left have index size-1
 			Token t = tokens.get(i);
@@ -403,12 +412,10 @@ public class TokenRewriteStream extends CommonTokenStream {
         // include stuff after end if it's last index in buffer
         // So, if they did an insertAfter(lastValidIndex, "foo"), include
         // foo if end==lastValidIndex.
-        if ( end==tokens.size()-1 ) {
+        if ( stop==tokens.size()-1 ) {
             // Scan any remaining operations after last token
             // should be included (they will be inserts).
-            Iterator<RewriteOperation> it = indexToOp.values().iterator();
-            while (it.hasNext()) {
-                RewriteOperation op = it.next();
+			for (RewriteOperation op : indexToOp.values()) {
                 if ( op.index >= tokens.size()-1 ) buf.append(op.text);
             }
         }
diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java
index 017b61933..e54b8d7a5 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java
@@ -29,6 +29,8 @@
 
 package org.antlr.v4.runtime;
 
+import org.antlr.v4.runtime.misc.Interval;
+
 /** A stream of tokens accessing tokens from a TokenSource */
 public interface TokenStream extends SymbolStream<Token> {
     /** Get Token at current input pointer + i ahead where i=1 is next Token.
@@ -61,18 +63,21 @@ public interface TokenStream extends SymbolStream<Token> {
 	 */
 	public TokenSource getTokenSource();
 
-	/** Return the text of all tokens from start to stop, inclusive.
-	 *  If the stream does not buffer all the tokens then it can just
-	 *  return "" or null;  Users should not access $ruleLabel.text in
-	 *  an action of course in that case.
+	/** Return the text of all tokens from within the interval.
+	 *  If the stream does not buffer all the tokens then it must
+	 *  throw UnsupportedOperationException;
+	 *  Users should not access $ruleLabel.text in an action of course in
+	 *  that case.
+	 * @param interval
 	 */
-	public String toString(int start, int stop);
+	public String getText(Interval interval);
 
 	/** Because the user is not required to use a token with an index stored
 	 *  in it, we must provide a means for two token objects themselves to
 	 *  indicate the start/end location.  Most often this will just delegate
-	 *  to the other toString(int,int).  This is also parallel with
-	 *  the TreeNodeStream.toString(Object,Object).
+	 *  to the other getText(Interval).
+	 *  If the stream does not buffer all the tokens then it must
+	 *  throw UnsupportedOperationException;
 	 */
-	public String toString(Token start, Token stop);
+	public String getText(Token start, Token stop);
 }
diff --git a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java
index 128dca261..4785aaf94 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java
@@ -29,6 +29,8 @@
 
 package org.antlr.v4.runtime;
 
+import org.antlr.v4.runtime.misc.Interval;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -195,7 +197,7 @@ public class UnbufferedCharStream implements CharStream {
     }
 
     @Override
-    public String substring(int start, int stop) {
-        return null; // map to buffer indexes
+    public String getText(Interval interval) {
+        throw new UnsupportedOperationException("unbuffered char stream");
     }
 }
diff --git a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
index 92d20ac5a..ddb84c85a 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
@@ -29,6 +29,7 @@
 
 package org.antlr.v4.runtime;
 
+import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.runtime.misc.LookaheadStream;
 
 /** A token stream that pulls tokens from the source on-demand and
@@ -78,12 +79,12 @@ public class UnbufferedTokenStream<T extends Token>
 	public TokenSource getTokenSource() { return tokenSource; }
 
     @Override
-	public String toString(int start, int stop) {
+	public String getText(Interval interval) {
         throw new UnsupportedOperationException("unbuffered stream can't give strings");
     }
 
     @Override
-	public String toString(Token start, Token stop) {
+	public String getText(Token start, Token stop) {
         throw new UnsupportedOperationException("unbuffered stream can't give strings");
     }
 
diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
index 19687e5d1..1873550a1 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
@@ -29,9 +29,15 @@
 
 package org.antlr.v4.runtime.atn;
 
-import org.antlr.v4.runtime.*;
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.IntStream;
+import org.antlr.v4.runtime.Lexer;
+import org.antlr.v4.runtime.LexerNoViableAltException;
+import org.antlr.v4.runtime.RuleContext;
+import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.runtime.dfa.DFAState;
+import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.misc.Nullable;
 
@@ -581,7 +587,7 @@ public class LexerATNSimulator extends ATNSimulator {
 		if ( dfa_debug ) {
 			System.out.format("no edge for %s\n", getTokenName(input.LA(1)));
 			System.out.format("ATN exec upon %s at DFA state %d = %s\n",
-							  input.substring(startIndex, input.index()), s.stateNumber, s.configset);
+							  input.getText(Interval.of(startIndex, input.index())), s.stateNumber, s.configset);
 		}
 
 		int ttype = execATN(input, s.configset, s);
@@ -713,7 +719,7 @@ public class LexerATNSimulator extends ATNSimulator {
 	/** Get the text of the current token */
 	@NotNull
 	public String getText(@NotNull CharStream input) {
-		return input.substring(this.startIndex, input.index());
+		return input.getText(Interval.of(startIndex, input.index()));
 	}
 
 	public int getLine() {
diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
index fc3f6c1f4..81250b5e7 100755
--- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
@@ -38,6 +38,7 @@ import org.antlr.v4.runtime.SymbolStream;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.runtime.dfa.DFAState;
+import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.runtime.misc.IntervalSet;
 import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.misc.Nullable;
@@ -396,8 +397,9 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 				if ( dfa_debug && t>=0 ) System.out.println("no edge for "+parser.getTokenNames()[t]);
 				int alt;
 				if ( dfa_debug ) {
+					Interval interval = Interval.of(startIndex, parser.getTokenStream().size()-1);
 					System.out.println("ATN exec upon "+
-                                       parser.getInputString(startIndex) +
+									   parser.getTokenStream().getText(interval) +
 									   " at DFA state "+s.stateNumber);
 				}
 
@@ -960,7 +962,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 				if ( debug ) System.out.println("Loop back; push "+config.state.stateNumber+", stack="+config.context);
 			}
 			else if ( config.state.getClass()==LoopEndState.class ) {
-				if ( debug ) System.out.println("Loop end; pop, stack="+config.context);
+				if ( debug ) System.out.println("Loop end; pop, stack=" + config.context);
 				RuleContext p = config.context;
 				LoopEndState end = (LoopEndState) config.state;
 				while ( !p.isEmpty() && p.invokingState == end.loopBackStateNumber ) {
@@ -1405,16 +1407,18 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 
 	public void reportAttemptingFullContext(DFA dfa, ATNConfigSet configs, int startIndex, int stopIndex) {
         if ( debug || retry_debug ) {
-            System.out.println("reportAttemptingFullContext decision="+dfa.decision+":"+configs+
-                               ", input="+parser.getInputString(startIndex, stopIndex));
+			Interval interval = Interval.of(startIndex, stopIndex);
+			System.out.println("reportAttemptingFullContext decision="+dfa.decision+":"+configs+
+                               ", input="+parser.getTokenStream().getText(interval));
         }
         if ( parser!=null ) parser.getErrorListenerDispatch().reportAttemptingFullContext(parser, dfa, startIndex, stopIndex, configs);
     }
 
 	public void reportContextSensitivity(DFA dfa, ATNConfigSet configs, int startIndex, int stopIndex) {
         if ( debug || retry_debug ) {
+			Interval interval = Interval.of(startIndex, stopIndex);
             System.out.println("reportContextSensitivity decision="+dfa.decision+":"+configs+
-                               ", input="+parser.getInputString(startIndex, stopIndex));
+                               ", input="+parser.getTokenStream().getText(interval));
         }
         if ( parser!=null ) parser.getErrorListenerDispatch().reportContextSensitivity(parser, dfa, startIndex, stopIndex, configs);
     }
@@ -1441,9 +1445,10 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 //				}
 //				i++;
 //			}
+			Interval interval = Interval.of(startIndex, stopIndex);
 			System.out.println("reportAmbiguity "+
 							   ambigAlts+":"+configs+
-                               ", input="+parser.getInputString(startIndex, stopIndex));
+                               ", input="+parser.getTokenStream().getText(interval));
         }
         if ( parser!=null ) parser.getErrorListenerDispatch().reportAmbiguity(parser, dfa, startIndex, stopIndex,
                                                                      ambigAlts, configs);
diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
index 267de37c6..4ab4e53f2 100644
--- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
+++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
@@ -538,12 +538,12 @@ TokenPropertyRef_int(t) ::= "(<ctx(t)>.<t.label>!=null?Integer.valueOf(<ctx(t)>.
 
 RulePropertyRef_start(r) ::= "(<ctx(r)>.<r.label>!=null?(<ctx(r)>.<r.label>.start):null)"
 RulePropertyRef_stop(r)	 ::= "(<ctx(r)>.<r.label>!=null?(<ctx(r)>.<r.label>.stop):null)"
-RulePropertyRef_text(r)	 ::= "(<ctx(r)>.<r.label>!=null?_input.toString(<ctx(r)>.<r.label>.start,<ctx(r)>.<r.label>.stop):null)"
+RulePropertyRef_text(r)	 ::= "(<ctx(r)>.<r.label>!=null?_input.getText(<ctx(r)>.<r.label>.start,<ctx(r)>.<r.label>.stop):null)"
 RulePropertyRef_ctx(r)	 ::= "<ctx(r)>.<r.label>"
 
 ThisRulePropertyRef_start(r) ::= "_localctx.start"
 ThisRulePropertyRef_stop(r)	 ::= "_localctx.stop"
-ThisRulePropertyRef_text(r)	 ::= "_input.toString(_localctx.start, _input.LT(-1))"
+ThisRulePropertyRef_text(r)	 ::= "_input.getText(_localctx.start, _input.LT(-1))"
 ThisRulePropertyRef_ctx(r)	 ::= "_localctx"
 
 NonLocalAttrRef(s)		 ::= "((<s.ruleName; format=\"cap\">Context)getInvokingContext(<s.ruleIndex>)).<s.name>"
@@ -639,7 +639,7 @@ public void <if(method.isEnter)>enter<else>exit<endif>Rule(ParseTreeListener\<<I
 
 VisitorDispatchMethod(method) ::= <<
 @Override
-public \<T> T accept(ParseTreeVisitor\<? extends T> visitor) { 
+public \<T> T accept(ParseTreeVisitor\<? extends T> visitor) {
 	if ( visitor instanceof <parser.grammarName>Visitor ) return ((<parser.grammarName>Visitor\<T>)visitor).visit<struct.derivedFromName; format="cap">(this);
 	else return null;
 }
diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java
index 08fe5b8a2..273c93ad4 100644
--- a/tool/test/org/antlr/v4/test/BaseTest.java
+++ b/tool/test/org/antlr/v4/test/BaseTest.java
@@ -35,7 +35,6 @@ import org.antlr.v4.automata.ATNPrinter;
 import org.antlr.v4.automata.LexerATNFactory;
 import org.antlr.v4.automata.ParserATNFactory;
 import org.antlr.v4.codegen.CodeGenerator;
-import org.antlr.v4.misc.Utils;
 import org.antlr.v4.runtime.ANTLRInputStream;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CommonToken;
@@ -50,6 +49,7 @@ import org.antlr.v4.runtime.atn.ATNState;
 import org.antlr.v4.runtime.atn.DecisionState;
 import org.antlr.v4.runtime.atn.LexerATNSimulator;
 import org.antlr.v4.runtime.dfa.DFA;
+import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.runtime.misc.Nullable;
 import org.antlr.v4.semantics.SemanticPipeline;
 import org.antlr.v4.tool.ANTLRMessage;
@@ -458,11 +458,11 @@ public abstract class BaseTest {
 								String input, boolean debug)
 	{
 		boolean success = rawGenerateAndBuildRecognizer(grammarFileName,
-									  grammarStr,
-									  parserName,
-									  lexerName,
-									  "-parse-listener",
-									  "-visitor");
+														grammarStr,
+														parserName,
+														lexerName,
+														"-parse-listener",
+														"-visitor");
 		assertTrue(success);
 		writeFile(tmpdir, "input", input);
 		return rawExecRecognizer(parserName,
@@ -1097,12 +1097,12 @@ public abstract class BaseTest {
 		}
 
 		@Override
-		public String toString(int start, int stop) {
+		public String getText(Interval interval) {
 			return null;
 		}
 
 		@Override
-		public String toString(Token start, Token stop) {
+		public String getText(Token start, Token stop) {
 			return null;
 		}
 	}
diff --git a/tool/test/org/antlr/v4/test/TestTokenRewriteStream.java b/tool/test/org/antlr/v4/test/TestTokenRewriteStream.java
index 6f8ad54c8..ef785bc72 100644
--- a/tool/test/org/antlr/v4/test/TestTokenRewriteStream.java
+++ b/tool/test/org/antlr/v4/test/TestTokenRewriteStream.java
@@ -29,6 +29,7 @@
 package org.antlr.v4.test;
 
 import org.antlr.v4.runtime.TokenRewriteStream;
+import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.tool.LexerGrammar;
 import org.antlr.v4.tool.interp.LexerInterpreter;
 import org.junit.Test;
@@ -158,11 +159,11 @@ public class TestTokenRewriteStream extends BaseTest {
         expecting = "x = 0;";
         assertEquals(expecting, result);
 
-        result = tokens.toString(0,9);
+        result = tokens.getText(Interval.of(0, 9));
         expecting = "x = 0;";
         assertEquals(expecting, result);
 
-        result = tokens.toString(4,8);
+        result = tokens.getText(Interval.of(4, 8));
         expecting = "0";
         assertEquals(expecting, result);
     }
@@ -193,28 +194,28 @@ public class TestTokenRewriteStream extends BaseTest {
         expecting = "x = 0 + 2 * 0;";
         assertEquals(expecting, result);
 
-        result = tokens.toString(0,17);
+        result = tokens.getText(Interval.of(0, 17));
         expecting = "x = 0 + 2 * 0;";
         assertEquals(expecting, result);
 
-        result = tokens.toString(4,8);
+        result = tokens.getText(Interval.of(4, 8));
         expecting = "0";
         assertEquals(expecting, result);
 
-        result = tokens.toString(0,8);
+        result = tokens.getText(Interval.of(0, 8));
         expecting = "x = 0";
         assertEquals(expecting, result);
 
-        result = tokens.toString(12,16);
+        result = tokens.getText(Interval.of(12, 16));
         expecting = "2 * 0";
         assertEquals(expecting, result);
 
         tokens.insertAfter(17, "// comment");
-        result = tokens.toString(12,18);
+        result = tokens.getText(Interval.of(12, 18));
         expecting = "2 * 0;// comment";
         assertEquals(expecting, result);
 
-        result = tokens.toString(0,8); // try again after insert at end
+        result = tokens.getText(Interval.of(0, 8)); // try again after insert at end
         expecting = "x = 0";
         assertEquals(expecting, result);
     }
@@ -484,7 +485,7 @@ public class TestTokenRewriteStream extends BaseTest {
 		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
 		tokens.fill();
 		tokens.replace(2, 4, "xyz");
-		String result = tokens.toString(0,6);
+		String result = tokens.getText(Interval.of(0, 6));
 		String expecting = "abxyzba";
 		assertEquals(expecting, result);
 	}

From e1870d16e9cd29dd4b4e1f3decd3ecb7af76e49b Mon Sep 17 00:00:00 2001
From: Terence Parr <parrt@antlr.org>
Date: Sun, 1 Apr 2012 12:23:45 -0700
Subject: [PATCH 05/34] added convenience getText(ctx) method to TokenStream.
 unlikley to need in CharStream and can get another way

---
 .../src/org/antlr/v4/runtime/BufferedTokenStream.java     | 3 +++
 runtime/Java/src/org/antlr/v4/runtime/TokenStream.java    | 7 ++-----
 .../src/org/antlr/v4/runtime/UnbufferedTokenStream.java   | 7 ++++++-
 tool/test/org/antlr/v4/test/BaseTest.java                 | 8 +++++++-
 4 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java
index d375c1181..50da6d8d1 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java
@@ -267,6 +267,9 @@ public class BufferedTokenStream<T extends Token> implements TokenStream {
 		return buf.toString();
     }
 
+	@Override
+	public String getText(RuleContext ctx) { return getText(ctx.getSourceInterval()); }
+
     @Override
     public String getText(Token start, Token stop) {
         if ( start!=null && stop!=null ) {
diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java
index e54b8d7a5..705674ae3 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java
@@ -43,11 +43,6 @@ public interface TokenStream extends SymbolStream<Token> {
     @Override
     public Token LT(int k);
 
-	/** How far ahead has the stream been asked to look?  The return
-	 *  value is a valid index from 0..n-1.
-	 */
-//	int range();
-
 	/** Get a token at an absolute index i; 0..n-1.  This is really only
 	 *  needed for profiling and debugging and token stream rewriting.
 	 *  If you don't want to buffer up tokens, then this method makes no
@@ -72,6 +67,8 @@ public interface TokenStream extends SymbolStream<Token> {
 	 */
 	public String getText(Interval interval);
 
+	public String getText(RuleContext ctx);
+
 	/** Because the user is not required to use a token with an index stored
 	 *  in it, we must provide a means for two token objects themselves to
 	 *  indicate the start/end location.  Most often this will just delegate
diff --git a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
index ddb84c85a..ff3adf08f 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
@@ -88,7 +88,12 @@ public class UnbufferedTokenStream<T extends Token>
         throw new UnsupportedOperationException("unbuffered stream can't give strings");
     }
 
-    @Override
+	@Override
+	public String getText(RuleContext ctx) {
+		throw new UnsupportedOperationException("unbuffered stream can't give strings");
+	}
+
+	@Override
     public int LA(int i) { return LT(i).getType(); }
 
     @Override
diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java
index 273c93ad4..dd845f858 100644
--- a/tool/test/org/antlr/v4/test/BaseTest.java
+++ b/tool/test/org/antlr/v4/test/BaseTest.java
@@ -40,6 +40,7 @@ import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CommonToken;
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.antlr.v4.runtime.Lexer;
+import org.antlr.v4.runtime.RuleContext;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.TokenSource;
 import org.antlr.v4.runtime.TokenStream;
@@ -1098,7 +1099,12 @@ public abstract class BaseTest {
 
 		@Override
 		public String getText(Interval interval) {
-			return null;
+			throw new UnsupportedOperationException("can't give strings");
+		}
+
+		@Override
+		public String getText(RuleContext ctx) {
+			throw new UnsupportedOperationException("can't give strings");
 		}
 
 		@Override

From d6e2e41c1f92e3f3fceac6948397b1aad2f9271d Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Sun, 1 Apr 2012 14:28:58 -0500
Subject: [PATCH 06/34] Fix stop index

---
 .../Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
index 81250b5e7..d324e836b 100755
--- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
@@ -397,7 +397,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 				if ( dfa_debug && t>=0 ) System.out.println("no edge for "+parser.getTokenNames()[t]);
 				int alt;
 				if ( dfa_debug ) {
-					Interval interval = Interval.of(startIndex, parser.getTokenStream().size()-1);
+					Interval interval = Interval.of(startIndex, parser.getTokenStream().index());
 					System.out.println("ATN exec upon "+
 									   parser.getTokenStream().getText(interval) +
 									   " at DFA state "+s.stateNumber);

From 8ec9ba14c911f776b5fa5141068706686e8e5695 Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Sun, 1 Apr 2012 18:22:43 -0500
Subject: [PATCH 07/34] Implement getText within marked intervals of unbuffered
 streams

---
 .../v4/runtime/UnbufferedCharStream.java      |  6 +++-
 .../v4/runtime/UnbufferedTokenStream.java     | 34 ++++++++++++++-----
 2 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java
index 4785aaf94..cf94ac16b 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java
@@ -198,6 +198,10 @@ public class UnbufferedCharStream implements CharStream {
 
     @Override
     public String getText(Interval interval) {
-        throw new UnsupportedOperationException("unbuffered char stream");
+		if (interval.a < bufferStartIndex || interval.b >= bufferStartIndex + n) {
+			throw new UnsupportedOperationException();
+		}
+
+		return new String(data, interval.a, interval.length());
     }
 }
diff --git a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
index ff3adf08f..b69fae69f 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
@@ -78,19 +78,37 @@ public class UnbufferedTokenStream<T extends Token>
     @Override
 	public TokenSource getTokenSource() { return tokenSource; }
 
-    @Override
+	@Override
 	public String getText(Interval interval) {
-        throw new UnsupportedOperationException("unbuffered stream can't give strings");
-    }
+		int bufferStartIndex = currentElementIndex - p;
+		int bufferStopIndex = bufferStartIndex + data.size() - 1;
 
-    @Override
-	public String getText(Token start, Token stop) {
-        throw new UnsupportedOperationException("unbuffered stream can't give strings");
-    }
+		int start = interval.a;
+		int stop = interval.b;
+		if (start < bufferStartIndex || stop > bufferStopIndex) {
+			throw new UnsupportedOperationException();
+		}
+
+		StringBuilder buf = new StringBuilder();
+		for (int i = start; i <= stop; i++) {
+			T t = data.get(i - bufferStartIndex);
+			buf.append(t.getText());
+		}
+
+		return buf.toString();
+	}
 
 	@Override
 	public String getText(RuleContext ctx) {
-		throw new UnsupportedOperationException("unbuffered stream can't give strings");
+		return getText(ctx.getSourceInterval());
+	}
+
+	@Override
+	public String getText(Token start, Token stop) {
+		if ( start!=null && stop!=null ) {
+			return getText(Interval.of(start.getTokenIndex(), stop.getTokenIndex()));
+		}
+		return null;
 	}
 
 	@Override

From 53c5aecb85d19bea27d446bf61b9b3c769965a36 Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Sun, 1 Apr 2012 18:23:32 -0500
Subject: [PATCH 08/34] Implement UnbufferedTokenStream.get(int) within marked
 intervals

---
 .../src/org/antlr/v4/runtime/UnbufferedTokenStream.java   | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
index b69fae69f..17a81abd9 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
@@ -116,7 +116,13 @@ public class UnbufferedTokenStream<T extends Token>
 
     @Override
     public T get(int i) {
-        throw new UnsupportedOperationException("Absolute token indexes are meaningless in an unbuffered stream");
+		int bufferStartIndex = currentElementIndex - p;
+		int bufferStopIndex = bufferStartIndex + data.size() - 1;
+		if (i < bufferStartIndex || i > bufferStopIndex) {
+			throw new UnsupportedOperationException();
+		}
+
+		return data.get(i - bufferStartIndex);
     }
 
     @Override

From adad53ee18f14f2e1c8a695c604b701bd0926a1e Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Sun, 1 Apr 2012 18:33:49 -0500
Subject: [PATCH 09/34] Fix unit tests that called TokenStream.toString (now
 TokenStream.getText)

---
 .../org/antlr/v4/test/TestActionTranslation.java | 12 ++++++------
 .../org/antlr/v4/test/TestNonGreedyLoops.java    | 16 ++++++++--------
 tool/test/org/antlr/v4/test/TestParserExec.java  | 14 +++++++-------
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/tool/test/org/antlr/v4/test/TestActionTranslation.java b/tool/test/org/antlr/v4/test/TestActionTranslation.java
index 1bba58098..150fc72eb 100644
--- a/tool/test/org/antlr/v4/test/TestActionTranslation.java
+++ b/tool/test/org/antlr/v4/test/TestActionTranslation.java
@@ -118,22 +118,22 @@ public class TestActionTranslation extends BaseTest {
 
     @Test public void testRuleRefs() throws Exception {
         String action = "$lab.start; $c.text;";
-		String expected = "(((AContext)_localctx).lab!=null?(((AContext)_localctx).lab.start):null); (((AContext)_localctx).c!=null?_input.toString(((AContext)_localctx).c.start,((AContext)_localctx).c.stop):null);";
+		String expected = "(((AContext)_localctx).lab!=null?(((AContext)_localctx).lab.start):null); (((AContext)_localctx).c!=null?_input.getText(((AContext)_localctx).c.start,((AContext)_localctx).c.stop):null);";
 		testActions(attributeTemplate, "inline", action, expected);
     }
 
 	@Test public void testRefToTextAttributeForCurrentRule() throws Exception {
         String action = "$a.text; $text";
 		String expected =
-			"(_localctx.a!=null?_input.toString(_localctx.a.start,_localctx.a.stop):" +
-			"null); _input.toString(_localctx.start, _input.LT(-1))";
+			"(_localctx.a!=null?_input.getText(_localctx.a.start,_localctx.a.stop):" +
+			"null); _input.getText(_localctx.start, _input.LT(-1))";
 		testActions(attributeTemplate, "init", action, expected);
 		expected =
-			"_input.toString(_localctx.start, _input.LT(-1)); _input.toString(_localctx.start, _input.LT(-1))";
+			"_input.getText(_localctx.start, _input.LT(-1)); _input.getText(_localctx.start, _input.LT(-1))";
 		testActions(attributeTemplate, "inline", action, expected);
 		expected =
-			"(_localctx.a!=null?_input.toString(_localctx.a.start,_localctx.a.stop):null);" +
-			" _input.toString(_localctx.start, _input.LT(-1))";
+			"(_localctx.a!=null?_input.getText(_localctx.a.start,_localctx.a.stop):null);" +
+			" _input.getText(_localctx.start, _input.LT(-1))";
 		testActions(attributeTemplate, "finally", action, expected);
     }
 
diff --git a/tool/test/org/antlr/v4/test/TestNonGreedyLoops.java b/tool/test/org/antlr/v4/test/TestNonGreedyLoops.java
index 23b6ab35f..861aaa681 100644
--- a/tool/test/org/antlr/v4/test/TestNonGreedyLoops.java
+++ b/tool/test/org/antlr/v4/test/TestNonGreedyLoops.java
@@ -35,7 +35,7 @@ public class TestNonGreedyLoops extends BaseTest {
 	@Test public void testNongreedyLoopOnEndIsNop() throws Exception {
 		String grammar =
 			"grammar T;\n" +
-			"s @after {dumpDFA();} : any ID EOF {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+			"s @after {dumpDFA();} : any ID EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 			"any : .* ;\n"+
 			"INT : '0'..'9'+ ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
@@ -58,7 +58,7 @@ public class TestNonGreedyLoops extends BaseTest {
 	@Test public void testNongreedyPlusLoopOnEndIsNop() throws Exception {
 		String grammar =
 			"grammar T;\n" +
-			"s @after {dumpDFA();} : any ID EOF {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+			"s @after {dumpDFA();} : any ID EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 			"any : .+ ;\n"+ // .+ on end of rule always gives no viable alt. can't bypass but can't match
 			"INT : '0'..'9'+ ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
@@ -196,7 +196,7 @@ public class TestNonGreedyLoops extends BaseTest {
 	@Test public void testNongreedyLoopCantSeeEOF() throws Exception {
 		String grammar =
 		"grammar T;\n" +
-		"s @after {dumpDFA();} : block EOF {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+		"s @after {dumpDFA();} : block EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 		"block : '{' .* '}' ;\n"+
 		"EQ : '=' ;\n" +
 		"INT : '0'..'9'+ ;\n" +
@@ -228,7 +228,7 @@ public class TestNonGreedyLoops extends BaseTest {
 	@Test public void testNongreedyLoop() throws Exception {
 		String grammar =
 			"grammar T;\n" +
-			"s @after {dumpDFA();} : ifstat ';' EOF {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+			"s @after {dumpDFA();} : ifstat ';' EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 			"ifstat : 'if' '(' .* ')' block ;\n" +
 			"block : '{' '}' ;\n"+
 			"EQ : '=' ;\n" +
@@ -267,7 +267,7 @@ public class TestNonGreedyLoops extends BaseTest {
 	@Test public void testNongreedyLoopPassingThroughAnotherNongreedy() throws Exception {
 		String grammar =
 			"grammar T;\n" +
-			"s @after {dumpDFA();} : ifstat ';' EOF {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+			"s @after {dumpDFA();} : ifstat ';' EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 			"ifstat : 'if' '(' .* ')' block ;\n" +
 			"block : '{' (block|.)* '}' ;\n"+
 			"EQ : '=' ;\n" +
@@ -330,7 +330,7 @@ public class TestNonGreedyLoops extends BaseTest {
 		// EOF on end means LL(*) can identify when to stop the loop.
 		String grammar =
 			"grammar T;\n" +
-			"s @after {dumpDFA();} : stat* ID '=' ID ';' EOF {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+			"s @after {dumpDFA();} : stat* ID '=' ID ';' EOF {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 			"stat : 'if' '(' INT ')' stat\n" +
 			"     | 'return' INT ';'\n" +
 			"     | ID '=' (INT|ID) ';'\n" +
@@ -392,7 +392,7 @@ public class TestNonGreedyLoops extends BaseTest {
 			"grammar T;\n" +
 			"random : s ;" + // call s so s isn't followed by EOF directly
 			"s @after {dumpDFA();} : (options {greedy=false;} : stat)* ID '=' ID ';'\n" +
-			"    {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+			"    {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 			"stat : 'if' '(' INT ')' stat\n" +
 			"     | 'return' INT ';'\n" +
 			"     | ID '=' (INT|ID) ';'\n" +
@@ -450,7 +450,7 @@ public class TestNonGreedyLoops extends BaseTest {
 	@Test public void testHTMLTags() throws Exception {
 		String grammar =
 			"grammar T;\n" +
-			"s @after {dumpDFA();} : (item)+ {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+			"s @after {dumpDFA();} : (item)+ {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 			"item : tag | . ;\n" +
 			"tag : '<' '/'? .* '>'  ;\n" +
 			"EQ : '=' ;\n" +
diff --git a/tool/test/org/antlr/v4/test/TestParserExec.java b/tool/test/org/antlr/v4/test/TestParserExec.java
index 0df311bc6..45c27422a 100644
--- a/tool/test/org/antlr/v4/test/TestParserExec.java
+++ b/tool/test/org/antlr/v4/test/TestParserExec.java
@@ -52,7 +52,7 @@ public class TestParserExec extends BaseTest {
 	@Test public void testBasic() throws Exception {
 		String grammar =
 			"grammar T;\n" +
-			"a : ID INT {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+			"a : ID INT {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +
 			"WS : (' '|'\\n') {skip();} ;\n";
@@ -80,7 +80,7 @@ public class TestParserExec extends BaseTest {
 	@Test public void testAPlus() throws Exception {
 		String grammar =
 			"grammar T;\n" +
-			"a : ID+ {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+			"a : ID+ {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"WS : (' '|'\\n') {skip();} ;\n";
 
@@ -93,7 +93,7 @@ public class TestParserExec extends BaseTest {
 	@Test public void testAorAPlus() throws Exception {
 		String grammar =
 			"grammar T;\n" +
-			"a : (ID|ID)+ {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+			"a : (ID|ID)+ {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"WS : (' '|'\\n') {skip();} ;\n";
 
@@ -105,7 +105,7 @@ public class TestParserExec extends BaseTest {
 	@Test public void testAStar() throws Exception {
 		String grammar =
 			"grammar T;\n" +
-			"a : ID* {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+			"a : ID* {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"WS : (' '|'\\n') {skip();} ;\n";
 
@@ -121,7 +121,7 @@ public class TestParserExec extends BaseTest {
 	@Test public void testAorAStar() throws Exception {
 		String grammar =
 			"grammar T;\n" +
-			"a : (ID|ID)* {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+			"a : (ID|ID)* {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"WS : (' '|'\\n') {skip();} ;\n";
 
@@ -136,7 +136,7 @@ public class TestParserExec extends BaseTest {
 	@Test public void testAorBPlus() throws Exception {
 		String grammar =
 			"grammar T;\n" +
-			"a : (ID|INT{;})+ {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+			"a : (ID|INT{;})+ {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +
 			"WS : (' '|'\\n') {skip();} ;\n";
@@ -149,7 +149,7 @@ public class TestParserExec extends BaseTest {
 	@Test public void testAorBStar() throws Exception {
 		String grammar =
 			"grammar T;\n" +
-			"a : (ID|INT{;})* {System.out.println(_input.toString(0,_input.index()-1));} ;\n" +
+			"a : (ID|INT{;})* {System.out.println(_input.getText(Interval.of(0,_input.index()-1)));} ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +
 			"WS : (' '|'\\n') {skip();} ;\n";

From 5fd8b223b66a5a0d1c24df537d14d8619c818438 Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Mon, 2 Apr 2012 18:13:47 -0500
Subject: [PATCH 10/34] Fix ErrorManager ignoring error code in messages

---
 tool/src/org/antlr/v4/tool/ErrorManager.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tool/src/org/antlr/v4/tool/ErrorManager.java b/tool/src/org/antlr/v4/tool/ErrorManager.java
index e52bd52ee..2229ff1c4 100644
--- a/tool/src/org/antlr/v4/tool/ErrorManager.java
+++ b/tool/src/org/antlr/v4/tool/ErrorManager.java
@@ -131,7 +131,7 @@ public class ErrorManager {
 			locationValid = true;
 		}
 
-		messageFormatST.add("id", msg.errorType.ordinal());
+		messageFormatST.add("id", msg.errorType.code);
 		messageFormatST.add("text", messageST);
 
 		if (locationValid) reportST.add("location", locationST);

From 78fcc134cdcd2a68e667e908a19cd8d6e8b2c0c2 Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Mon, 2 Apr 2012 18:22:05 -0500
Subject: [PATCH 11/34] Fix error codes in unit tests

---
 .../antlr/v4/test/TestAttributeChecks.java    | 62 +++++++++----------
 .../v4/test/TestBasicSemanticErrors.java      | 24 +++----
 .../org/antlr/v4/test/TestSymbolIssues.java   | 28 ++++-----
 .../antlr/v4/test/TestToolSyntaxErrors.java   | 40 ++++++------
 4 files changed, 77 insertions(+), 77 deletions(-)

diff --git a/tool/test/org/antlr/v4/test/TestAttributeChecks.java b/tool/test/org/antlr/v4/test/TestAttributeChecks.java
index 071ab27aa..2d00e7ad3 100644
--- a/tool/test/org/antlr/v4/test/TestAttributeChecks.java
+++ b/tool/test/org/antlr/v4/test/TestAttributeChecks.java
@@ -24,8 +24,8 @@ public class TestAttributeChecks extends BaseTest {
 		"d	 :   ;\n";
 
     String[] membersChecks = {
-		"$a",			"error(29): A.g4:2:11: unknown attribute reference a in $a\n",
-        "$a.y",			"error(29): A.g4:2:11: unknown attribute reference a in $a.y\n",
+		"$a",			"error(63): A.g4:2:11: unknown attribute reference a in $a\n",
+        "$a.y",			"error(63): A.g4:2:11: unknown attribute reference a in $a.y\n",
     };
 
     String[] initChecks = {
@@ -36,8 +36,8 @@ public class TestAttributeChecks extends BaseTest {
 		"$lab.e",		"",
 		"$ids",			"",
 
-		"$c",			"error(29): A.g4:4:8: unknown attribute reference c in $c\n",
-		"$a.q",			"error(31): A.g4:4:10: unknown attribute q for rule a in $a.q\n",
+		"$c",			"error(63): A.g4:4:8: unknown attribute reference c in $c\n",
+		"$a.q",			"error(65): A.g4:4:10: unknown attribute q for rule a in $a.q\n",
     };
 
 	String[] inlineChecks = {
@@ -58,19 +58,19 @@ public class TestAttributeChecks extends BaseTest {
 	};
 
 	String[] bad_inlineChecks = {
-		"$lab",			"error(33): A.g4:6:4: missing attribute access on rule reference lab in $lab\n",
-		"$q",           "error(29): A.g4:6:4: unknown attribute reference q in $q\n",
-		"$q.y",         "error(29): A.g4:6:4: unknown attribute reference q in $q.y\n",
-		"$q = 3",       "error(29): A.g4:6:4: unknown attribute reference q in $q\n",
-		"$q = 3;",      "error(29): A.g4:6:4: unknown attribute reference q in $q = 3;\n",
-		"$q.y = 3;",    "error(29): A.g4:6:4: unknown attribute reference q in $q.y = 3;\n",
-		"$q = $blort;", "error(29): A.g4:6:4: unknown attribute reference q in $q = $blort;\n" +
-						"error(29): A.g4:6:9: unknown attribute reference blort in $blort\n",
-		"$a.ick",       "error(31): A.g4:6:6: unknown attribute ick for rule a in $a.ick\n",
-		"$a.ick = 3;",  "error(31): A.g4:6:6: unknown attribute ick for rule a in $a.ick = 3;\n",
-		"$b.d",         "error(30): A.g4:6:6: cannot access rule d's parameter: $b.d\n",  // can't see rule ref's arg
-		"$d.text",      "error(29): A.g4:6:4: unknown attribute reference d in $d.text\n", // valid rule, but no ref
-		"$lab.d",		"error(30): A.g4:6:8: cannot access rule d's parameter: $lab.d\n",
+		"$lab",			"error(67): A.g4:6:4: missing attribute access on rule reference lab in $lab\n",
+		"$q",           "error(63): A.g4:6:4: unknown attribute reference q in $q\n",
+		"$q.y",         "error(63): A.g4:6:4: unknown attribute reference q in $q.y\n",
+		"$q = 3",       "error(63): A.g4:6:4: unknown attribute reference q in $q\n",
+		"$q = 3;",      "error(63): A.g4:6:4: unknown attribute reference q in $q = 3;\n",
+		"$q.y = 3;",    "error(63): A.g4:6:4: unknown attribute reference q in $q.y = 3;\n",
+		"$q = $blort;", "error(63): A.g4:6:4: unknown attribute reference q in $q = $blort;\n" +
+						"error(63): A.g4:6:9: unknown attribute reference blort in $blort\n",
+		"$a.ick",       "error(65): A.g4:6:6: unknown attribute ick for rule a in $a.ick\n",
+		"$a.ick = 3;",  "error(65): A.g4:6:6: unknown attribute ick for rule a in $a.ick = 3;\n",
+		"$b.d",         "error(64): A.g4:6:6: cannot access rule d's parameter: $b.d\n",  // can't see rule ref's arg
+		"$d.text",      "error(63): A.g4:6:4: unknown attribute reference d in $d.text\n", // valid rule, but no ref
+		"$lab.d",		"error(64): A.g4:6:8: cannot access rule d's parameter: $lab.d\n",
 	};
 
 	String[] finallyChecks = {
@@ -84,20 +84,20 @@ public class TestAttributeChecks extends BaseTest {
 		"$id.text",		"",
 		"$ids",			"",
 
-		"$lab",			"error(33): A.g4:9:14: missing attribute access on rule reference lab in $lab\n",
-		"$q",           "error(29): A.g4:9:14: unknown attribute reference q in $q\n",
-		"$q.y",         "error(29): A.g4:9:14: unknown attribute reference q in $q.y\n",
-		"$q = 3",       "error(29): A.g4:9:14: unknown attribute reference q in $q\n",
-		"$q = 3;",      "error(29): A.g4:9:14: unknown attribute reference q in $q = 3;\n",
-		"$q.y = 3;",    "error(29): A.g4:9:14: unknown attribute reference q in $q.y = 3;\n",
-		"$q = $blort;", "error(29): A.g4:9:14: unknown attribute reference q in $q = $blort;\n" +
-						"error(29): A.g4:9:19: unknown attribute reference blort in $blort\n",
-		"$a.ick",       "error(31): A.g4:9:16: unknown attribute ick for rule a in $a.ick\n",
-		"$a.ick = 3;",  "error(31): A.g4:9:16: unknown attribute ick for rule a in $a.ick = 3;\n",
-		"$b.e",			"error(29): A.g4:9:14: unknown attribute reference b in $b.e\n", // can't see rule refs outside alts
-		"$b.d",         "error(29): A.g4:9:14: unknown attribute reference b in $b.d\n",
-		"$c.text",      "error(29): A.g4:9:14: unknown attribute reference c in $c.text\n",
-		"$lab.d",		"error(30): A.g4:9:18: cannot access rule d's parameter: $lab.d\n",
+		"$lab",			"error(67): A.g4:9:14: missing attribute access on rule reference lab in $lab\n",
+		"$q",           "error(63): A.g4:9:14: unknown attribute reference q in $q\n",
+		"$q.y",         "error(63): A.g4:9:14: unknown attribute reference q in $q.y\n",
+		"$q = 3",       "error(63): A.g4:9:14: unknown attribute reference q in $q\n",
+		"$q = 3;",      "error(63): A.g4:9:14: unknown attribute reference q in $q = 3;\n",
+		"$q.y = 3;",    "error(63): A.g4:9:14: unknown attribute reference q in $q.y = 3;\n",
+		"$q = $blort;", "error(63): A.g4:9:14: unknown attribute reference q in $q = $blort;\n" +
+						"error(63): A.g4:9:19: unknown attribute reference blort in $blort\n",
+		"$a.ick",       "error(65): A.g4:9:16: unknown attribute ick for rule a in $a.ick\n",
+		"$a.ick = 3;",  "error(65): A.g4:9:16: unknown attribute ick for rule a in $a.ick = 3;\n",
+		"$b.e",			"error(63): A.g4:9:14: unknown attribute reference b in $b.e\n", // can't see rule refs outside alts
+		"$b.d",         "error(63): A.g4:9:14: unknown attribute reference b in $b.d\n",
+		"$c.text",      "error(63): A.g4:9:14: unknown attribute reference c in $c.text\n",
+		"$lab.d",		"error(64): A.g4:9:18: cannot access rule d's parameter: $lab.d\n",
 	};
 
 	String[] dynMembersChecks = {
diff --git a/tool/test/org/antlr/v4/test/TestBasicSemanticErrors.java b/tool/test/org/antlr/v4/test/TestBasicSemanticErrors.java
index 4b4ad4cb5..c8903c9e1 100644
--- a/tool/test/org/antlr/v4/test/TestBasicSemanticErrors.java
+++ b/tool/test/org/antlr/v4/test/TestBasicSemanticErrors.java
@@ -50,18 +50,18 @@ public class TestBasicSemanticErrors extends BaseTest {
         "b : ( options { ick=bar; greedy=true; } : ID )+ ;\n" +
         "c : ID<blue> ID<x=y> ;",
         // YIELDS
-		"warning(48): U.g4:2:10: illegal option foo\n" +
-		"warning(48): U.g4:2:19: illegal option k\n" +
-		"error(26): U.g4:4:8: token names must start with an uppercase letter: f\n" +
-		"error(25): U.g4:4:8: can't assign string value to token name f in non-combined grammar\n" +
-		"error(25): U.g4:5:8: can't assign string value to token name S in non-combined grammar\n" +
-		"warning(48): U.g4:8:10: illegal option x\n" +
-		"error(20): U.g4:8:0: repeated grammar prequel spec (option, token, or import); please merge\n" +
-		"error(20): U.g4:7:0: repeated grammar prequel spec (option, token, or import); please merge\n" +
-		"warning(48): U.g4:11:10: illegal option blech\n" +
-		"warning(48): U.g4:11:21: illegal option greedy\n" +
-		"warning(48): U.g4:14:16: illegal option ick\n" +
-		"warning(48): U.g4:15:16: illegal option x\n",
+		"warning(83): U.g4:2:10: illegal option foo\n" +
+		"warning(83): U.g4:2:19: illegal option k\n" +
+		"error(60): U.g4:4:8: token names must start with an uppercase letter: f\n" +
+		"error(59): U.g4:4:8: can't assign string value to token name f in non-combined grammar\n" +
+		"error(59): U.g4:5:8: can't assign string value to token name S in non-combined grammar\n" +
+		"warning(83): U.g4:8:10: illegal option x\n" +
+		"error(54): U.g4:8:0: repeated grammar prequel spec (option, token, or import); please merge\n" +
+		"error(54): U.g4:7:0: repeated grammar prequel spec (option, token, or import); please merge\n" +
+		"warning(83): U.g4:11:10: illegal option blech\n" +
+		"warning(83): U.g4:11:21: illegal option greedy\n" +
+		"warning(83): U.g4:14:16: illegal option ick\n" +
+		"warning(83): U.g4:15:16: illegal option x\n",
     };
 
 	@Test public void testU() { super.testErrors(U, false); }
diff --git a/tool/test/org/antlr/v4/test/TestSymbolIssues.java b/tool/test/org/antlr/v4/test/TestSymbolIssues.java
index a7af0fe8c..75dc79304 100644
--- a/tool/test/org/antlr/v4/test/TestSymbolIssues.java
+++ b/tool/test/org/antlr/v4/test/TestSymbolIssues.java
@@ -20,9 +20,9 @@ public class TestSymbolIssues extends BaseTest {
         "\n" +
         "ID : 'a'..'z'+ ID ;",
         // YIELDS
-        "warning(48): A.g4:2:10: illegal option opt\n" +
-        "warning(48): A.g4:2:21: illegal option k\n" +
-		"error(59): A.g4:7:1: redefinition of header action\n" +
+        "warning(83): A.g4:2:10: illegal option opt\n" +
+        "warning(83): A.g4:2:21: illegal option k\n" +
+		"error(94): A.g4:7:1: redefinition of header action\n" +
 		"warning(51): A.g4:2:10: illegal option opt\n" +
 		"error(19): A.g4:11:0: rule a redefinition\n" +
 		"error(60): A.g4:5:1: redefinition of members action\n" +
@@ -42,11 +42,11 @@ public class TestSymbolIssues extends BaseTest {
         "\n" +
         "s : FOO ;",
         // YIELDS
-        "error(25): B.g4:2:9: can't assign string value to token name X in non-combined grammar\n" +
-		"error(35): B.g4:4:4: label s conflicts with rule with same name\n" +
-		"error(35): B.g4:4:9: label b conflicts with rule with same name\n" +
-		"error(36): B.g4:4:15: label X conflicts with token with same name\n" +
-		"error(40): B.g4:6:9: label x type mismatch with previous definition: TOKEN_LIST_LABEL!=TOKEN_LABEL\n"
+        "error(59): B.g4:2:9: can't assign string value to token name X in non-combined grammar\n" +
+		"error(69): B.g4:4:4: label s conflicts with rule with same name\n" +
+		"error(69): B.g4:4:9: label b conflicts with rule with same name\n" +
+		"error(70): B.g4:4:15: label X conflicts with token with same name\n" +
+		"error(75): B.g4:6:9: label x type mismatch with previous definition: TOKEN_LIST_LABEL!=TOKEN_LABEL\n"
     };
 
     static String[] D = {
@@ -61,8 +61,8 @@ public class TestSymbolIssues extends BaseTest {
         "        :       ID ;",
 
         // YIELDS
-        "error(37): D.g4:3:21: label j conflicts with rule a's return value or parameter with same name\n" +
-		"error(41): D.g4:5:0: rule b's argument i conflicts a return value with same name\n"
+        "error(72): D.g4:3:21: label j conflicts with rule a's return value or parameter with same name\n" +
+		"error(76): D.g4:5:0: rule b's argument i conflicts a return value with same name\n"
     };
 
 	static String[] E = {
@@ -78,10 +78,10 @@ public class TestSymbolIssues extends BaseTest {
 		"a : A ;\n",
 
 		// YIELDS
-		"error(73): E.g4:4:8: cannot redefine B; token name already defined\n" +
-		"error(73): E.g4:5:4: cannot redefine C; token name already defined\n" +
-		"error(73): E.g4:6:8: cannot redefine D; token name already defined\n" +
-		"error(72): E.g4:7:8: cannot alias X='e'; string already assigned to E\n"
+		"error(108): E.g4:4:8: cannot redefine B; token name already defined\n" +
+		"error(108): E.g4:5:4: cannot redefine C; token name already defined\n" +
+		"error(108): E.g4:6:8: cannot redefine D; token name already defined\n" +
+		"error(107): E.g4:7:8: cannot alias X='e'; string already assigned to E\n"
 	};
 
     @Test public void testA() { super.testErrors(A, false); }
diff --git a/tool/test/org/antlr/v4/test/TestToolSyntaxErrors.java b/tool/test/org/antlr/v4/test/TestToolSyntaxErrors.java
index 6b7e3fa65..bee018fc6 100644
--- a/tool/test/org/antlr/v4/test/TestToolSyntaxErrors.java
+++ b/tool/test/org/antlr/v4/test/TestToolSyntaxErrors.java
@@ -8,37 +8,37 @@ public class TestToolSyntaxErrors extends BaseTest {
         "grammar A;\n" +
         "",
         // YIELDS
-        "error(64): A.g4::: grammar A has no rules\n",
+        "error(99): A.g4::: grammar A has no rules\n",
 
 		"A;",
-		"error(16): <string>:1:0: 'A' came as a complete surprise to me\n",
+		"error(50): <string>:1:0: 'A' came as a complete surprise to me\n",
 
 		"grammar ;",
-		"error(16): <string>:1:8: ';' came as a complete surprise to me while looking for an identifier\n",
+		"error(50): <string>:1:8: ';' came as a complete surprise to me while looking for an identifier\n",
 
 		"grammar A\n" +
 		"a : ID ;\n",
-		"error(16): <string>:2:0: missing SEMI at 'a'\n",
+		"error(50): <string>:2:0: missing SEMI at 'a'\n",
 
 		"grammar A;\n" +
 		"a : ID ;;\n"+
 		"b : B ;",
-		"error(16): A.g4:2:8: ';' came as a complete surprise to me\n",
+		"error(50): A.g4:2:8: ';' came as a complete surprise to me\n",
 
 		"grammar A;;\n" +
 		"a : ID ;\n",
-		"error(16): A;.g4:1:10: ';' came as a complete surprise to me\n",
+		"error(50): A;.g4:1:10: ';' came as a complete surprise to me\n",
 
 		"grammar A;\n" +
 		"a @init : ID ;\n",
-		"error(16): A.g4:2:8: mismatched input ':' expecting ACTION while matching rule preamble\n",
+		"error(50): A.g4:2:8: mismatched input ':' expecting ACTION while matching rule preamble\n",
 
 		"grammar A;\n" +
 		"a  ( A | B ) D ;\n" +
 		"b : B ;",
-		"error(16): A.g4:2:3: '(' came as a complete surprise to me while matching rule preamble\n" +
-		"error(16): A.g4:2:11: mismatched input ')' expecting SEMI while matching a rule\n" +
-		"error(16): A.g4:2:15: mismatched input ';' expecting COLON while matching a lexer rule\n",
+		"error(50): A.g4:2:3: '(' came as a complete surprise to me while matching rule preamble\n" +
+		"error(50): A.g4:2:11: mismatched input ')' expecting SEMI while matching a rule\n" +
+		"error(50): A.g4:2:15: mismatched input ';' expecting COLON while matching a lexer rule\n",
     };
 
 	@Test public void testA() { super.testErrors(A, true); }
@@ -48,7 +48,7 @@ public class TestToolSyntaxErrors extends BaseTest {
 			"grammar A;\n" +
 			"a : : A ;\n" +
 			"b : B ;",
-			"error(16): A.g4:2:4: ':' came as a complete surprise to me while matching alternative\n",
+			"error(50): A.g4:2:4: ':' came as a complete surprise to me while matching alternative\n",
 		};
 		super.testErrors(pair, true);
 	}
@@ -58,7 +58,7 @@ public class TestToolSyntaxErrors extends BaseTest {
 			"grammar A;\n" +
 			"a : A \n" +
 			"b : B ;",
-			"error(16): A.g4:3:0: unterminated rule (missing ';') detected at 'b :' while looking for rule element\n",
+			"error(50): A.g4:3:0: unterminated rule (missing ';') detected at 'b :' while looking for rule element\n",
 		};
 		super.testErrors(pair, true);
 	}
@@ -68,7 +68,7 @@ public class TestToolSyntaxErrors extends BaseTest {
 			"lexer grammar A;\n" +
 			"A : 'a' \n" +
 			"B : 'b' ;",
-			"error(16): A.g4:3:0: unterminated rule (missing ';') detected at 'B :' while looking for lexer rule element\n",
+			"error(50): A.g4:3:0: unterminated rule (missing ';') detected at 'B :' while looking for lexer rule element\n",
 		};
 		super.testErrors(pair, true);
 	}
@@ -78,7 +78,7 @@ public class TestToolSyntaxErrors extends BaseTest {
 			"grammar A;\n" +
 			"a : A \n" +
 			"b[int i] returns [int y] : B ;",
-			"error(16): A.g4:3:9: unterminated rule (missing ';') detected at 'returns int y' while looking for rule element\n"
+			"error(50): A.g4:3:9: unterminated rule (missing ';') detected at 'returns int y' while looking for rule element\n"
 		};
 		super.testErrors(pair, true);
 	}
@@ -90,7 +90,7 @@ public class TestToolSyntaxErrors extends BaseTest {
 			"  catch [Exception e] {...}\n" +
 			"b : B ;\n",
 
-			"error(16): A.g4:2:4: unterminated rule (missing ';') detected at 'b catch' while looking for rule element\n"
+			"error(50): A.g4:2:4: unterminated rule (missing ';') detected at 'b catch' while looking for rule element\n"
 		};
 		super.testErrors(pair, true);
 	}
@@ -101,7 +101,7 @@ public class TestToolSyntaxErrors extends BaseTest {
 			"a : A \n" +
 			"  catch [Exception e] {...}\n",
 
-			"error(16): A.g4:2:4: unterminated rule (missing ';') detected at 'A catch' while looking for rule element\n"
+			"error(50): A.g4:2:4: unterminated rule (missing ';') detected at 'A catch' while looking for rule element\n"
 		};
 		super.testErrors(pair, true);
 	}
@@ -112,7 +112,7 @@ public class TestToolSyntaxErrors extends BaseTest {
 			"a @ options {k=1;} : A ;\n" +
 			"b : B ;",
 
-			"error(16): A.g4:2:4: 'options {' came as a complete surprise to me while looking for an identifier\n"
+			"error(50): A.g4:2:4: 'options {' came as a complete surprise to me while looking for an identifier\n"
 		};
 		super.testErrors(pair, true);
 	}
@@ -123,7 +123,7 @@ public class TestToolSyntaxErrors extends BaseTest {
 			"a } : A ;\n" +
 			"b : B ;",
 
-			"error(16): A.g4:2:2: '}' came as a complete surprise to me while matching rule preamble\n"
+			"error(50): A.g4:2:2: '}' came as a complete surprise to me while matching rule preamble\n"
 		};
 		super.testErrors(pair, true);
 	}
@@ -135,8 +135,8 @@ public class TestToolSyntaxErrors extends BaseTest {
 			"mode foo;\n" +
 			"b : B ;",
 
-			"error(16): A.g4:4:0: 'b' came as a complete surprise to me\n" +
-			"error(16): A.g4:4:6: mismatched input ';' expecting COLON while matching a lexer rule\n"
+			"error(50): A.g4:4:0: 'b' came as a complete surprise to me\n" +
+			"error(50): A.g4:4:6: mismatched input ';' expecting COLON while matching a lexer rule\n"
 		};
 		super.testErrors(pair, true);
 	}

From 3849cd572d3fd7ef0d99048051bb27a43f50158a Mon Sep 17 00:00:00 2001
From: Terence Parr <parrt@antlr.org>
Date: Mon, 2 Apr 2012 16:49:04 -0700
Subject: [PATCH 12/34] rm SymbolStream<T>, change all to TokenStream. all but
 5 tests pass.

---
 .../v4/runtime/DefaultErrorStrategy.java      |  4 +-
 .../v4/runtime/NoViableAltException.java      | 16 +++---
 .../org/antlr/v4/runtime/SymbolStream.java    | 51 -------------------
 .../src/org/antlr/v4/runtime/TokenStream.java |  4 +-
 .../v4/runtime/atn/ParserATNSimulator.java    | 16 +++---
 .../v4/tool/interp/ParserInterpreter.java     | 14 +++--
 6 files changed, 29 insertions(+), 76 deletions(-)
 delete mode 100644 runtime/Java/src/org/antlr/v4/runtime/SymbolStream.java

diff --git a/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java b/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java
index 40557dc22..13e0add35 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java
@@ -165,7 +165,7 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy {
 		// If already recovering, don't try to sync
         if ( errorRecoveryMode ) return;
 
-        SymbolStream<Token> tokens = recognizer.getInputStream();
+        TokenStream tokens = recognizer.getInputStream();
         int la = tokens.LA(1);
 
         // try cheaper subset first; might get lucky. seems to shave a wee bit off
@@ -201,7 +201,7 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy {
 										  NoViableAltException e)
 	throws RecognitionException
 	{
-		SymbolStream<Token> tokens = recognizer.getInputStream();
+		TokenStream tokens = recognizer.getInputStream();
 		String input;
 		if (tokens instanceof TokenStream) {
 			if ( e.startToken.getType()==Token.EOF ) input = "<EOF>";
diff --git a/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java b/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java
index 6b9cc102c..661ee961c 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java
@@ -41,10 +41,10 @@ public class NoViableAltException extends RecognitionException {
 	 * 	not be buffering tokens so get a reference to it. (At the
 	 *  time the error occurred, of course the stream needs to keep a
 	 *  buffer all of the tokens but later we might not have access to those.)
- 	 */
+	 */
 	public Token startToken;
 
-	public <Symbol extends Token> NoViableAltException(Parser recognizer) { // LL(1) error
+	public NoViableAltException(Parser recognizer) { // LL(1) error
 		this(recognizer,recognizer.getInputStream(),
 			 recognizer.getCurrentToken(),
 			 recognizer.getCurrentToken(),
@@ -52,12 +52,12 @@ public class NoViableAltException extends RecognitionException {
 			 recognizer._ctx);
 	}
 
-	public <Symbol> NoViableAltException(Parser recognizer,
-										 SymbolStream<Symbol> input,
-										 Token startToken,
-										 Token offendingToken,
-										 ATNConfigSet deadEndConfigs,
-										 ParserRuleContext<?> ctx)
+	public NoViableAltException(Parser recognizer,
+								TokenStream input,
+								Token startToken,
+								Token offendingToken,
+								ATNConfigSet deadEndConfigs,
+								ParserRuleContext<?> ctx)
 	{
 		super(recognizer, input, ctx);
 		this.deadEndConfigs = deadEndConfigs;
diff --git a/runtime/Java/src/org/antlr/v4/runtime/SymbolStream.java b/runtime/Java/src/org/antlr/v4/runtime/SymbolStream.java
deleted file mode 100644
index fcab38a0f..000000000
--- a/runtime/Java/src/org/antlr/v4/runtime/SymbolStream.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- [The "BSD license"]
- Copyright (c) 2011 Terence Parr
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
-
- 1. Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
- 3. The name of the author may not be used to endorse or promote products
-    derived from this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-package org.antlr.v4.runtime;
-
-/**  A stream of either tokens or tree nodes */
-public interface SymbolStream<T> extends IntStream {
-	/** Get the symbol at absolute index i; 0..n-1.
-	 *  This is only valid if the underlying stream implementation buffers
-	 *  all of the incoming objects.
-	 *
-	 *  @throws UnsupportedOperationException if the index {@code i} is outside
-	 *    the marked region and the stream does not support accessing symbols by
-	 *    index outside of marked regions.
-	 */
-	public T get(int i);
-
-	/** Get symbol at current input pointer + {@code k} ahead where {@code k=1}
-	 *  is next symbol. k&lt;0 indicates objects in the past.  So -1 is previous
-	 *  Object and -2 is two Objects ago. {@code LT(0)} is undefined.  For i>=n,
-	 *  return an object representing EOF. Return {@code null} for {@code LT(0)}
-	 *  and any index that results in an absolute index that is negative.
-	 */
-	T LT(int k);
-}
diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java
index 705674ae3..0dc079189 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java
@@ -32,7 +32,7 @@ package org.antlr.v4.runtime;
 import org.antlr.v4.runtime.misc.Interval;
 
 /** A stream of tokens accessing tokens from a TokenSource */
-public interface TokenStream extends SymbolStream<Token> {
+public interface TokenStream extends IntStream {
     /** Get Token at current input pointer + i ahead where i=1 is next Token.
 	 *  i&lt;0 indicates tokens in the past.  So -1 is previous token and -2 is
 	 *  two tokens ago. LT(0) is undefined.  For i>=n, return Token.EOFToken.
@@ -40,7 +40,6 @@ public interface TokenStream extends SymbolStream<Token> {
 	 *  that is negative.
      *  TODO (Sam): Throw exception for invalid k?
 	 */
-    @Override
     public Token LT(int k);
 
 	/** Get a token at an absolute index i; 0..n-1.  This is really only
@@ -50,7 +49,6 @@ public interface TokenStream extends SymbolStream<Token> {
 	 *  I believe DebugTokenStream can easily be altered to not use
 	 *  this method, removing the dependency.
 	 */
-	@Override
 	public Token get(int i);
 
 	/** Where is this stream pulling tokens from?  This is not the name, but
diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
index d324e836b..5d5e3b3db 100755
--- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
@@ -34,8 +34,8 @@ import org.antlr.v4.runtime.NoViableAltException;
 import org.antlr.v4.runtime.Parser;
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.RuleContext;
-import org.antlr.v4.runtime.SymbolStream;
 import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.TokenStream;
 import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.runtime.dfa.DFAState;
 import org.antlr.v4.runtime.misc.Interval;
@@ -286,7 +286,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 	public void reset() {
 	}
 
-	public int adaptivePredict(@NotNull SymbolStream<? extends Symbol> input, int decision,
+	public int adaptivePredict(@NotNull TokenStream input, int decision,
 							   @Nullable ParserRuleContext<?> outerContext)
 	{
 		predict_calls++;
@@ -312,7 +312,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 		}
 	}
 
-	public int predictATN(@NotNull DFA dfa, @NotNull SymbolStream<? extends Symbol> input,
+	public int predictATN(@NotNull DFA dfa, @NotNull TokenStream input,
 						  @Nullable ParserRuleContext<?> outerContext)
 	{
 		if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY;
@@ -344,7 +344,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 	}
 
 	public int execDFA(@NotNull DFA dfa, @NotNull DFAState s0,
-					   @NotNull SymbolStream<? extends Symbol> input, int startIndex,
+					   @NotNull TokenStream input, int startIndex,
                        @Nullable ParserRuleContext<?> outerContext)
     {
 		if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY;
@@ -501,7 +501,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 
 	 */
 	public int execATN(@NotNull DFA dfa, @NotNull DFAState s0,
-					   @NotNull SymbolStream<? extends Symbol> input, int startIndex,
+					   @NotNull TokenStream input, int startIndex,
 					   ParserRuleContext<?> outerContext)
 	{
 		if ( debug ) System.out.println("execATN decision "+dfa.decision+" exec LA(1)=="+ getLookaheadName(input));
@@ -628,7 +628,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 	public ATNConfigSet execATNWithFullContext(DFA dfa,
 											   DFAState D, // how far we got before failing over
 											   @NotNull ATNConfigSet s0,
-											   @NotNull SymbolStream<? extends Symbol> input, int startIndex,
+											   @NotNull TokenStream input, int startIndex,
 											   ParserRuleContext<?> outerContext,
 											   int nalts,
 											   boolean greedy)
@@ -1295,7 +1295,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 		return String.valueOf(t);
 	}
 
-	public String getLookaheadName(SymbolStream<? extends Symbol> input) {
+	public String getLookaheadName(TokenStream input) {
 		return getTokenName(input.LA(1));
 	}
 
@@ -1320,7 +1320,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 	}
 
 	@NotNull
-	public NoViableAltException noViableAlt(@NotNull SymbolStream<? extends Symbol> input,
+	public NoViableAltException noViableAlt(@NotNull TokenStream input,
 											@NotNull ParserRuleContext<?> outerContext,
 											@NotNull ATNConfigSet configs,
 											int startIndex)
diff --git a/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java b/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java
index aaf5d2d4d..21318e151 100644
--- a/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java
+++ b/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java
@@ -30,8 +30,14 @@
 package org.antlr.v4.tool.interp;
 
 import org.antlr.v4.Tool;
-import org.antlr.v4.runtime.*;
-import org.antlr.v4.runtime.atn.*;
+import org.antlr.v4.runtime.Parser;
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.TokenStream;
+import org.antlr.v4.runtime.atn.ATN;
+import org.antlr.v4.runtime.atn.ATNState;
+import org.antlr.v4.runtime.atn.DecisionState;
+import org.antlr.v4.runtime.atn.ParserATNSimulator;
 import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.misc.Nullable;
@@ -80,14 +86,14 @@ public class ParserInterpreter {
 		atnSimulator = new ParserATNSimulator<Token>(new DummyParser(g, input), g.atn);
 	}
 
-	public int predictATN(@NotNull DFA dfa, @NotNull SymbolStream<Token> input,
+	public int predictATN(@NotNull DFA dfa, @NotNull TokenStream input,
 						  @Nullable ParserRuleContext outerContext,
 						  boolean useContext)
 	{
 		return atnSimulator.predictATN(dfa, input, outerContext);
 	}
 
-	public int adaptivePredict(@NotNull SymbolStream<Token> input, int decision,
+	public int adaptivePredict(@NotNull TokenStream input, int decision,
 							   @Nullable ParserRuleContext outerContext)
 	{
 		return atnSimulator.adaptivePredict(input, decision, outerContext);

From bf9c7c3a28c04e0f8ad2e304416d828d308894ec Mon Sep 17 00:00:00 2001
From: Terence Parr <parrt@antlr.org>
Date: Mon, 2 Apr 2012 18:10:32 -0700
Subject: [PATCH 13/34]  Remove old TokenRewriteStream in favor of new
 TokenStreamRewriter that is not a stream; it feeds off of the stream to
 provide transformations of that original stream. Updated the comment at the
 top of the file. Instead of toString() we use getText(). Altered the unit
 tests to create a buffered stream and then create the rewriter attached to
 that.

---
 .../src/org/antlr/v4/runtime/TokenStream.java |   2 +
 .../antlr/v4/runtime/TokenStreamRewriter.java | 562 +++++++++++
 .../v4/runtime/UnbufferedTokenStream.java     |   5 +
 tool/test/org/antlr/v4/test/BaseTest.java     |   5 +
 .../antlr/v4/test/TestTokenRewriteStream.java | 807 ----------------
 .../v4/test/TestTokenStreamRewriter.java      | 879 ++++++++++++++++++
 6 files changed, 1453 insertions(+), 807 deletions(-)
 create mode 100644 runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
 delete mode 100644 tool/test/org/antlr/v4/test/TestTokenRewriteStream.java
 create mode 100644 tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java

diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java
index 0dc079189..5acfd96f0 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java
@@ -65,6 +65,8 @@ public interface TokenStream extends IntStream {
 	 */
 	public String getText(Interval interval);
 
+	public String getText();
+
 	public String getText(RuleContext ctx);
 
 	/** Because the user is not required to use a token with an index stored
diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
new file mode 100644
index 000000000..2fff8d18a
--- /dev/null
+++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
@@ -0,0 +1,562 @@
+package org.antlr.v4.runtime;
+
+import org.antlr.v4.runtime.misc.Interval;
+import org.antlr.v4.runtime.misc.Nullable;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Useful for rewriting out a buffered input token stream after doing some
+ *  augmentation or other manipulations on it.
+ *
+ *  You can insert stuff, replace, and delete chunks.  Note that the
+ *  operations are done lazily--only if you convert the buffer to a
+ *  String with getText(). This is very efficient because you are not moving
+ *  data around all the time.  As the buffer of tokens is converted to strings,
+ *  the getText() method(s) scan the input token stream and check
+ *  to see if there is an operation at the current index.
+ *  If so, the operation is done and then normal String
+ *  rendering continues on the buffer.  This is like having multiple Turing
+ *  machine instruction streams (programs) operating on a single input tape. :)
+ *
+ *  This rewriter makes no modifications to the token stream. It does not
+ *  ask the stream to fill itself up nor does it advance the input cursor.
+ *  The token stream index() will return the same value before and after
+ *  any getText() call.
+ *
+ *  The rewriter only works on tokens that you have in the buffer and
+ *  ignores the current input cursor. If you are buffering tokens on-demand,
+ *  calling getText() halfway through the input will only do rewrites
+ *  for those tokens in the first half of the file.
+ *
+ *  Since the operations are done lazily at getText-time, operations do not
+ *  screw up the token index values.  That is, an insert operation at token
+ *  index i does not change the index values for tokens i+1..n-1.
+ *
+ *  Because operations never actually alter the buffer, you may always get
+ *  the original token stream back without undoing anything.  Since
+ *  the instructions are queued up, you can easily simulate transactions and
+ *  roll back any changes if there is an error just by removing instructions.
+ *  For example,
+ *
+ *   CharStream input = new ANTLRFileStream("input");
+ *   TLexer lex = new TLexer(input);
+ *   CommonTokenStream tokens = new CommonTokenStream(lex);
+ *   T parser = new T(tokens);
+ *   TokenStreamRewriter rewriter = new TokenStreamRewriter(tokens);
+ *   parser.startRule();
+ *
+ * 	 Then in the rules, you can execute (assuming rewriter is visible):
+ *      Token t,u;
+ *      ...
+ *      rewriter.insertAfter(t, "text to put after t");}
+ * 		rewriter.insertAfter(u, "text after u");}
+ * 		System.out.println(tokens.toString());
+ *
+ *  You can also have multiple "instruction streams" and get multiple
+ *  rewrites from a single pass over the input.  Just name the instruction
+ *  streams and use that name again when printing the buffer.  This could be
+ *  useful for generating a C file and also its header file--all from the
+ *  same buffer:
+ *
+ *      tokens.insertAfter("pass1", t, "text to put after t");}
+ * 		tokens.insertAfter("pass2", u, "text after u");}
+ * 		System.out.println(tokens.toString("pass1"));
+ * 		System.out.println(tokens.toString("pass2"));
+ *
+ *  If you don't use named rewrite streams, a "default" stream is used as
+ *  the first example shows.
+ */
+public class TokenStreamRewriter {
+	public static final String DEFAULT_PROGRAM_NAME = "default";
+    public static final int PROGRAM_INIT_SIZE = 100;
+	public static final int MIN_TOKEN_INDEX = 0;
+
+	// Define the rewrite operation hierarchy
+
+	public class RewriteOperation {
+        /** What index into rewrites List are we? */
+        protected int instructionIndex;
+        /** Token buffer index. */
+        protected int index;
+		protected Object text;
+
+		protected RewriteOperation(int index) {
+			this.index = index;
+		}
+
+		protected RewriteOperation(int index, Object text) {
+			this.index = index;
+			this.text = text;
+		}
+		/** Execute the rewrite operation by possibly adding to the buffer.
+		 *  Return the index of the next token to operate on.
+		 */
+		public int execute(StringBuilder buf) {
+			return index;
+		}
+
+		@Override
+		public String toString() {
+			String opName = getClass().getName();
+			int $index = opName.indexOf('$');
+			opName = opName.substring($index+1, opName.length());
+			return "<"+opName+"@"+tokens.get(index)+
+				   ":\""+text+"\">";
+		}
+	}
+
+	class InsertBeforeOp extends RewriteOperation {
+		public InsertBeforeOp(int index, Object text) {
+			super(index,text);
+		}
+
+		@Override
+		public int execute(StringBuilder buf) {
+			buf.append(text);
+			if ( tokens.get(index).getType()!=Token.EOF ) {
+				buf.append(tokens.get(index).getText());
+			}
+			return index+1;
+		}
+	}
+
+	/** I'm going to try replacing range from x..y with (y-x)+1 ReplaceOp
+	 *  instructions.
+	 */
+	class ReplaceOp extends RewriteOperation {
+		protected int lastIndex;
+		public ReplaceOp(int from, int to, Object text) {
+			super(from,text);
+			lastIndex = to;
+		}
+		@Override
+		public int execute(StringBuilder buf) {
+			if ( text!=null ) {
+				buf.append(text);
+			}
+			return lastIndex+1;
+		}
+		@Override
+		public String toString() {
+			if ( text==null ) {
+				return "<DeleteOp@"+tokens.get(index)+
+					   ".."+tokens.get(lastIndex)+">";
+			}
+			return "<ReplaceOp@"+tokens.get(index)+
+				   ".."+tokens.get(lastIndex)+":\""+text+"\">";
+		}
+	}
+
+	/** Our source stream */
+	protected final BufferedTokenStream tokens;
+
+	/** You may have multiple, named streams of rewrite operations.
+	 *  I'm calling these things "programs."
+	 *  Maps String (name) -> rewrite (List)
+	 */
+	protected final Map<String, List<RewriteOperation>> programs;
+
+	/** Map String (program name) -> Integer index */
+	protected final Map<String, Integer> lastRewriteTokenIndexes;
+
+	public TokenStreamRewriter(BufferedTokenStream tokens) {
+		this.tokens = tokens;
+		programs = new HashMap<String, List<RewriteOperation>>();
+		programs.put(DEFAULT_PROGRAM_NAME,
+					 new ArrayList<RewriteOperation>(PROGRAM_INIT_SIZE));
+		lastRewriteTokenIndexes = new HashMap<String, Integer>();
+	}
+
+	public void rollback(int instructionIndex) {
+		rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
+	}
+
+	/** Rollback the instruction stream for a program so that
+	 *  the indicated instruction (via instructionIndex) is no
+	 *  longer in the stream.  UNTESTED!
+	 */
+	public void rollback(String programName, int instructionIndex) {
+		List<RewriteOperation> is = programs.get(programName);
+		if ( is!=null ) {
+			programs.put(programName, is.subList(MIN_TOKEN_INDEX,instructionIndex));
+		}
+	}
+
+	public void deleteProgram() {
+		deleteProgram(DEFAULT_PROGRAM_NAME);
+	}
+
+	/** Reset the program so that no instructions exist */
+	public void deleteProgram(String programName) {
+		rollback(programName, MIN_TOKEN_INDEX);
+	}
+
+	public void insertAfter(Token t, Object text) {
+		insertAfter(DEFAULT_PROGRAM_NAME, t, text);
+	}
+
+	public void insertAfter(int index, Object text) {
+		insertAfter(DEFAULT_PROGRAM_NAME, index, text);
+	}
+
+	public void insertAfter(String programName, Token t, Object text) {
+		insertAfter(programName,t.getTokenIndex(), text);
+	}
+
+	public void insertAfter(String programName, int index, Object text) {
+		// to insert after, just insert before next index (even if past end)
+		insertBefore(programName,index+1, text);
+	}
+
+	public void insertBefore(Token t, Object text) {
+		insertBefore(DEFAULT_PROGRAM_NAME, t, text);
+	}
+
+	public void insertBefore(int index, Object text) {
+		insertBefore(DEFAULT_PROGRAM_NAME, index, text);
+	}
+
+	public void insertBefore(String programName, Token t, Object text) {
+		insertBefore(programName, t.getTokenIndex(), text);
+	}
+
+	public void insertBefore(String programName, int index, Object text) {
+		RewriteOperation op = new InsertBeforeOp(index,text);
+		List<RewriteOperation> rewrites = getProgram(programName);
+        op.instructionIndex = rewrites.size();
+        rewrites.add(op);
+	}
+
+	public void replace(int index, Object text) {
+		replace(DEFAULT_PROGRAM_NAME, index, index, text);
+	}
+
+	public void replace(int from, int to, Object text) {
+		replace(DEFAULT_PROGRAM_NAME, from, to, text);
+	}
+
+	public void replace(Token indexT, Object text) {
+		replace(DEFAULT_PROGRAM_NAME, indexT, indexT, text);
+	}
+
+	public void replace(Token from, Token to, Object text) {
+		replace(DEFAULT_PROGRAM_NAME, from, to, text);
+	}
+
+	public void replace(String programName, int from, int to, @Nullable Object text) {
+		if ( from > to || from<0 || to<0 || to >= tokens.size() ) {
+			throw new IllegalArgumentException("replace: range invalid: "+from+".."+to+"(size="+tokens.size()+")");
+		}
+		RewriteOperation op = new ReplaceOp(from, to, text);
+		List<RewriteOperation> rewrites = getProgram(programName);
+        op.instructionIndex = rewrites.size();
+        rewrites.add(op);
+	}
+
+	public void replace(String programName, Token from, Token to, @Nullable Object text) {
+		replace(programName,
+				from.getTokenIndex(),
+				to.getTokenIndex(),
+				text);
+	}
+
+	public void delete(int index) {
+		delete(DEFAULT_PROGRAM_NAME, index, index);
+	}
+
+	public void delete(int from, int to) {
+		delete(DEFAULT_PROGRAM_NAME, from, to);
+	}
+
+	public void delete(Token indexT) {
+		delete(DEFAULT_PROGRAM_NAME, indexT, indexT);
+	}
+
+	public void delete(Token from, Token to) {
+		delete(DEFAULT_PROGRAM_NAME, from, to);
+	}
+
+	public void delete(String programName, int from, int to) {
+		replace(programName,from,to,null);
+	}
+
+	public void delete(String programName, Token from, Token to) {
+		replace(programName,from,to,null);
+	}
+
+	public int getLastRewriteTokenIndex() {
+		return getLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
+	}
+
+	protected int getLastRewriteTokenIndex(String programName) {
+		Integer I = lastRewriteTokenIndexes.get(programName);
+		if ( I==null ) {
+			return -1;
+		}
+		return I;
+	}
+
+	protected void setLastRewriteTokenIndex(String programName, int i) {
+		lastRewriteTokenIndexes.put(programName, i);
+	}
+
+	protected List<RewriteOperation> getProgram(String name) {
+		List<RewriteOperation> is = programs.get(name);
+		if ( is==null ) {
+			is = initializeProgram(name);
+		}
+		return is;
+	}
+
+	private List<RewriteOperation> initializeProgram(String name) {
+		List<RewriteOperation> is = new ArrayList<RewriteOperation>(PROGRAM_INIT_SIZE);
+		programs.put(name, is);
+		return is;
+	}
+
+	public String getOriginalText() {
+		return tokens.getText();
+	}
+
+	/** The text collected from all tokens in the input token stream */
+	public String getOriginalText(Interval interval) {
+		return tokens.getText(interval);
+	}
+
+	/** Return the text from the original tokens altered per the
+	 *  instructions given to this rewriter.
+ 	 */
+	public String getText() {
+		return getText(DEFAULT_PROGRAM_NAME, Interval.of(0,tokens.size()-1));
+	}
+
+	/** Return the text associated with the tokens in the interval from the
+	 *  original token stream but with the alterations given to this rewriter.
+	 *  The interval refers to the indexes in the original token stream.
+	 *  We do not alter the token stream in any way, so the indexes
+	 *  and intervals are still consistent. Includes any operations done
+	 *  to the first and last token in the interval. So, if you did an
+	 *  insertBefore on the first token, you would get that insertion.
+	 *  The same is true if you do an insertAfter the stop token.
+ 	 */
+	public String getText(Interval interval) {
+		return getText(DEFAULT_PROGRAM_NAME, interval);
+	}
+
+	public String getText(String programName, Interval interval) {
+		List<RewriteOperation> rewrites = programs.get(programName);
+		int start = interval.a;
+		int stop = interval.b;
+
+        // ensure start/end are in range
+        if ( stop>tokens.size()-1 ) stop = tokens.size()-1;
+        if ( start<0 ) start = 0;
+
+        if ( rewrites==null || rewrites.isEmpty() ) {
+			return tokens.getText(interval); // no instructions to execute
+		}
+		StringBuilder buf = new StringBuilder();
+
+		// First, optimize instruction stream
+		Map<Integer, RewriteOperation> indexToOp = reduceToSingleOperationPerIndex(rewrites);
+
+        // Walk buffer, executing instructions and emitting tokens
+        int i = start;
+        while ( i <= stop && i < tokens.size() ) {
+			RewriteOperation op = indexToOp.get(i);
+			indexToOp.remove(i); // remove so any left have index size-1
+			Token t = tokens.get(i);
+			if ( op==null ) {
+				// no operation at that index, just dump token
+				if ( t.getType()!=Token.EOF ) buf.append(t.getText());
+				i++; // move to next token
+			}
+			else {
+				i = op.execute(buf); // execute operation and skip
+			}
+		}
+
+        // include stuff after end if it's last index in buffer
+        // So, if they did an insertAfter(lastValidIndex, "foo"), include
+        // foo if end==lastValidIndex.
+        if ( stop==tokens.size()-1 ) {
+            // Scan any remaining operations after last token
+            // should be included (they will be inserts).
+			for (RewriteOperation op : indexToOp.values()) {
+                if ( op.index >= tokens.size()-1 ) buf.append(op.text);
+            }
+        }
+        return buf.toString();
+	}
+
+	/** We need to combine operations and report invalid operations (like
+	 *  overlapping replaces that are not completed nested).  Inserts to
+	 *  same index need to be combined etc...   Here are the cases:
+	 *
+	 *  I.i.u I.j.v								leave alone, nonoverlapping
+	 *  I.i.u I.i.v								combine: Iivu
+	 *
+	 *  R.i-j.u R.x-y.v	| i-j in x-y			delete first R
+	 *  R.i-j.u R.i-j.v							delete first R
+	 *  R.i-j.u R.x-y.v	| x-y in i-j			ERROR
+	 *  R.i-j.u R.x-y.v	| boundaries overlap	ERROR
+	 *
+	 *  Delete special case of replace (text==null):
+	 *  D.i-j.u D.x-y.v	| boundaries overlap	combine to max(min)..max(right)
+	 *
+	 *  I.i.u R.x-y.v | i in (x+1)-y			delete I (since insert before
+	 *											we're not deleting i)
+	 *  I.i.u R.x-y.v | i not in (x+1)-y		leave alone, nonoverlapping
+	 *  R.x-y.v I.i.u | i in x-y				ERROR
+	 *  R.x-y.v I.x.u 							R.x-y.uv (combine, delete I)
+	 *  R.x-y.v I.i.u | i not in x-y			leave alone, nonoverlapping
+	 *
+	 *  I.i.u = insert u before op @ index i
+	 *  R.x-y.u = replace x-y indexed tokens with u
+	 *
+	 *  First we need to examine replaces.  For any replace op:
+	 *
+	 * 		1. wipe out any insertions before op within that range.
+	 *		2. Drop any replace op before that is contained completely within
+	 *         that range.
+	 *		3. Throw exception upon boundary overlap with any previous replace.
+	 *
+	 *  Then we can deal with inserts:
+	 *
+	 * 		1. for any inserts to same index, combine even if not adjacent.
+	 * 		2. for any prior replace with same left boundary, combine this
+	 *         insert with replace and delete this replace.
+	 * 		3. throw exception if index in same range as previous replace
+	 *
+	 *  Don't actually delete; make op null in list. Easier to walk list.
+	 *  Later we can throw as we add to index -> op map.
+	 *
+	 *  Note that I.2 R.2-2 will wipe out I.2 even though, technically, the
+	 *  inserted stuff would be before the replace range.  But, if you
+	 *  add tokens in front of a method body '{' and then delete the method
+	 *  body, I think the stuff before the '{' you added should disappear too.
+	 *
+	 *  Return a map from token index to operation.
+	 */
+	protected Map<Integer, RewriteOperation> reduceToSingleOperationPerIndex(List<RewriteOperation> rewrites) {
+//		System.out.println("rewrites="+rewrites);
+
+		// WALK REPLACES
+		for (int i = 0; i < rewrites.size(); i++) {
+			RewriteOperation op = rewrites.get(i);
+			if ( op==null ) continue;
+			if ( !(op instanceof ReplaceOp) ) continue;
+			ReplaceOp rop = (ReplaceOp)rewrites.get(i);
+			// Wipe prior inserts within range
+			List<? extends InsertBeforeOp> inserts = getKindOfOps(rewrites, InsertBeforeOp.class, i);
+			for (InsertBeforeOp iop : inserts) {
+				if ( iop.index == rop.index ) {
+					// E.g., insert before 2, delete 2..2; update replace
+					// text to include insert before, kill insert
+					rewrites.set(iop.instructionIndex, null);
+					rop.text = iop.text.toString() + (rop.text!=null?rop.text.toString():"");
+				}
+				else if ( iop.index > rop.index && iop.index <= rop.lastIndex ) {
+                    // delete insert as it's a no-op.
+                    rewrites.set(iop.instructionIndex, null);
+				}
+			}
+			// Drop any prior replaces contained within
+			List<? extends ReplaceOp> prevReplaces = getKindOfOps(rewrites, ReplaceOp.class, i);
+			for (ReplaceOp prevRop : prevReplaces) {
+				if ( prevRop.index>=rop.index && prevRop.lastIndex <= rop.lastIndex ) {
+                    // delete replace as it's a no-op.
+                    rewrites.set(prevRop.instructionIndex, null);
+					continue;
+				}
+				// throw exception unless disjoint or identical
+				boolean disjoint =
+					prevRop.lastIndex<rop.index || prevRop.index > rop.lastIndex;
+				boolean same =
+					prevRop.index==rop.index && prevRop.lastIndex==rop.lastIndex;
+				// Delete special case of replace (text==null):
+				// D.i-j.u D.x-y.v	| boundaries overlap	combine to max(min)..max(right)
+				if ( prevRop.text==null && rop.text==null && !disjoint ) {
+					//System.out.println("overlapping deletes: "+prevRop+", "+rop);
+					rewrites.set(prevRop.instructionIndex, null); // kill first delete
+					rop.index = Math.min(prevRop.index, rop.index);
+					rop.lastIndex = Math.max(prevRop.lastIndex, rop.lastIndex);
+					System.out.println("new rop "+rop);
+				}
+				else if ( !disjoint && !same ) {
+					throw new IllegalArgumentException("replace op boundaries of "+rop+
+													   " overlap with previous "+prevRop);
+				}
+			}
+		}
+
+		// WALK INSERTS
+		for (int i = 0; i < rewrites.size(); i++) {
+			RewriteOperation op = rewrites.get(i);
+			if ( op==null ) continue;
+			if ( !(op instanceof InsertBeforeOp) ) continue;
+			InsertBeforeOp iop = (InsertBeforeOp)rewrites.get(i);
+			// combine current insert with prior if any at same index
+			List<? extends InsertBeforeOp> prevInserts = getKindOfOps(rewrites, InsertBeforeOp.class, i);
+			for (InsertBeforeOp prevIop : prevInserts) {
+				if ( prevIop.index == iop.index ) { // combine objects
+					// convert to strings...we're in process of toString'ing
+					// whole token buffer so no lazy eval issue with any templates
+					iop.text = catOpText(iop.text,prevIop.text);
+                    // delete redundant prior insert
+                    rewrites.set(prevIop.instructionIndex, null);
+				}
+			}
+			// look for replaces where iop.index is in range; error
+			List<? extends ReplaceOp> prevReplaces = getKindOfOps(rewrites, ReplaceOp.class, i);
+			for (ReplaceOp rop : prevReplaces) {
+				if ( iop.index == rop.index ) {
+					rop.text = catOpText(iop.text,rop.text);
+					rewrites.set(i, null);  // delete current insert
+					continue;
+				}
+				if ( iop.index >= rop.index && iop.index <= rop.lastIndex ) {
+					throw new IllegalArgumentException("insert op "+iop+
+													   " within boundaries of previous "+rop);
+				}
+			}
+		}
+		// System.out.println("rewrites after="+rewrites);
+		Map<Integer, RewriteOperation> m = new HashMap<Integer, RewriteOperation>();
+		for (int i = 0; i < rewrites.size(); i++) {
+			RewriteOperation op = rewrites.get(i);
+			if ( op==null ) continue; // ignore deleted ops
+			if ( m.get(op.index)!=null ) {
+				throw new Error("should only be one op per index");
+			}
+			m.put(op.index, op);
+		}
+		//System.out.println("index to op: "+m);
+		return m;
+	}
+
+	protected String catOpText(Object a, Object b) {
+		String x = "";
+		String y = "";
+		if ( a!=null ) x = a.toString();
+		if ( b!=null ) y = b.toString();
+		return x+y;
+	}
+
+    /** Get all operations before an index of a particular kind */
+    protected <T extends RewriteOperation> List<? extends T> getKindOfOps(List<? extends RewriteOperation> rewrites, Class<T> kind, int before) {
+		List<T> ops = new ArrayList<T>();
+		for (int i=0; i<before && i<rewrites.size(); i++) {
+			RewriteOperation op = rewrites.get(i);
+			if ( op==null ) continue; // ignore deleted
+			if ( kind.isInstance(op) ) {
+				ops.add((T)op);
+			}
+		}
+		return ops;
+	}
+
+}
diff --git a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
index 17a81abd9..4dda8d151 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java
@@ -98,6 +98,11 @@ public class UnbufferedTokenStream<T extends Token>
 		return buf.toString();
 	}
 
+	@Override
+	public String getText() {
+		return getText(Interval.of(0,index()));
+	}
+
 	@Override
 	public String getText(RuleContext ctx) {
 		return getText(ctx.getSourceInterval());
diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java
index dd845f858..a26d63303 100644
--- a/tool/test/org/antlr/v4/test/BaseTest.java
+++ b/tool/test/org/antlr/v4/test/BaseTest.java
@@ -1097,6 +1097,11 @@ public abstract class BaseTest {
 			return null;
 		}
 
+		@Override
+		public String getText() {
+			throw new UnsupportedOperationException("can't give strings");
+		}
+
 		@Override
 		public String getText(Interval interval) {
 			throw new UnsupportedOperationException("can't give strings");
diff --git a/tool/test/org/antlr/v4/test/TestTokenRewriteStream.java b/tool/test/org/antlr/v4/test/TestTokenRewriteStream.java
deleted file mode 100644
index ef785bc72..000000000
--- a/tool/test/org/antlr/v4/test/TestTokenRewriteStream.java
+++ /dev/null
@@ -1,807 +0,0 @@
-/*
- [The "BSD license"]
- Copyright (c) 2011 Terence Parr
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
-
- 1. Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
- 3. The name of the author may not be used to endorse or promote products
-    derived from this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-package org.antlr.v4.test;
-
-import org.antlr.v4.runtime.TokenRewriteStream;
-import org.antlr.v4.runtime.misc.Interval;
-import org.antlr.v4.tool.LexerGrammar;
-import org.antlr.v4.tool.interp.LexerInterpreter;
-import org.junit.Test;
-
-public class TestTokenRewriteStream extends BaseTest {
-
-    /** Public default constructor used by TestRig */
-    public TestTokenRewriteStream() {
-    }
-
-	@Test public void testInsertBeforeIndex0() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		LexerInterpreter lexInterp = new LexerInterpreter(g, "abc");
-		TokenRewriteStream tokens = new TokenRewriteStream(lexInterp);
-		tokens.insertBefore(0, "0");
-		String result = tokens.toString();
-		String expecting = "0abc";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testInsertAfterLastIndex() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.insertAfter(2, "x");
-		String result = tokens.toString();
-		String expecting = "abcx";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void test2InsertBeforeAfterMiddleIndex() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.insertBefore(1, "x");
-		tokens.insertAfter(1, "x");
-		String result = tokens.toString();
-		String expecting = "axbxc";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testReplaceIndex0() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(0, "x");
-		String result = tokens.toString();
-		String expecting = "xbc";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testReplaceLastIndex() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(2, "x");
-		String result = tokens.toString();
-		String expecting = "abx";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testReplaceMiddleIndex() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(1, "x");
-		String result = tokens.toString();
-		String expecting = "axc";
-		assertEquals(expecting, result);
-	}
-
-    @Test public void testToStringStartStop() throws Exception {
-        LexerGrammar g = new LexerGrammar(
-            "lexer grammar t;\n"+
-            "ID : 'a'..'z'+;\n" +
-            "INT : '0'..'9'+;\n" +
-            "SEMI : ';';\n" +
-            "MUL : '*';\n" +
-            "ASSIGN : '=';\n" +
-            "WS : ' '+;\n");
-        // Tokens: 0123456789
-        // Input:  x = 3 * 0;
-        String input = "x = 3 * 0;";
-        LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-        TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-        tokens.fill();
-        tokens.replace(4, 8, "0"); // replace 3 * 0 with 0
-
-        String result = tokens.toOriginalString();
-        String expecting = "x = 3 * 0;";
-        assertEquals(expecting, result);
-
-        result = tokens.toString();
-        expecting = "x = 0;";
-        assertEquals(expecting, result);
-
-        result = tokens.getText(Interval.of(0, 9));
-        expecting = "x = 0;";
-        assertEquals(expecting, result);
-
-        result = tokens.getText(Interval.of(4, 8));
-        expecting = "0";
-        assertEquals(expecting, result);
-    }
-
-    @Test public void testToStringStartStop2() throws Exception {
-        LexerGrammar g = new LexerGrammar(
-            "lexer grammar t;\n"+
-            "ID : 'a'..'z'+;\n" +
-            "INT : '0'..'9'+;\n" +
-            "SEMI : ';';\n" +
-            "ASSIGN : '=';\n" +
-            "PLUS : '+';\n" +
-            "MULT : '*';\n" +
-            "WS : ' '+;\n");
-        // Tokens: 012345678901234567
-        // Input:  x = 3 * 0 + 2 * 0;
-        String input = "x = 3 * 0 + 2 * 0;";
-        LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-        TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-        tokens.fill();
-
-        String result = tokens.toOriginalString();
-        String expecting = "x = 3 * 0 + 2 * 0;";
-        assertEquals(expecting, result);
-
-        tokens.replace(4, 8, "0"); // replace 3 * 0 with 0
-        result = tokens.toString();
-        expecting = "x = 0 + 2 * 0;";
-        assertEquals(expecting, result);
-
-        result = tokens.getText(Interval.of(0, 17));
-        expecting = "x = 0 + 2 * 0;";
-        assertEquals(expecting, result);
-
-        result = tokens.getText(Interval.of(4, 8));
-        expecting = "0";
-        assertEquals(expecting, result);
-
-        result = tokens.getText(Interval.of(0, 8));
-        expecting = "x = 0";
-        assertEquals(expecting, result);
-
-        result = tokens.getText(Interval.of(12, 16));
-        expecting = "2 * 0";
-        assertEquals(expecting, result);
-
-        tokens.insertAfter(17, "// comment");
-        result = tokens.getText(Interval.of(12, 18));
-        expecting = "2 * 0;// comment";
-        assertEquals(expecting, result);
-
-        result = tokens.getText(Interval.of(0, 8)); // try again after insert at end
-        expecting = "x = 0";
-        assertEquals(expecting, result);
-    }
-
-
-    @Test public void test2ReplaceMiddleIndex() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(1, "x");
-		tokens.replace(1, "y");
-		String result = tokens.toString();
-		String expecting = "ayc";
-		assertEquals(expecting, result);
-	}
-
-    @Test public void test2ReplaceMiddleIndex1InsertBefore() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-        tokens.insertBefore(0, "_");
-        tokens.replace(1, "x");
-		tokens.replace(1, "y");
-		String result = tokens.toString();
-		String expecting = "_ayc";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testReplaceThenDeleteMiddleIndex() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(1, "x");
-		tokens.delete(1);
-		String result = tokens.toString();
-		String expecting = "ac";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testInsertInPriorReplace() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(0, 2, "x");
-		tokens.insertBefore(1, "0");
-		Exception exc = null;
-		try {
-			tokens.toString();
-		}
-		catch (IllegalArgumentException iae) {
-			exc = iae;
-		}
-		String expecting = "insert op <InsertBeforeOp@[@1,1:1='b',<4>,1:1]:\"0\"> within boundaries of previous <ReplaceOp@[@0,0:0='a',<3>,1:0]..[@2,2:2='c',<5>,1:2]:\"x\">";
-		assertNotNull(exc);
-		assertEquals(expecting, exc.getMessage());
-	}
-
-	@Test public void testInsertThenReplaceSameIndex() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.insertBefore(0, "0");
-		tokens.replace(0, "x"); // supercedes insert at 0
-		String result = tokens.toString();
-		String expecting = "0xbc";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void test2InsertMiddleIndex() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.insertBefore(1, "x");
-		tokens.insertBefore(1, "y");
-		String result = tokens.toString();
-		String expecting = "ayxbc";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void test2InsertThenReplaceIndex0() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.insertBefore(0, "x");
-		tokens.insertBefore(0, "y");
-		tokens.replace(0, "z");
-		String result = tokens.toString();
-		String expecting = "yxzbc";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testReplaceThenInsertBeforeLastIndex() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(2, "x");
-		tokens.insertBefore(2, "y");
-		String result = tokens.toString();
-		String expecting = "abyx";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testInsertThenReplaceLastIndex() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.insertBefore(2, "y");
-		tokens.replace(2, "x");
-		String result = tokens.toString();
-		String expecting = "abyx";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testReplaceThenInsertAfterLastIndex() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(2, "x");
-		tokens.insertAfter(2, "y");
-		String result = tokens.toString();
-		String expecting = "abxy";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testReplaceRangeThenInsertAtLeftEdge() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcccba";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(2, 4, "x");
-		tokens.insertBefore(2, "y");
-		String result = tokens.toString();
-		String expecting = "abyxba";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testReplaceRangeThenInsertAtRightEdge() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcccba";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(2, 4, "x");
-		tokens.insertBefore(4, "y"); // no effect; within range of a replace
-		Exception exc = null;
-		try {
-			tokens.toString();
-		}
-		catch (IllegalArgumentException iae) {
-			exc = iae;
-		}
-		String expecting = "insert op <InsertBeforeOp@[@4,4:4='c',<5>,1:4]:\"y\"> within boundaries of previous <ReplaceOp@[@2,2:2='c',<5>,1:2]..[@4,4:4='c',<5>,1:4]:\"x\">";
-		assertNotNull(exc);
-		assertEquals(expecting, exc.getMessage());
-	}
-
-	@Test public void testReplaceRangeThenInsertAfterRightEdge() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcccba";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(2, 4, "x");
-		tokens.insertAfter(4, "y");
-		String result = tokens.toString();
-		String expecting = "abxyba";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testReplaceAll() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcccba";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(0, 6, "x");
-		String result = tokens.toString();
-		String expecting = "x";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testReplaceSubsetThenFetch() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcccba";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(2, 4, "xyz");
-		String result = tokens.getText(Interval.of(0, 6));
-		String expecting = "abxyzba";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testReplaceThenReplaceSuperset() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcccba";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(2, 4, "xyz");
-		tokens.replace(3, 5, "foo"); // overlaps, error
-		Exception exc = null;
-		try {
-			tokens.toString();
-		}
-		catch (IllegalArgumentException iae) {
-			exc = iae;
-		}
-		String expecting = "replace op boundaries of <ReplaceOp@[@3,3:3='c',<5>,1:3]..[@5,5:5='b',<4>,1:5]:\"foo\"> overlap with previous <ReplaceOp@[@2,2:2='c',<5>,1:2]..[@4,4:4='c',<5>,1:4]:\"xyz\">";
-		assertNotNull(exc);
-		assertEquals(expecting, exc.getMessage());
-	}
-
-	@Test public void testReplaceThenReplaceLowerIndexedSuperset() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcccba";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(2, 4, "xyz");
-		tokens.replace(1, 3, "foo"); // overlap, error
-		Exception exc = null;
-		try {
-			tokens.toString();
-		}
-		catch (IllegalArgumentException iae) {
-			exc = iae;
-		}
-		String expecting = "replace op boundaries of <ReplaceOp@[@1,1:1='b',<4>,1:1]..[@3,3:3='c',<5>,1:3]:\"foo\"> overlap with previous <ReplaceOp@[@2,2:2='c',<5>,1:2]..[@4,4:4='c',<5>,1:4]:\"xyz\">";
-		assertNotNull(exc);
-		assertEquals(expecting, exc.getMessage());
-	}
-
-	@Test public void testReplaceSingleMiddleThenOverlappingSuperset() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcba";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(2, 2, "xyz");
-		tokens.replace(0, 3, "foo");
-		String result = tokens.toString();
-		String expecting = "fooa";
-		assertEquals(expecting, result);
-	}
-
-	// June 2, 2008 I rewrote core of rewrite engine; just adding lots more tests here
-
-	@Test public void testCombineInserts() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.insertBefore(0, "x");
-		tokens.insertBefore(0, "y");
-		String result = tokens.toString();
-		String expecting = "yxabc";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testCombine3Inserts() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.insertBefore(1, "x");
-		tokens.insertBefore(0, "y");
-		tokens.insertBefore(1, "z");
-		String result = tokens.toString();
-		String expecting = "yazxbc";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testCombineInsertOnLeftWithReplace() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(0, 2, "foo");
-		tokens.insertBefore(0, "z"); // combine with left edge of rewrite
-		String result = tokens.toString();
-		String expecting = "zfoo";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testCombineInsertOnLeftWithDelete() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.delete(0, 2);
-		tokens.insertBefore(0, "z"); // combine with left edge of rewrite
-		String result = tokens.toString();
-		String expecting = "z"; // make sure combo is not znull
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testDisjointInserts() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.insertBefore(1, "x");
-		tokens.insertBefore(2, "y");
-		tokens.insertBefore(0, "z");
-		String result = tokens.toString();
-		String expecting = "zaxbyc";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testOverlappingReplace() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(1, 2, "foo");
-		tokens.replace(0, 3, "bar"); // wipes prior nested replace
-		String result = tokens.toString();
-		String expecting = "bar";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testOverlappingReplace2() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(0, 3, "bar");
-		tokens.replace(1, 2, "foo"); // cannot split earlier replace
-		Exception exc = null;
-		try {
-			tokens.toString();
-		}
-		catch (IllegalArgumentException iae) {
-			exc = iae;
-		}
-		String expecting = "replace op boundaries of <ReplaceOp@[@1,1:1='b',<4>,1:1]..[@2,2:2='c',<5>,1:2]:\"foo\"> overlap with previous <ReplaceOp@[@0,0:0='a',<3>,1:0]..[@3,3:3='c',<5>,1:3]:\"bar\">";
-		assertNotNull(exc);
-		assertEquals(expecting, exc.getMessage());
-	}
-
-	@Test public void testOverlappingReplace3() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(1, 2, "foo");
-		tokens.replace(0, 2, "bar"); // wipes prior nested replace
-		String result = tokens.toString();
-		String expecting = "barc";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testOverlappingReplace4() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(1, 2, "foo");
-		tokens.replace(1, 3, "bar"); // wipes prior nested replace
-		String result = tokens.toString();
-		String expecting = "abar";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testDropIdenticalReplace() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(1, 2, "foo");
-		tokens.replace(1, 2, "foo"); // drop previous, identical
-		String result = tokens.toString();
-		String expecting = "afooc";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testDropPrevCoveredInsert() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.insertBefore(1, "foo");
-		tokens.replace(1, 2, "foo"); // kill prev insert
-		String result = tokens.toString();
-		String expecting = "afoofoo";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testLeaveAloneDisjointInsert() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.insertBefore(1, "x");
-		tokens.replace(2, 3, "foo");
-		String result = tokens.toString();
-		String expecting = "axbfoo";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testLeaveAloneDisjointInsert2() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abcc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.replace(2, 3, "foo");
-		tokens.insertBefore(1, "x");
-		String result = tokens.toString();
-		String expecting = "axbfoo";
-		assertEquals(expecting, result);
-	}
-
-	@Test public void testInsertBeforeTokenThenDeleteThatToken() throws Exception {
-		LexerGrammar g = new LexerGrammar(
-			"lexer grammar t;\n"+
-			"A : 'a';\n" +
-			"B : 'b';\n" +
-			"C : 'c';\n");
-		String input = "abc";
-		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
-		TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
-		tokens.fill();
-		tokens.insertBefore(2, "y");
-		tokens.delete(2);
-		String result = tokens.toString();
-		String expecting = "aby";
-		assertEquals(expecting, result);
-	}
-
-}
diff --git a/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java b/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java
new file mode 100644
index 000000000..977c1b76d
--- /dev/null
+++ b/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java
@@ -0,0 +1,879 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2011 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+    derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.v4.test;
+
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.TokenStreamRewriter;
+import org.antlr.v4.runtime.misc.Interval;
+import org.antlr.v4.tool.LexerGrammar;
+import org.antlr.v4.tool.interp.LexerInterpreter;
+import org.junit.Test;
+
+public class TestTokenStreamRewriter extends BaseTest {
+
+	/** Public default constructor used by TestRig */
+	public TestTokenStreamRewriter() {
+	}
+
+	@Test public void testInsertBeforeIndex0() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		LexerInterpreter lexEngine = new LexerInterpreter(g, "abc");
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(0, "0");
+		String result = tokens.getText();
+		String expecting = "0abc";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testInsertAfterLastIndex() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertAfter(2, "x");
+		String result = tokens.getText();
+		String expecting = "abcx";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void test2InsertBeforeAfterMiddleIndex() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(1, "x");
+		tokens.insertAfter(1, "x");
+		String result = tokens.getText();
+		String expecting = "axbxc";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testReplaceIndex0() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(0, "x");
+		String result = tokens.getText();
+		String expecting = "xbc";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testReplaceLastIndex() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(2, "x");
+		String result = tokens.getText();
+		String expecting = "abx";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testReplaceMiddleIndex() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(1, "x");
+		String result = tokens.getText();
+		String expecting = "axc";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testToStringStartStop() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "ID : 'a'..'z'+;\n" +
+											 "INT : '0'..'9'+;\n" +
+											 "SEMI : ';';\n" +
+											 "MUL : '*';\n" +
+											 "ASSIGN : '=';\n" +
+											 "WS : ' '+;\n");
+		// Tokens: 0123456789
+		// Input:  x = 3 * 0;
+		String input = "x = 3 * 0;";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(4, 8, "0");
+		stream.fill();
+// replace 3 * 0 with 0
+
+		String result = tokens.getOriginalText();
+		String expecting = "x = 3 * 0;";
+		assertEquals(expecting, result);
+
+		result = tokens.getText();
+		expecting = "x = 0;";
+		assertEquals(expecting, result);
+
+		result = tokens.getText(Interval.of(0, 9));
+		expecting = "x = 0;";
+		assertEquals(expecting, result);
+
+		result = tokens.getText(Interval.of(4, 8));
+		expecting = "0";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testToStringStartStop2() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "ID : 'a'..'z'+;\n" +
+											 "INT : '0'..'9'+;\n" +
+											 "SEMI : ';';\n" +
+											 "ASSIGN : '=';\n" +
+											 "PLUS : '+';\n" +
+											 "MULT : '*';\n" +
+											 "WS : ' '+;\n");
+		// Tokens: 012345678901234567
+		// Input:  x = 3 * 0 + 2 * 0;
+		String input = "x = 3 * 0 + 2 * 0;";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+
+		String result = tokens.getOriginalText();
+		String expecting = "x = 3 * 0 + 2 * 0;";
+		assertEquals(expecting, result);
+
+		tokens.replace(4, 8, "0");
+		stream.fill();
+// replace 3 * 0 with 0
+		result = tokens.getText();
+		expecting = "x = 0 + 2 * 0;";
+		assertEquals(expecting, result);
+
+		result = tokens.getText(Interval.of(0, 17));
+		expecting = "x = 0 + 2 * 0;";
+		assertEquals(expecting, result);
+
+		result = tokens.getText(Interval.of(4, 8));
+		expecting = "0";
+		assertEquals(expecting, result);
+
+		result = tokens.getText(Interval.of(0, 8));
+		expecting = "x = 0";
+		assertEquals(expecting, result);
+
+		result = tokens.getText(Interval.of(12, 16));
+		expecting = "2 * 0";
+		assertEquals(expecting, result);
+
+		tokens.insertAfter(17, "// comment");
+		result = tokens.getText(Interval.of(12, 18));
+		expecting = "2 * 0;// comment";
+		assertEquals(expecting, result);
+
+		result = tokens.getText(Interval.of(0, 8));
+		stream.fill();
+// try again after insert at end
+		expecting = "x = 0";
+		assertEquals(expecting, result);
+	}
+
+
+	@Test public void test2ReplaceMiddleIndex() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(1, "x");
+		tokens.replace(1, "y");
+		String result = tokens.getText();
+		String expecting = "ayc";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void test2ReplaceMiddleIndex1InsertBefore() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(0, "_");
+		tokens.replace(1, "x");
+		tokens.replace(1, "y");
+		String result = tokens.getText();
+		String expecting = "_ayc";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testReplaceThenDeleteMiddleIndex() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(1, "x");
+		tokens.delete(1);
+		String result = tokens.getText();
+		String expecting = "ac";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testInsertInPriorReplace() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(0, 2, "x");
+		tokens.insertBefore(1, "0");
+		Exception exc = null;
+		try {
+			tokens.getText();
+		}
+		catch (IllegalArgumentException iae) {
+			exc = iae;
+		}
+		String expecting = "insert op <InsertBeforeOp@[@1,1:1='b',<4>,1:1]:\"0\"> within boundaries of previous <ReplaceOp@[@0,0:0='a',<3>,1:0]..[@2,2:2='c',<5>,1:2]:\"x\">";
+		assertNotNull(exc);
+		assertEquals(expecting, exc.getMessage());
+	}
+
+	@Test public void testInsertThenReplaceSameIndex() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(0, "0");
+		tokens.replace(0, "x");
+		stream.fill();
+// supercedes insert at 0
+		String result = tokens.getText();
+		String expecting = "0xbc";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void test2InsertMiddleIndex() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(1, "x");
+		tokens.insertBefore(1, "y");
+		String result = tokens.getText();
+		String expecting = "ayxbc";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void test2InsertThenReplaceIndex0() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(0, "x");
+		tokens.insertBefore(0, "y");
+		tokens.replace(0, "z");
+		String result = tokens.getText();
+		String expecting = "yxzbc";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testReplaceThenInsertBeforeLastIndex() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(2, "x");
+		tokens.insertBefore(2, "y");
+		String result = tokens.getText();
+		String expecting = "abyx";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testInsertThenReplaceLastIndex() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(2, "y");
+		tokens.replace(2, "x");
+		String result = tokens.getText();
+		String expecting = "abyx";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testReplaceThenInsertAfterLastIndex() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(2, "x");
+		tokens.insertAfter(2, "y");
+		String result = tokens.getText();
+		String expecting = "abxy";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testReplaceRangeThenInsertAtLeftEdge() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcccba";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(2, 4, "x");
+		tokens.insertBefore(2, "y");
+		String result = tokens.getText();
+		String expecting = "abyxba";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testReplaceRangeThenInsertAtRightEdge() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcccba";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(2, 4, "x");
+		tokens.insertBefore(4, "y");
+		stream.fill(); // no effect; within range of a replace
+		Exception exc = null;
+		try {
+			tokens.getText();
+		}
+		catch (IllegalArgumentException iae) {
+			exc = iae;
+		}
+		String expecting = "insert op <InsertBeforeOp@[@4,4:4='c',<5>,1:4]:\"y\"> within boundaries of previous <ReplaceOp@[@2,2:2='c',<5>,1:2]..[@4,4:4='c',<5>,1:4]:\"x\">";
+		assertNotNull(exc);
+		assertEquals(expecting, exc.getMessage());
+	}
+
+	@Test public void testReplaceRangeThenInsertAfterRightEdge() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcccba";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(2, 4, "x");
+		tokens.insertAfter(4, "y");
+		String result = tokens.getText();
+		String expecting = "abxyba";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testReplaceAll() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcccba";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(0, 6, "x");
+		String result = tokens.getText();
+		String expecting = "x";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testReplaceSubsetThenFetch() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcccba";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(2, 4, "xyz");
+		String result = tokens.getText(Interval.of(0, 6));
+		String expecting = "abxyzba";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testReplaceThenReplaceSuperset() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcccba";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(2, 4, "xyz");
+		tokens.replace(3, 5, "foo");
+		stream.fill();
+// overlaps, error
+		Exception exc = null;
+		try {
+			tokens.getText();
+		}
+		catch (IllegalArgumentException iae) {
+			exc = iae;
+		}
+		String expecting = "replace op boundaries of <ReplaceOp@[@3,3:3='c',<5>,1:3]..[@5,5:5='b',<4>,1:5]:\"foo\"> overlap with previous <ReplaceOp@[@2,2:2='c',<5>,1:2]..[@4,4:4='c',<5>,1:4]:\"xyz\">";
+		assertNotNull(exc);
+		assertEquals(expecting, exc.getMessage());
+	}
+
+	@Test public void testReplaceThenReplaceLowerIndexedSuperset() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcccba";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(2, 4, "xyz");
+		tokens.replace(1, 3, "foo");
+		stream.fill();
+// overlap, error
+		Exception exc = null;
+		try {
+			tokens.getText();
+		}
+		catch (IllegalArgumentException iae) {
+			exc = iae;
+		}
+		String expecting = "replace op boundaries of <ReplaceOp@[@1,1:1='b',<4>,1:1]..[@3,3:3='c',<5>,1:3]:\"foo\"> overlap with previous <ReplaceOp@[@2,2:2='c',<5>,1:2]..[@4,4:4='c',<5>,1:4]:\"xyz\">";
+		assertNotNull(exc);
+		assertEquals(expecting, exc.getMessage());
+	}
+
+	@Test public void testReplaceSingleMiddleThenOverlappingSuperset() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcba";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(2, 2, "xyz");
+		tokens.replace(0, 3, "foo");
+		String result = tokens.getText();
+		String expecting = "fooa";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testCombineInserts() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(0, "x");
+		tokens.insertBefore(0, "y");
+		String result = tokens.getText();
+		String expecting = "yxabc";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testCombine3Inserts() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(1, "x");
+		tokens.insertBefore(0, "y");
+		tokens.insertBefore(1, "z");
+		String result = tokens.getText();
+		String expecting = "yazxbc";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testCombineInsertOnLeftWithReplace() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(0, 2, "foo");
+		tokens.insertBefore(0, "z");
+		stream.fill();
+// combine with left edge of rewrite
+		String result = tokens.getText();
+		String expecting = "zfoo";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testCombineInsertOnLeftWithDelete() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.delete(0, 2);
+		tokens.insertBefore(0, "z");
+		stream.fill();
+// combine with left edge of rewrite
+		String result = tokens.getText();
+		String expecting = "z";
+		stream.fill();
+// make sure combo is not znull
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testDisjointInserts() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(1, "x");
+		tokens.insertBefore(2, "y");
+		tokens.insertBefore(0, "z");
+		String result = tokens.getText();
+		String expecting = "zaxbyc";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testOverlappingReplace() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(1, 2, "foo");
+		tokens.replace(0, 3, "bar");
+		stream.fill();
+// wipes prior nested replace
+		String result = tokens.getText();
+		String expecting = "bar";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testOverlappingReplace2() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(0, 3, "bar");
+		tokens.replace(1, 2, "foo");
+		stream.fill();
+// cannot split earlier replace
+		Exception exc = null;
+		try {
+			tokens.getText();
+		}
+		catch (IllegalArgumentException iae) {
+			exc = iae;
+		}
+		String expecting = "replace op boundaries of <ReplaceOp@[@1,1:1='b',<4>,1:1]..[@2,2:2='c',<5>,1:2]:\"foo\"> overlap with previous <ReplaceOp@[@0,0:0='a',<3>,1:0]..[@3,3:3='c',<5>,1:3]:\"bar\">";
+		assertNotNull(exc);
+		assertEquals(expecting, exc.getMessage());
+	}
+
+	@Test public void testOverlappingReplace3() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(1, 2, "foo");
+		tokens.replace(0, 2, "bar");
+		stream.fill();
+// wipes prior nested replace
+		String result = tokens.getText();
+		String expecting = "barc";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testOverlappingReplace4() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(1, 2, "foo");
+		tokens.replace(1, 3, "bar");
+		stream.fill();
+// wipes prior nested replace
+		String result = tokens.getText();
+		String expecting = "abar";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testDropIdenticalReplace() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(1, 2, "foo");
+		tokens.replace(1, 2, "foo");
+		stream.fill();
+// drop previous, identical
+		String result = tokens.getText();
+		String expecting = "afooc";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testDropPrevCoveredInsert() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(1, "foo");
+		tokens.replace(1, 2, "foo");
+		stream.fill();
+// kill prev insert
+		String result = tokens.getText();
+		String expecting = "afoofoo";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testLeaveAloneDisjointInsert() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(1, "x");
+		tokens.replace(2, 3, "foo");
+		String result = tokens.getText();
+		String expecting = "axbfoo";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testLeaveAloneDisjointInsert2() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abcc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.replace(2, 3, "foo");
+		tokens.insertBefore(1, "x");
+		String result = tokens.getText();
+		String expecting = "axbfoo";
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testInsertBeforeTokenThenDeleteThatToken() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar t;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "abc";
+		LexerInterpreter lexEngine = new LexerInterpreter(g, input);
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(2, "y");
+		tokens.delete(2);
+		String result = tokens.getText();
+		String expecting = "aby";
+		assertEquals(expecting, result);
+	}
+
+}

From b433c2ae345dab266cdf1c9aee700c6e042a97a7 Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Tue, 3 Apr 2012 16:11:26 -0500
Subject: [PATCH 14/34] Add warning 125: implicit definition of token in parser

---
 .../src/org/antlr/v4/semantics/SemanticPipeline.java | 12 +++++++++++-
 tool/src/org/antlr/v4/tool/ErrorType.java            |  2 ++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java
index 127733b5d..a75fd4e74 100644
--- a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java
+++ b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java
@@ -31,6 +31,7 @@ package org.antlr.v4.semantics;
 
 import org.antlr.v4.analysis.LeftRecursiveRuleTransformer;
 import org.antlr.v4.parse.ANTLRParser;
+import org.antlr.v4.runtime.Token;
 import org.antlr.v4.tool.*;
 import org.antlr.v4.tool.ast.GrammarAST;
 
@@ -174,6 +175,9 @@ public class SemanticPipeline {
 				String lit = alias.getChild(1).getText();
 				g.defineTokenAlias(name, lit);
 			}
+			else {
+				g.defineTokenName(alias.getText());
+			}
 		}
 
 		// DEFINE TOKEN TYPES FOR X : 'x' ; RULES
@@ -187,7 +191,13 @@ public class SemanticPipeline {
 		   */
 
 		// DEFINE TOKEN TYPES FOR TOKEN REFS LIKE ID, INT
-		for (GrammarAST idAST : tokenIDs) { g.defineTokenName(idAST.getText()); }
+		for (GrammarAST idAST : tokenIDs) {
+			if (g.getTokenType(idAST.getText()) == Token.INVALID_TYPE) {
+				g.tool.errMgr.grammarError(ErrorType.IMPLICIT_TOKEN_DEFINITION, g.fileName, idAST.token, idAST.getText());
+			}
+
+			g.defineTokenName(idAST.getText());
+		}
 
 		// DEFINE TOKEN TYPES FOR STRING LITERAL REFS LIKE 'while', ';'
 		for (String s : strings) { g.defineStringLiteral(s); }
diff --git a/tool/src/org/antlr/v4/tool/ErrorType.java b/tool/src/org/antlr/v4/tool/ErrorType.java
index 3d406d9dd..07ca65eec 100644
--- a/tool/src/org/antlr/v4/tool/ErrorType.java
+++ b/tool/src/org/antlr/v4/tool/ErrorType.java
@@ -144,6 +144,8 @@ public enum ErrorType {
 	RULE_WITH_TOO_FEW_ALT_LABELS(122, "rule <arg>: must label all alternatives or none", ErrorSeverity.ERROR),
 	ALT_LABEL_REDEF(123, "rule alt label <arg> redefined in rule <arg2>, originally in <arg3>", ErrorSeverity.ERROR),
 	ALT_LABEL_CONFLICTS_WITH_RULE(124, "rule alt label <arg> conflicts with rule <arg2>", ErrorSeverity.ERROR),
+	IMPLICIT_TOKEN_DEFINITION(125, "implicit definition of token <arg> in parser", ErrorSeverity.WARNING),
+
 	/** Documentation comment is unterminated */
     //UNTERMINATED_DOC_COMMENT(, "", ErrorSeverity.ERROR),
 

From c3e5c42c519897bec0a17c70cc3fe54f33594e1d Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Tue, 3 Apr 2012 16:42:22 -0500
Subject: [PATCH 15/34] Fix (broken) unit tests which included implicitly
 defined tokens

---
 .../antlr/v4/test/TestAttributeChecks.java    | 60 ++++++++++---------
 .../v4/test/TestBasicSemanticErrors.java      | 19 +++---
 .../antlr/v4/test/TestCompositeGrammars.java  | 10 +++-
 .../org/antlr/v4/test/TestSymbolIssues.java   |  9 +--
 4 files changed, 54 insertions(+), 44 deletions(-)

diff --git a/tool/test/org/antlr/v4/test/TestAttributeChecks.java b/tool/test/org/antlr/v4/test/TestAttributeChecks.java
index 2d00e7ad3..137b451d4 100644
--- a/tool/test/org/antlr/v4/test/TestAttributeChecks.java
+++ b/tool/test/org/antlr/v4/test/TestAttributeChecks.java
@@ -9,6 +9,7 @@ public class TestAttributeChecks extends BaseTest {
     String attributeTemplate =
         "parser grammar A;\n"+
         "@members {<members>}\n" +
+		"tokens{ID;}\n" +
         "a[int x] returns [int y]\n" +
         "@init {<init>}\n" +
         "    :   id=ID ids+=ID lab=b[34] {\n" +
@@ -36,8 +37,8 @@ public class TestAttributeChecks extends BaseTest {
 		"$lab.e",		"",
 		"$ids",			"",
 
-		"$c",			"error(63): A.g4:4:8: unknown attribute reference c in $c\n",
-		"$a.q",			"error(65): A.g4:4:10: unknown attribute q for rule a in $a.q\n",
+		"$c",			"error(63): A.g4:5:8: unknown attribute reference c in $c\n",
+		"$a.q",			"error(65): A.g4:5:10: unknown attribute q for rule a in $a.q\n",
     };
 
 	String[] inlineChecks = {
@@ -58,19 +59,19 @@ public class TestAttributeChecks extends BaseTest {
 	};
 
 	String[] bad_inlineChecks = {
-		"$lab",			"error(67): A.g4:6:4: missing attribute access on rule reference lab in $lab\n",
-		"$q",           "error(63): A.g4:6:4: unknown attribute reference q in $q\n",
-		"$q.y",         "error(63): A.g4:6:4: unknown attribute reference q in $q.y\n",
-		"$q = 3",       "error(63): A.g4:6:4: unknown attribute reference q in $q\n",
-		"$q = 3;",      "error(63): A.g4:6:4: unknown attribute reference q in $q = 3;\n",
-		"$q.y = 3;",    "error(63): A.g4:6:4: unknown attribute reference q in $q.y = 3;\n",
-		"$q = $blort;", "error(63): A.g4:6:4: unknown attribute reference q in $q = $blort;\n" +
-						"error(63): A.g4:6:9: unknown attribute reference blort in $blort\n",
-		"$a.ick",       "error(65): A.g4:6:6: unknown attribute ick for rule a in $a.ick\n",
-		"$a.ick = 3;",  "error(65): A.g4:6:6: unknown attribute ick for rule a in $a.ick = 3;\n",
-		"$b.d",         "error(64): A.g4:6:6: cannot access rule d's parameter: $b.d\n",  // can't see rule ref's arg
-		"$d.text",      "error(63): A.g4:6:4: unknown attribute reference d in $d.text\n", // valid rule, but no ref
-		"$lab.d",		"error(64): A.g4:6:8: cannot access rule d's parameter: $lab.d\n",
+		"$lab",			"error(67): A.g4:7:4: missing attribute access on rule reference lab in $lab\n",
+		"$q",           "error(63): A.g4:7:4: unknown attribute reference q in $q\n",
+		"$q.y",         "error(63): A.g4:7:4: unknown attribute reference q in $q.y\n",
+		"$q = 3",       "error(63): A.g4:7:4: unknown attribute reference q in $q\n",
+		"$q = 3;",      "error(63): A.g4:7:4: unknown attribute reference q in $q = 3;\n",
+		"$q.y = 3;",    "error(63): A.g4:7:4: unknown attribute reference q in $q.y = 3;\n",
+		"$q = $blort;", "error(63): A.g4:7:4: unknown attribute reference q in $q = $blort;\n" +
+						"error(63): A.g4:7:9: unknown attribute reference blort in $blort\n",
+		"$a.ick",       "error(65): A.g4:7:6: unknown attribute ick for rule a in $a.ick\n",
+		"$a.ick = 3;",  "error(65): A.g4:7:6: unknown attribute ick for rule a in $a.ick = 3;\n",
+		"$b.d",         "error(64): A.g4:7:6: cannot access rule d's parameter: $b.d\n",  // can't see rule ref's arg
+		"$d.text",      "error(63): A.g4:7:4: unknown attribute reference d in $d.text\n", // valid rule, but no ref
+		"$lab.d",		"error(64): A.g4:7:8: cannot access rule d's parameter: $lab.d\n",
 	};
 
 	String[] finallyChecks = {
@@ -84,20 +85,20 @@ public class TestAttributeChecks extends BaseTest {
 		"$id.text",		"",
 		"$ids",			"",
 
-		"$lab",			"error(67): A.g4:9:14: missing attribute access on rule reference lab in $lab\n",
-		"$q",           "error(63): A.g4:9:14: unknown attribute reference q in $q\n",
-		"$q.y",         "error(63): A.g4:9:14: unknown attribute reference q in $q.y\n",
-		"$q = 3",       "error(63): A.g4:9:14: unknown attribute reference q in $q\n",
-		"$q = 3;",      "error(63): A.g4:9:14: unknown attribute reference q in $q = 3;\n",
-		"$q.y = 3;",    "error(63): A.g4:9:14: unknown attribute reference q in $q.y = 3;\n",
-		"$q = $blort;", "error(63): A.g4:9:14: unknown attribute reference q in $q = $blort;\n" +
-						"error(63): A.g4:9:19: unknown attribute reference blort in $blort\n",
-		"$a.ick",       "error(65): A.g4:9:16: unknown attribute ick for rule a in $a.ick\n",
-		"$a.ick = 3;",  "error(65): A.g4:9:16: unknown attribute ick for rule a in $a.ick = 3;\n",
-		"$b.e",			"error(63): A.g4:9:14: unknown attribute reference b in $b.e\n", // can't see rule refs outside alts
-		"$b.d",         "error(63): A.g4:9:14: unknown attribute reference b in $b.d\n",
-		"$c.text",      "error(63): A.g4:9:14: unknown attribute reference c in $c.text\n",
-		"$lab.d",		"error(64): A.g4:9:18: cannot access rule d's parameter: $lab.d\n",
+		"$lab",			"error(67): A.g4:10:14: missing attribute access on rule reference lab in $lab\n",
+		"$q",           "error(63): A.g4:10:14: unknown attribute reference q in $q\n",
+		"$q.y",         "error(63): A.g4:10:14: unknown attribute reference q in $q.y\n",
+		"$q = 3",       "error(63): A.g4:10:14: unknown attribute reference q in $q\n",
+		"$q = 3;",      "error(63): A.g4:10:14: unknown attribute reference q in $q = 3;\n",
+		"$q.y = 3;",    "error(63): A.g4:10:14: unknown attribute reference q in $q.y = 3;\n",
+		"$q = $blort;", "error(63): A.g4:10:14: unknown attribute reference q in $q = $blort;\n" +
+						"error(63): A.g4:10:19: unknown attribute reference blort in $blort\n",
+		"$a.ick",       "error(65): A.g4:10:16: unknown attribute ick for rule a in $a.ick\n",
+		"$a.ick = 3;",  "error(65): A.g4:10:16: unknown attribute ick for rule a in $a.ick = 3;\n",
+		"$b.e",			"error(63): A.g4:10:14: unknown attribute reference b in $b.e\n", // can't see rule refs outside alts
+		"$b.d",         "error(63): A.g4:10:14: unknown attribute reference b in $b.d\n",
+		"$c.text",      "error(63): A.g4:10:14: unknown attribute reference c in $c.text\n",
+		"$lab.d",		"error(64): A.g4:10:18: cannot access rule d's parameter: $lab.d\n",
 	};
 
 	String[] dynMembersChecks = {
@@ -200,6 +201,7 @@ public class TestAttributeChecks extends BaseTest {
 	@Test public void testTokenRef() throws RecognitionException {
 		String grammar =
 			"parser grammar S;\n" +
+			"tokens{ID;}\n" +
 			"a : x=ID {Token t = $x; t = $ID;} ;\n";
 		String expected =
 			"";
diff --git a/tool/test/org/antlr/v4/test/TestBasicSemanticErrors.java b/tool/test/org/antlr/v4/test/TestBasicSemanticErrors.java
index c8903c9e1..524693ace 100644
--- a/tool/test/org/antlr/v4/test/TestBasicSemanticErrors.java
+++ b/tool/test/org/antlr/v4/test/TestBasicSemanticErrors.java
@@ -37,6 +37,7 @@ public class TestBasicSemanticErrors extends BaseTest {
         "parser grammar U;\n" +
         "options { foo=bar; k=\"3\";}\n" +
         "tokens {\n" +
+		"        ID;\n" +
         "        f='fkj';\n" +
         "        S = 'a';\n" +
         "}\n" +
@@ -52,16 +53,16 @@ public class TestBasicSemanticErrors extends BaseTest {
         // YIELDS
 		"warning(83): U.g4:2:10: illegal option foo\n" +
 		"warning(83): U.g4:2:19: illegal option k\n" +
-		"error(60): U.g4:4:8: token names must start with an uppercase letter: f\n" +
-		"error(59): U.g4:4:8: can't assign string value to token name f in non-combined grammar\n" +
-		"error(59): U.g4:5:8: can't assign string value to token name S in non-combined grammar\n" +
-		"warning(83): U.g4:8:10: illegal option x\n" +
+		"error(60): U.g4:5:8: token names must start with an uppercase letter: f\n" +
+		"error(59): U.g4:5:8: can't assign string value to token name f in non-combined grammar\n" +
+		"error(59): U.g4:6:8: can't assign string value to token name S in non-combined grammar\n" +
+		"warning(83): U.g4:9:10: illegal option x\n" +
+		"error(54): U.g4:9:0: repeated grammar prequel spec (option, token, or import); please merge\n" +
 		"error(54): U.g4:8:0: repeated grammar prequel spec (option, token, or import); please merge\n" +
-		"error(54): U.g4:7:0: repeated grammar prequel spec (option, token, or import); please merge\n" +
-		"warning(83): U.g4:11:10: illegal option blech\n" +
-		"warning(83): U.g4:11:21: illegal option greedy\n" +
-		"warning(83): U.g4:14:16: illegal option ick\n" +
-		"warning(83): U.g4:15:16: illegal option x\n",
+		"warning(83): U.g4:12:10: illegal option blech\n" +
+		"warning(83): U.g4:12:21: illegal option greedy\n" +
+		"warning(83): U.g4:15:16: illegal option ick\n" +
+		"warning(83): U.g4:16:16: illegal option x\n",
     };
 
 	@Test public void testU() { super.testErrors(U, false); }
diff --git a/tool/test/org/antlr/v4/test/TestCompositeGrammars.java b/tool/test/org/antlr/v4/test/TestCompositeGrammars.java
index ea2be8676..4af6c53fd 100644
--- a/tool/test/org/antlr/v4/test/TestCompositeGrammars.java
+++ b/tool/test/org/antlr/v4/test/TestCompositeGrammars.java
@@ -501,29 +501,34 @@ public class TestCompositeGrammars extends BaseTest {
 		ErrorQueue equeue = new ErrorQueue();
 		String slave =
 			"parser grammar T;\n" +
+			"tokens{T;}\n" +
 			"x : T ;\n" ;
 		mkdir(tmpdir);
 		writeFile(tmpdir, "T.g4", slave);
 		slave =
 			"parser grammar S;\n" +
 			"import T;\n" +
+			"tokens{S;}\n" +
 			"y : S ;\n" ;
 		mkdir(tmpdir);
 		writeFile(tmpdir, "S.g4", slave);
 
 		slave =
 			"parser grammar C;\n" +
+			"tokens{C;}\n" +
 			"i : C ;\n" ;
 		mkdir(tmpdir);
 		writeFile(tmpdir, "C.g4", slave);
 		slave =
 			"parser grammar B;\n" +
+			"tokens{B;}\n" +
 			"j : B ;\n" ;
 		mkdir(tmpdir);
 		writeFile(tmpdir, "B.g4", slave);
 		slave =
 			"parser grammar A;\n" +
 			"import B,C;\n" +
+			"tokens{A;}\n" +
 			"k : A ;\n" ;
 		mkdir(tmpdir);
 		writeFile(tmpdir, "A.g4", slave);
@@ -531,12 +536,13 @@ public class TestCompositeGrammars extends BaseTest {
 		String master =
 			"grammar M;\n" +
 			"import S,A;\n" +
+			"tokens{M;}\n" +
 			"a : M ;\n" ;
 		writeFile(tmpdir, "M.g4", master);
 		Grammar g = new Grammar(tmpdir+"/M.g4", master, equeue);
 
-		assertEquals(equeue.errors.toString(), "[]");
-		assertEquals(equeue.warnings.toString(), "[]");
+		assertEquals("[]", equeue.errors.toString());
+		assertEquals("[]", equeue.warnings.toString());
 		String expectedTokenIDToTypeMap = "{EOF=-1, M=3, S=4, T=5, A=6, B=7, C=8}";
 		String expectedStringLiteralToTypeMap = "{}";
 		String expectedTypeToTokenList = "[M, S, T, A, B, C]";
diff --git a/tool/test/org/antlr/v4/test/TestSymbolIssues.java b/tool/test/org/antlr/v4/test/TestSymbolIssues.java
index 75dc79304..f9acba031 100644
--- a/tool/test/org/antlr/v4/test/TestSymbolIssues.java
+++ b/tool/test/org/antlr/v4/test/TestSymbolIssues.java
@@ -34,7 +34,7 @@ public class TestSymbolIssues extends BaseTest {
     static String[] B = {
         // INPUT
         "parser grammar B;\n" +
-        "tokens { X='x'; Y; }\n" +
+        "tokens { ID; FOO; X='x'; Y; }\n" +
         "\n" +
         "a : s=ID b+=ID X=ID '.' ;\n" +
         "\n" +
@@ -42,7 +42,7 @@ public class TestSymbolIssues extends BaseTest {
         "\n" +
         "s : FOO ;",
         // YIELDS
-        "error(59): B.g4:2:9: can't assign string value to token name X in non-combined grammar\n" +
+        "error(59): B.g4:2:18: can't assign string value to token name X in non-combined grammar\n" +
 		"error(69): B.g4:4:4: label s conflicts with rule with same name\n" +
 		"error(69): B.g4:4:9: label b conflicts with rule with same name\n" +
 		"error(70): B.g4:4:15: label X conflicts with token with same name\n" +
@@ -52,6 +52,7 @@ public class TestSymbolIssues extends BaseTest {
     static String[] D = {
         // INPUT
         "parser grammar D;\n" +
+		"tokens{ID;}\n" +
         "a[int j] \n" +
         "        :       i=ID j=ID ;\n" +
         "\n" +
@@ -61,8 +62,8 @@ public class TestSymbolIssues extends BaseTest {
         "        :       ID ;",
 
         // YIELDS
-        "error(72): D.g4:3:21: label j conflicts with rule a's return value or parameter with same name\n" +
-		"error(76): D.g4:5:0: rule b's argument i conflicts a return value with same name\n"
+        "error(72): D.g4:4:21: label j conflicts with rule a's return value or parameter with same name\n" +
+		"error(76): D.g4:6:0: rule b's argument i conflicts a return value with same name\n"
     };
 
 	static String[] E = {

From 9ce85cc6e4f50a38290402e103853187e10f4283 Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Wed, 4 Apr 2012 09:38:28 -0500
Subject: [PATCH 16/34] Add error 126: implicit definition of string literal
 token in non-combined grammar

---
 .../antlr/v4/semantics/SemanticPipeline.java    | 17 +++++++++++++----
 tool/src/org/antlr/v4/tool/ErrorType.java       |  1 +
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java
index a75fd4e74..740d11a3f 100644
--- a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java
+++ b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java
@@ -117,7 +117,7 @@ public class SemanticPipeline {
 		}
 		else {
 			assignTokenTypes(g, collector.tokensDefs,
-							 collector.tokenIDRefs, collector.strings);
+							 collector.tokenIDRefs, collector.terminals);
 		}
 
 		// CHECK RULE REFS NOW (that we've defined rules in grammar)
@@ -164,7 +164,7 @@ public class SemanticPipeline {
 	}
 
 	void assignTokenTypes(Grammar g, List<GrammarAST> tokensDefs,
-						  List<GrammarAST> tokenIDs, Set<String> strings)
+						  List<GrammarAST> tokenIDs, List<GrammarAST> terminals)
 	{
 		//Grammar G = g.getOutermostGrammar(); // put in root, even if imported
 
@@ -199,8 +199,17 @@ public class SemanticPipeline {
 			g.defineTokenName(idAST.getText());
 		}
 
-		// DEFINE TOKEN TYPES FOR STRING LITERAL REFS LIKE 'while', ';'
-		for (String s : strings) { g.defineStringLiteral(s); }
+		// VERIFY TOKEN TYPES FOR STRING LITERAL REFS LIKE 'while', ';'
+		for (GrammarAST termAST : terminals) {
+			if (termAST.getType() != ANTLRParser.STRING_LITERAL) {
+				continue;
+			}
+
+			if (g.getTokenType(termAST.getText()) == Token.INVALID_TYPE) {
+				g.tool.errMgr.grammarError(ErrorType.IMPLICIT_STRING_DEFINITION, g.fileName, termAST.token, termAST.getText());
+			}
+		}
+
 		g.tool.log("semantics", "tokens="+g.tokenNameToTypeMap);
         g.tool.log("semantics", "strings="+g.stringLiteralToTypeMap);
 	}
diff --git a/tool/src/org/antlr/v4/tool/ErrorType.java b/tool/src/org/antlr/v4/tool/ErrorType.java
index 07ca65eec..d6fdea15f 100644
--- a/tool/src/org/antlr/v4/tool/ErrorType.java
+++ b/tool/src/org/antlr/v4/tool/ErrorType.java
@@ -145,6 +145,7 @@ public enum ErrorType {
 	ALT_LABEL_REDEF(123, "rule alt label <arg> redefined in rule <arg2>, originally in <arg3>", ErrorSeverity.ERROR),
 	ALT_LABEL_CONFLICTS_WITH_RULE(124, "rule alt label <arg> conflicts with rule <arg2>", ErrorSeverity.ERROR),
 	IMPLICIT_TOKEN_DEFINITION(125, "implicit definition of token <arg> in parser", ErrorSeverity.WARNING),
+	IMPLICIT_STRING_DEFINITION(126, "cannot create implicit token for string literal <arg> in non-combined grammar", ErrorSeverity.ERROR),
 
 	/** Documentation comment is unterminated */
     //UNTERMINATED_DOC_COMMENT(, "", ErrorSeverity.ERROR),

From 251490cfc9ee34bb41e9fc3324fce004b6b80e10 Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Wed, 4 Apr 2012 09:39:44 -0500
Subject: [PATCH 17/34] Fix unit tests which included implicitly defined string
 literal tokens

---
 tool/test/org/antlr/v4/test/TestATNConstruction.java | 4 ++--
 tool/test/org/antlr/v4/test/TestSymbolIssues.java    | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/tool/test/org/antlr/v4/test/TestATNConstruction.java b/tool/test/org/antlr/v4/test/TestATNConstruction.java
index 7d6f4ebf5..2f0a8c211 100644
--- a/tool/test/org/antlr/v4/test/TestATNConstruction.java
+++ b/tool/test/org/antlr/v4/test/TestATNConstruction.java
@@ -360,14 +360,14 @@ public class TestATNConstruction extends BaseTest {
 	@Test public void testNestedAstar() throws Exception {
 		Grammar g = new Grammar(
 			"parser grammar P;\n"+
-			"a : (',' ID*)*;");
+			"a : (COMMA ID*)*;");
 		String expecting =
 			"RuleStart_a_0->StarLoopEntry_13\n" +
 			"StarLoopEntry_13->StarBlockStart_11\n" +
 			"StarLoopEntry_13->s14\n" +
 			"StarBlockStart_11->s2\n" +
 			"s14->RuleStop_a_1\n" +
-			"s2-','->StarLoopEntry_8\n" +
+			"s2-COMMA->StarLoopEntry_8\n" +
 			"RuleStop_a_1-EOF->s16\n" +
 			"StarLoopEntry_8->StarBlockStart_6\n" +
 			"StarLoopEntry_8->s9\n" +
diff --git a/tool/test/org/antlr/v4/test/TestSymbolIssues.java b/tool/test/org/antlr/v4/test/TestSymbolIssues.java
index f9acba031..56f29689e 100644
--- a/tool/test/org/antlr/v4/test/TestSymbolIssues.java
+++ b/tool/test/org/antlr/v4/test/TestSymbolIssues.java
@@ -46,7 +46,8 @@ public class TestSymbolIssues extends BaseTest {
 		"error(69): B.g4:4:4: label s conflicts with rule with same name\n" +
 		"error(69): B.g4:4:9: label b conflicts with rule with same name\n" +
 		"error(70): B.g4:4:15: label X conflicts with token with same name\n" +
-		"error(75): B.g4:6:9: label x type mismatch with previous definition: TOKEN_LIST_LABEL!=TOKEN_LABEL\n"
+		"error(75): B.g4:6:9: label x type mismatch with previous definition: TOKEN_LIST_LABEL!=TOKEN_LABEL\n" +
+		"error(126): B.g4:4:20: cannot create implicit token for string literal '.' in non-combined grammar\n"
     };
 
     static String[] D = {

From 1c9e52b5d639a9394ef3227d94fb6ea4073db50e Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Wed, 4 Apr 2012 09:50:51 -0500
Subject: [PATCH 18/34] Use CopyOnWriteArrayList instead of
 Collections.synchronizedList

---
 tool/src/org/antlr/v4/Tool.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java
index dad9c8bab..741107254 100644
--- a/tool/src/org/antlr/v4/Tool.java
+++ b/tool/src/org/antlr/v4/Tool.java
@@ -74,6 +74,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
 
 public class Tool {
 	public String VERSION = "4.0-"+new Date();
@@ -166,8 +167,7 @@ public class Tool {
 	public ErrorManager errMgr = new ErrorManager(this);
     public LogManager logMgr = new LogManager();
 
-	List<ANTLRToolListener> listeners =
-	Collections.synchronizedList(new ArrayList<ANTLRToolListener>());
+	List<ANTLRToolListener> listeners = new CopyOnWriteArrayList<ANTLRToolListener>();
 
 	/** Track separately so if someone adds a listener, it's the only one
 	 *  instead of it and the default stderr listener.

From 2c1f673f621dba72df40ceb161301768dfdbb12a Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Wed, 4 Apr 2012 09:51:23 -0500
Subject: [PATCH 19/34] Mark ErrorType fields final

---
 tool/src/org/antlr/v4/tool/ErrorType.java | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tool/src/org/antlr/v4/tool/ErrorType.java b/tool/src/org/antlr/v4/tool/ErrorType.java
index 3d406d9dd..0d49e560c 100644
--- a/tool/src/org/antlr/v4/tool/ErrorType.java
+++ b/tool/src/org/antlr/v4/tool/ErrorType.java
@@ -185,16 +185,18 @@ public enum ErrorType {
 
     ;
 
-	public String msg;
-    public int code; // unique, deterministic unchanging error code once we release
-    public ErrorSeverity severity;
-    public Boolean abortsAnalysis;
-    public Boolean abortsCodegen;
+	public final String msg;
+    public final int code; // unique, deterministic unchanging error code once we release
+    public final ErrorSeverity severity;
+    public final Boolean abortsAnalysis;
+    public final Boolean abortsCodegen;
 
 	ErrorType(int code, String msg, ErrorSeverity severity) {
         this.code = code;
 		this.msg = msg;
         this.severity = severity;
+		this.abortsAnalysis = false;
+		this.abortsCodegen = false;
 	}
 
 //	ErrorType(String msg, ErrorSeverity severity, boolean abortsAnalysis) {

From 16046a7ec17ffec9dcc263058ecb69e441e79056 Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Wed, 4 Apr 2012 09:52:06 -0500
Subject: [PATCH 20/34] Proper maven reference for org.abego.treelayout.core
 dependency

---
 runtime/Java/pom.xml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml
index d0e95cf41..c85c24cb5 100644
--- a/runtime/Java/pom.xml
+++ b/runtime/Java/pom.xml
@@ -23,17 +23,17 @@
             <scope>compile</scope>
         </dependency>
         <dependency>
-            <groupId>org.abego</groupId>
-            <artifactId>treelayout.core</artifactId>
-            <version>1.0</version>
-            <scope>system</scope>
-            <systemPath>${project.basedir}/lib/org.abego.treelayout.core.jar</systemPath>
+            <groupId>org.abego.treelayout</groupId>
+            <artifactId>org.abego.treelayout.core</artifactId>
+            <version>1.0.1</version>
+            <scope>compile</scope>
         </dependency>
     </dependencies>
 
     <build>
 
         <sourceDirectory>src</sourceDirectory>
+        <resources/>
 
         <plugins>
             <plugin>

From 64e0a8734f8bd5d479f9c3ac41bddfd00cd37b2c Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Wed, 4 Apr 2012 09:53:10 -0500
Subject: [PATCH 21/34] Report warnings as well as errors

---
 tool/test/org/antlr/v4/test/BaseTest.java | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java
index a26d63303..0ee9561f9 100644
--- a/tool/test/org/antlr/v4/test/BaseTest.java
+++ b/tool/test/org/antlr/v4/test/BaseTest.java
@@ -402,7 +402,7 @@ public abstract class BaseTest {
 			Tool antlr = newTool(optionsA);
 			antlr.addListener(equeue);
 			antlr.processGrammarsOnCommandLine();
-			if ( equeue.errors.size()>0 ) {
+			if ( !equeue.errors.isEmpty() ) {
 				allIsWell = false;
 				System.err.println("antlr reports errors from "+options);
 				for (int i = 0; i < equeue.errors.size(); i++) {
@@ -413,6 +413,13 @@ public abstract class BaseTest {
 				System.out.println(grammarStr);
 				System.out.println("###");
 			}
+			if ( !equeue.warnings.isEmpty() ) {
+				System.err.println("antlr reports warnings from "+options);
+				for (int i = 0; i < equeue.warnings.size(); i++) {
+					ANTLRMessage msg = equeue.warnings.get(i);
+					System.err.println(msg);
+				}
+			}
 		}
 		catch (Exception e) {
 			allIsWell = false;

From 70825902408948581bca6f10d272a201eb6b33e9 Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Wed, 4 Apr 2012 09:56:32 -0500
Subject: [PATCH 22/34] Allow tests to use the default tool listener for error
 reporting (prints formatted error messages). Use this in TestPerformance.

---
 tool/test/org/antlr/v4/test/BaseTest.java     | 25 +++++++++++++++----
 .../antlr/v4/test/TestCompositeGrammars.java  |  2 +-
 .../org/antlr/v4/test/TestPerformance.java    |  2 +-
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java
index 0ee9561f9..0340c6649 100644
--- a/tool/test/org/antlr/v4/test/BaseTest.java
+++ b/tool/test/org/antlr/v4/test/BaseTest.java
@@ -54,6 +54,7 @@ import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.runtime.misc.Nullable;
 import org.antlr.v4.semantics.SemanticPipeline;
 import org.antlr.v4.tool.ANTLRMessage;
+import org.antlr.v4.tool.DefaultToolListener;
 import org.antlr.v4.tool.DOTGenerator;
 import org.antlr.v4.tool.Grammar;
 import org.antlr.v4.tool.GrammarSemanticsMessage;
@@ -383,7 +384,7 @@ public abstract class BaseTest {
 
 
 	/** Return true if all is ok, no errors */
-	protected boolean antlr(String fileName, String grammarFileName, String grammarStr, String... extraOptions) {
+	protected boolean antlr(String fileName, String grammarFileName, String grammarStr, boolean defaultListener, String... extraOptions) {
 		boolean allIsWell = true;
 		System.out.println("dir "+tmpdir);
 		mkdir(tmpdir);
@@ -401,9 +402,12 @@ public abstract class BaseTest {
 			ErrorQueue equeue = new ErrorQueue();
 			Tool antlr = newTool(optionsA);
 			antlr.addListener(equeue);
+			if (defaultListener) {
+				antlr.addListener(new DefaultToolListener(antlr));
+			}
 			antlr.processGrammarsOnCommandLine();
-			if ( !equeue.errors.isEmpty() ) {
-				allIsWell = false;
+			allIsWell = equeue.errors.isEmpty();
+			if ( !defaultListener && !equeue.errors.isEmpty() ) {
 				System.err.println("antlr reports errors from "+options);
 				for (int i = 0; i < equeue.errors.size(); i++) {
 					ANTLRMessage msg = equeue.errors.get(i);
@@ -413,7 +417,7 @@ public abstract class BaseTest {
 				System.out.println(grammarStr);
 				System.out.println("###");
 			}
-			if ( !equeue.warnings.isEmpty() ) {
+			if ( !defaultListener && !equeue.warnings.isEmpty() ) {
 				System.err.println("antlr reports warnings from "+options);
 				for (int i = 0; i < equeue.warnings.size(); i++) {
 					ANTLRMessage msg = equeue.warnings.get(i);
@@ -485,9 +489,20 @@ public abstract class BaseTest {
 													@Nullable String parserName,
 													String lexerName,
 													String... extraOptions)
+	{
+		return rawGenerateAndBuildRecognizer(grammarFileName, grammarStr, parserName, lexerName, false, extraOptions);
+	}
+
+	/** Return true if all is well */
+	protected boolean rawGenerateAndBuildRecognizer(String grammarFileName,
+													String grammarStr,
+													@Nullable String parserName,
+													String lexerName,
+													boolean defaultListener,
+													String... extraOptions)
 	{
 		boolean allIsWell =
-			antlr(grammarFileName, grammarFileName, grammarStr, extraOptions);
+			antlr(grammarFileName, grammarFileName, grammarStr, defaultListener, extraOptions);
 		boolean ok;
 		List<String> files = new ArrayList<String>();
 		if ( lexerName!=null ) {
diff --git a/tool/test/org/antlr/v4/test/TestCompositeGrammars.java b/tool/test/org/antlr/v4/test/TestCompositeGrammars.java
index ea2be8676..d2245bbc4 100644
--- a/tool/test/org/antlr/v4/test/TestCompositeGrammars.java
+++ b/tool/test/org/antlr/v4/test/TestCompositeGrammars.java
@@ -653,7 +653,7 @@ public class TestCompositeGrammars extends BaseTest {
 			"s : a ;\n" +
 			"B : 'b' ;" + // defines B from inherited token space
 			"WS : (' '|'\\n') {skip();} ;\n" ;
-		boolean ok = antlr("M.g4", "M.g4", master);
+		boolean ok = antlr("M.g4", "M.g4", master, false);
 		boolean expecting = true; // should be ok
 		assertEquals(expecting, ok);
 	}
diff --git a/tool/test/org/antlr/v4/test/TestPerformance.java b/tool/test/org/antlr/v4/test/TestPerformance.java
index 84773cb7a..65bbe9b0d 100644
--- a/tool/test/org/antlr/v4/test/TestPerformance.java
+++ b/tool/test/org/antlr/v4/test/TestPerformance.java
@@ -433,7 +433,7 @@ public class TestPerformance extends BaseTest {
             extraOptions.add("-atn");
         }
         String[] extraOptionsArray = extraOptions.toArray(new String[extraOptions.size()]);
-        boolean success = rawGenerateAndBuildRecognizer(grammarFileName, body, "JavaParser", "JavaLexer", extraOptionsArray);
+        boolean success = rawGenerateAndBuildRecognizer(grammarFileName, body, "JavaParser", "JavaLexer", true, extraOptionsArray);
         assertTrue(success);
     }
 

From d054a358a4158bde67bfb910d56459f2dd0f34b4 Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Wed, 4 Apr 2012 09:57:38 -0500
Subject: [PATCH 23/34] Don't run compiler if tool reported error(s)

---
 tool/test/org/antlr/v4/test/BaseTest.java | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java
index 0340c6649..333452ab3 100644
--- a/tool/test/org/antlr/v4/test/BaseTest.java
+++ b/tool/test/org/antlr/v4/test/BaseTest.java
@@ -503,7 +503,10 @@ public abstract class BaseTest {
 	{
 		boolean allIsWell =
 			antlr(grammarFileName, grammarFileName, grammarStr, defaultListener, extraOptions);
-		boolean ok;
+		if (!allIsWell) {
+			return false;
+		}
+
 		List<String> files = new ArrayList<String>();
 		if ( lexerName!=null ) {
 			files.add(lexerName+".java");
@@ -521,8 +524,7 @@ public abstract class BaseTest {
 				files.add(grammarFileName.substring(0, grammarFileName.lastIndexOf('.'))+"BaseParseListener.java");
 			}
 		}
-		ok = compile(files.toArray(new String[files.size()]));
-		if ( !ok ) { allIsWell = false; }
+		allIsWell = compile(files.toArray(new String[files.size()]));
 		return allIsWell;
 	}
 

From e4612935f2afd5eb18b1cb79bb64eb435b41d102 Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Wed, 4 Apr 2012 10:10:33 -0500
Subject: [PATCH 24/34] Only need TokenStream (was overly restricted to
 BufferedTokenStream)

---
 .../Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
index 2fff8d18a..60d1d2b91 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
@@ -151,7 +151,7 @@ public class TokenStreamRewriter {
 	}
 
 	/** Our source stream */
-	protected final BufferedTokenStream tokens;
+	protected final TokenStream tokens;
 
 	/** You may have multiple, named streams of rewrite operations.
 	 *  I'm calling these things "programs."
@@ -162,7 +162,7 @@ public class TokenStreamRewriter {
 	/** Map String (program name) -> Integer index */
 	protected final Map<String, Integer> lastRewriteTokenIndexes;
 
-	public TokenStreamRewriter(BufferedTokenStream tokens) {
+	public TokenStreamRewriter(TokenStream tokens) {
 		this.tokens = tokens;
 		programs = new HashMap<String, List<RewriteOperation>>();
 		programs.put(DEFAULT_PROGRAM_NAME,

From a85f1c17a14e195c5ff45749087507f7dc189b5e Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Wed, 4 Apr 2012 10:14:09 -0500
Subject: [PATCH 25/34] Add TokenStreamRewriter.getSource()

---
 .../Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
index 60d1d2b91..86a34d5cd 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
@@ -170,6 +170,10 @@ public class TokenStreamRewriter {
 		lastRewriteTokenIndexes = new HashMap<String, Integer>();
 	}
 
+	public final TokenStream getSource() {
+		return tokens;
+	}
+
 	public void rollback(int instructionIndex) {
 		rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
 	}

From bbd61dca35bf942d5150431a76a105799d9e2e7c Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Wed, 4 Apr 2012 10:15:29 -0500
Subject: [PATCH 26/34] Remove TokenStreamRewriter.getOriginalText() (use
 rewriter.getSource().getText() instead)

---
 .../src/org/antlr/v4/runtime/TokenStreamRewriter.java    | 9 ---------
 tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java | 4 ++--
 2 files changed, 2 insertions(+), 11 deletions(-)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
index 86a34d5cd..7391cf87e 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
@@ -321,15 +321,6 @@ public class TokenStreamRewriter {
 		return is;
 	}
 
-	public String getOriginalText() {
-		return tokens.getText();
-	}
-
-	/** The text collected from all tokens in the input token stream */
-	public String getOriginalText(Interval interval) {
-		return tokens.getText(interval);
-	}
-
 	/** Return the text from the original tokens altered per the
 	 *  instructions given to this rewriter.
  	 */
diff --git a/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java b/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java
index 977c1b76d..4b79e2d30 100644
--- a/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java
+++ b/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java
@@ -163,7 +163,7 @@ public class TestTokenStreamRewriter extends BaseTest {
 		stream.fill();
 // replace 3 * 0 with 0
 
-		String result = tokens.getOriginalText();
+		String result = tokens.getSource().getText();
 		String expecting = "x = 3 * 0;";
 		assertEquals(expecting, result);
 
@@ -198,7 +198,7 @@ public class TestTokenStreamRewriter extends BaseTest {
 		stream.fill();
 		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
 
-		String result = tokens.getOriginalText();
+		String result = tokens.getSource().getText();
 		String expecting = "x = 3 * 0 + 2 * 0;";
 		assertEquals(expecting, result);
 

From ff2655d37979a00872c907227ea9c2ad4e6e1b0d Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Wed, 4 Apr 2012 15:20:45 -0500
Subject: [PATCH 27/34] Rename getSource() to getTokenStream()

---
 .../Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java    | 2 +-
 tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java      | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
index 7391cf87e..45c2b4e79 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
@@ -170,7 +170,7 @@ public class TokenStreamRewriter {
 		lastRewriteTokenIndexes = new HashMap<String, Integer>();
 	}
 
-	public final TokenStream getSource() {
+	public final TokenStream getTokenStream() {
 		return tokens;
 	}
 
diff --git a/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java b/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java
index 4b79e2d30..9b8248c05 100644
--- a/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java
+++ b/tool/test/org/antlr/v4/test/TestTokenStreamRewriter.java
@@ -163,7 +163,7 @@ public class TestTokenStreamRewriter extends BaseTest {
 		stream.fill();
 // replace 3 * 0 with 0
 
-		String result = tokens.getSource().getText();
+		String result = tokens.getTokenStream().getText();
 		String expecting = "x = 3 * 0;";
 		assertEquals(expecting, result);
 
@@ -198,7 +198,7 @@ public class TestTokenStreamRewriter extends BaseTest {
 		stream.fill();
 		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
 
-		String result = tokens.getSource().getText();
+		String result = tokens.getTokenStream().getText();
 		String expecting = "x = 3 * 0 + 2 * 0;";
 		assertEquals(expecting, result);
 

From e11dc07275faf339715ce78bee92d74e99d64be1 Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Wed, 4 Apr 2012 15:33:03 -0500
Subject: [PATCH 28/34] Remove TokenRewriteStream

---
 .../antlr/v4/runtime/TokenRewriteStream.java  | 607 ------------------
 1 file changed, 607 deletions(-)
 delete mode 100644 runtime/Java/src/org/antlr/v4/runtime/TokenRewriteStream.java

diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenRewriteStream.java b/runtime/Java/src/org/antlr/v4/runtime/TokenRewriteStream.java
deleted file mode 100644
index 85f09c99f..000000000
--- a/runtime/Java/src/org/antlr/v4/runtime/TokenRewriteStream.java
+++ /dev/null
@@ -1,607 +0,0 @@
-/*
- [The "BSD license"]
- Copyright (c) 2011 Terence Parr
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
-
- 1. Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
- 3. The name of the author may not be used to endorse or promote products
-    derived from this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-package org.antlr.v4.runtime;
-
-import org.antlr.v4.runtime.misc.Interval;
-import org.antlr.v4.runtime.misc.Nullable;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/** Useful for dumping out the input stream after doing some
- *  augmentation or other manipulations.
- *
- *  You can insert stuff, replace, and delete chunks.  Note that the
- *  operations are done lazily--only if you convert the buffer to a
- *  String.  This is very efficient because you are not moving data around
- *  all the time.  As the buffer of tokens is converted to strings, the
- *  toString() method(s) check to see if there is an operation at the
- *  current index.  If so, the operation is done and then normal String
- *  rendering continues on the buffer.  This is like having multiple Turing
- *  machine instruction streams (programs) operating on a single input tape. :)
- *
- *  Since the operations are done lazily at toString-time, operations do not
- *  screw up the token index values.  That is, an insert operation at token
- *  index i does not change the index values for tokens i+1..n-1.
- *
- *  Because operations never actually alter the buffer, you may always get
- *  the original token stream back without undoing anything.  Since
- *  the instructions are queued up, you can easily simulate transactions and
- *  roll back any changes if there is an error just by removing instructions.
- *  For example,
- *
- *   CharStream input = new ANTLRFileStream("input");
- *   TLexer lex = new TLexer(input);
- *   TokenRewriteStream tokens = new TokenRewriteStream(lex);
- *   T parser = new T(tokens);
- *   parser.startRule();
- *
- * 	 Then in the rules, you can execute
- *      Token t,u;
- *      ...
- *      input.insertAfter(t, "text to put after t");}
- * 		input.insertAfter(u, "text after u");}
- * 		System.out.println(tokens.toString());
- *
- *  Actually, you have to cast the 'input' to a TokenRewriteStream. :(
- *
- *  You can also have multiple "instruction streams" and get multiple
- *  rewrites from a single pass over the input.  Just name the instruction
- *  streams and use that name again when printing the buffer.  This could be
- *  useful for generating a C file and also its header file--all from the
- *  same buffer:
- *
- *      tokens.insertAfter("pass1", t, "text to put after t");}
- * 		tokens.insertAfter("pass2", u, "text after u");}
- * 		System.out.println(tokens.toString("pass1"));
- * 		System.out.println(tokens.toString("pass2"));
- *
- *  If you don't use named rewrite streams, a "default" stream is used as
- *  the first example shows.
- */
-public class TokenRewriteStream extends CommonTokenStream {
-	public static final String DEFAULT_PROGRAM_NAME = "default";
-    public static final int PROGRAM_INIT_SIZE = 100;
-	public static final int MIN_TOKEN_INDEX = 0;
-
-	// Define the rewrite operation hierarchy
-
-	public class RewriteOperation {
-        /** What index into rewrites List are we? */
-        protected int instructionIndex;
-        /** Token buffer index. */
-        protected int index;
-		protected Object text;
-
-		protected RewriteOperation(int index) {
-			this.index = index;
-		}
-
-		protected RewriteOperation(int index, Object text) {
-			this.index = index;
-			this.text = text;
-		}
-		/** Execute the rewrite operation by possibly adding to the buffer.
-		 *  Return the index of the next token to operate on.
-		 */
-		public int execute(StringBuilder buf) {
-			return index;
-		}
-
-		@Override
-		public String toString() {
-			String opName = getClass().getName();
-			int $index = opName.indexOf('$');
-			opName = opName.substring($index+1, opName.length());
-			return "<"+opName+"@"+tokens.get(index)+
-				   ":\""+text+"\">";
-		}
-	}
-
-	class InsertBeforeOp extends RewriteOperation {
-		public InsertBeforeOp(int index, Object text) {
-			super(index,text);
-		}
-
-		@Override
-		public int execute(StringBuilder buf) {
-			buf.append(text);
-			if ( tokens.get(index).getType()!=Token.EOF ) {
-				buf.append(tokens.get(index).getText());
-			}
-			return index+1;
-		}
-	}
-
-	/** I'm going to try replacing range from x..y with (y-x)+1 ReplaceOp
-	 *  instructions.
-	 */
-	class ReplaceOp extends RewriteOperation {
-		protected int lastIndex;
-		public ReplaceOp(int from, int to, Object text) {
-			super(from,text);
-			lastIndex = to;
-		}
-		@Override
-		public int execute(StringBuilder buf) {
-			if ( text!=null ) {
-				buf.append(text);
-			}
-			return lastIndex+1;
-		}
-		@Override
-		public String toString() {
-			if ( text==null ) {
-				return "<DeleteOp@"+tokens.get(index)+
-					   ".."+tokens.get(lastIndex)+">";
-			}
-			return "<ReplaceOp@"+tokens.get(index)+
-				   ".."+tokens.get(lastIndex)+":\""+text+"\">";
-		}
-	}
-
-	/** You may have multiple, named streams of rewrite operations.
-	 *  I'm calling these things "programs."
-	 *  Maps String (name) -> rewrite (List)
-	 */
-	protected Map<String, List<RewriteOperation>> programs = null;
-
-	/** Map String (program name) -> Integer index */
-	protected Map<String, Integer> lastRewriteTokenIndexes = null;
-
-	public TokenRewriteStream() {
-		init();
-	}
-
-	protected void init() {
-		programs = new HashMap<String, List<RewriteOperation>>();
-		programs.put(DEFAULT_PROGRAM_NAME, new ArrayList<RewriteOperation>(PROGRAM_INIT_SIZE));
-		lastRewriteTokenIndexes = new HashMap<String, Integer>();
-	}
-
-	public TokenRewriteStream(TokenSource tokenSource) {
-	    super(tokenSource);
-		init();
-	}
-
-	public TokenRewriteStream(TokenSource tokenSource, int channel) {
-		super(tokenSource, channel);
-		init();
-	}
-
-	public void rollback(int instructionIndex) {
-		rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
-	}
-
-	/** Rollback the instruction stream for a program so that
-	 *  the indicated instruction (via instructionIndex) is no
-	 *  longer in the stream.  UNTESTED!
-	 */
-	public void rollback(String programName, int instructionIndex) {
-		List<RewriteOperation> is = programs.get(programName);
-		if ( is!=null ) {
-			programs.put(programName, is.subList(MIN_TOKEN_INDEX,instructionIndex));
-		}
-	}
-
-	public void deleteProgram() {
-		deleteProgram(DEFAULT_PROGRAM_NAME);
-	}
-
-	/** Reset the program so that no instructions exist */
-	public void deleteProgram(String programName) {
-		rollback(programName, MIN_TOKEN_INDEX);
-	}
-
-	public void insertAfter(Token t, Object text) {
-		insertAfter(DEFAULT_PROGRAM_NAME, t, text);
-	}
-
-	public void insertAfter(int index, Object text) {
-		insertAfter(DEFAULT_PROGRAM_NAME, index, text);
-	}
-
-	public void insertAfter(String programName, Token t, Object text) {
-		insertAfter(programName,t.getTokenIndex(), text);
-	}
-
-	public void insertAfter(String programName, int index, Object text) {
-		// to insert after, just insert before next index (even if past end)
-		insertBefore(programName,index+1, text);
-	}
-
-	public void insertBefore(Token t, Object text) {
-		insertBefore(DEFAULT_PROGRAM_NAME, t, text);
-	}
-
-	public void insertBefore(int index, Object text) {
-		insertBefore(DEFAULT_PROGRAM_NAME, index, text);
-	}
-
-	public void insertBefore(String programName, Token t, Object text) {
-		insertBefore(programName, t.getTokenIndex(), text);
-	}
-
-	public void insertBefore(String programName, int index, Object text) {
-		RewriteOperation op = new InsertBeforeOp(index,text);
-		List<RewriteOperation> rewrites = getProgram(programName);
-        op.instructionIndex = rewrites.size();
-        rewrites.add(op);
-	}
-
-	public void replace(int index, Object text) {
-		replace(DEFAULT_PROGRAM_NAME, index, index, text);
-	}
-
-	public void replace(int from, int to, Object text) {
-		replace(DEFAULT_PROGRAM_NAME, from, to, text);
-	}
-
-	public void replace(Token indexT, Object text) {
-		replace(DEFAULT_PROGRAM_NAME, indexT, indexT, text);
-	}
-
-	public void replace(Token from, Token to, Object text) {
-		replace(DEFAULT_PROGRAM_NAME, from, to, text);
-	}
-
-	public void replace(String programName, int from, int to, @Nullable Object text) {
-		if ( from > to || from<0 || to<0 || to >= tokens.size() ) {
-			throw new IllegalArgumentException("replace: range invalid: "+from+".."+to+"(size="+tokens.size()+")");
-		}
-		RewriteOperation op = new ReplaceOp(from, to, text);
-		List<RewriteOperation> rewrites = getProgram(programName);
-        op.instructionIndex = rewrites.size();
-        rewrites.add(op);
-	}
-
-	public void replace(String programName, Token from, Token to, @Nullable Object text) {
-		replace(programName,
-				from.getTokenIndex(),
-				to.getTokenIndex(),
-				text);
-	}
-
-	public void delete(int index) {
-		delete(DEFAULT_PROGRAM_NAME, index, index);
-	}
-
-	public void delete(int from, int to) {
-		delete(DEFAULT_PROGRAM_NAME, from, to);
-	}
-
-	public void delete(Token indexT) {
-		delete(DEFAULT_PROGRAM_NAME, indexT, indexT);
-	}
-
-	public void delete(Token from, Token to) {
-		delete(DEFAULT_PROGRAM_NAME, from, to);
-	}
-
-	public void delete(String programName, int from, int to) {
-		replace(programName,from,to,null);
-	}
-
-	public void delete(String programName, Token from, Token to) {
-		replace(programName,from,to,null);
-	}
-
-	public int getLastRewriteTokenIndex() {
-		return getLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
-	}
-
-	protected int getLastRewriteTokenIndex(String programName) {
-		Integer I = lastRewriteTokenIndexes.get(programName);
-		if ( I==null ) {
-			return -1;
-		}
-		return I;
-	}
-
-	protected void setLastRewriteTokenIndex(String programName, int i) {
-		lastRewriteTokenIndexes.put(programName, i);
-	}
-
-	protected List<RewriteOperation> getProgram(String name) {
-		List<RewriteOperation> is = programs.get(name);
-		if ( is==null ) {
-			is = initializeProgram(name);
-		}
-		return is;
-	}
-
-	private List<RewriteOperation> initializeProgram(String name) {
-		List<RewriteOperation> is = new ArrayList<RewriteOperation>(PROGRAM_INIT_SIZE);
-		programs.put(name, is);
-		return is;
-	}
-
-	public String toOriginalString() {
-        fill();
-		return toOriginalString(MIN_TOKEN_INDEX, size()-1);
-	}
-
-	public String toOriginalString(int start, int end) {
-		StringBuilder buf = new StringBuilder();
-		for (int i=start; i>=MIN_TOKEN_INDEX && i<=end && i<tokens.size(); i++) {
-			if ( get(i).getType()!=Token.EOF ) buf.append(get(i).getText());
-		}
-		return buf.toString();
-	}
-
-	@Override
-	public String toString() { return getText(); }
-
-	public String toString(String programName) {
-        fill();
-		return toString(programName, Interval.of(0, size() - 1));
-	}
-
-	@Override
-	public String getText() {
-		fill();
-		return getText(Interval.of(0,size()-1));
-	}
-
-	@Override
-	public String getText(Interval interval) {
-		return toString(DEFAULT_PROGRAM_NAME, interval);
-	}
-
-	public String toString(String programName, Interval interval) {
-		List<RewriteOperation> rewrites = programs.get(programName);
-		int start = interval.a;
-		int stop = interval.b;
-
-        // ensure start/end are in range
-        if ( stop>tokens.size()-1 ) stop = tokens.size()-1;
-        if ( start<0 ) start = 0;
-
-        if ( rewrites==null || rewrites.isEmpty() ) {
-			return toOriginalString(start,stop); // no instructions to execute
-		}
-		StringBuilder buf = new StringBuilder();
-
-		// First, optimize instruction stream
-		Map<Integer, RewriteOperation> indexToOp = reduceToSingleOperationPerIndex(rewrites);
-
-        // Walk buffer, executing instructions and emitting tokens
-        int i = start;
-        while ( i <= stop && i < tokens.size() ) {
-			RewriteOperation op = indexToOp.get(i);
-			indexToOp.remove(i); // remove so any left have index size-1
-			Token t = tokens.get(i);
-			if ( op==null ) {
-				// no operation at that index, just dump token
-				if ( t.getType()!=Token.EOF ) buf.append(t.getText());
-				i++; // move to next token
-			}
-			else {
-				i = op.execute(buf); // execute operation and skip
-			}
-		}
-
-        // include stuff after end if it's last index in buffer
-        // So, if they did an insertAfter(lastValidIndex, "foo"), include
-        // foo if end==lastValidIndex.
-        if ( stop==tokens.size()-1 ) {
-            // Scan any remaining operations after last token
-            // should be included (they will be inserts).
-			for (RewriteOperation op : indexToOp.values()) {
-                if ( op.index >= tokens.size()-1 ) buf.append(op.text);
-            }
-        }
-        return buf.toString();
-	}
-
-	/** We need to combine operations and report invalid operations (like
-	 *  overlapping replaces that are not completed nested).  Inserts to
-	 *  same index need to be combined etc...   Here are the cases:
-	 *
-	 *  I.i.u I.j.v								leave alone, nonoverlapping
-	 *  I.i.u I.i.v								combine: Iivu
-	 *
-	 *  R.i-j.u R.x-y.v	| i-j in x-y			delete first R
-	 *  R.i-j.u R.i-j.v							delete first R
-	 *  R.i-j.u R.x-y.v	| x-y in i-j			ERROR
-	 *  R.i-j.u R.x-y.v	| boundaries overlap	ERROR
-	 *
-	 *  Delete special case of replace (text==null):
-	 *  D.i-j.u D.x-y.v	| boundaries overlap	combine to max(min)..max(right)
-	 *
-	 *  I.i.u R.x-y.v | i in (x+1)-y			delete I (since insert before
-	 *											we're not deleting i)
-	 *  I.i.u R.x-y.v | i not in (x+1)-y		leave alone, nonoverlapping
-	 *  R.x-y.v I.i.u | i in x-y				ERROR
-	 *  R.x-y.v I.x.u 							R.x-y.uv (combine, delete I)
-	 *  R.x-y.v I.i.u | i not in x-y			leave alone, nonoverlapping
-	 *
-	 *  I.i.u = insert u before op @ index i
-	 *  R.x-y.u = replace x-y indexed tokens with u
-	 *
-	 *  First we need to examine replaces.  For any replace op:
-	 *
-	 * 		1. wipe out any insertions before op within that range.
-	 *		2. Drop any replace op before that is contained completely within
-	 *         that range.
-	 *		3. Throw exception upon boundary overlap with any previous replace.
-	 *
-	 *  Then we can deal with inserts:
-	 *
-	 * 		1. for any inserts to same index, combine even if not adjacent.
-	 * 		2. for any prior replace with same left boundary, combine this
-	 *         insert with replace and delete this replace.
-	 * 		3. throw exception if index in same range as previous replace
-	 *
-	 *  Don't actually delete; make op null in list. Easier to walk list.
-	 *  Later we can throw as we add to index -> op map.
-	 *
-	 *  Note that I.2 R.2-2 will wipe out I.2 even though, technically, the
-	 *  inserted stuff would be before the replace range.  But, if you
-	 *  add tokens in front of a method body '{' and then delete the method
-	 *  body, I think the stuff before the '{' you added should disappear too.
-	 *
-	 *  Return a map from token index to operation.
-	 */
-	protected Map<Integer, RewriteOperation> reduceToSingleOperationPerIndex(List<RewriteOperation> rewrites) {
-//		System.out.println("rewrites="+rewrites);
-
-		// WALK REPLACES
-		for (int i = 0; i < rewrites.size(); i++) {
-			RewriteOperation op = rewrites.get(i);
-			if ( op==null ) continue;
-			if ( !(op instanceof ReplaceOp) ) continue;
-			ReplaceOp rop = (ReplaceOp)rewrites.get(i);
-			// Wipe prior inserts within range
-			List<? extends InsertBeforeOp> inserts = getKindOfOps(rewrites, InsertBeforeOp.class, i);
-			for (InsertBeforeOp iop : inserts) {
-				if ( iop.index == rop.index ) {
-					// E.g., insert before 2, delete 2..2; update replace
-					// text to include insert before, kill insert
-					rewrites.set(iop.instructionIndex, null);
-					rop.text = iop.text.toString() + (rop.text!=null?rop.text.toString():"");
-				}
-				else if ( iop.index > rop.index && iop.index <= rop.lastIndex ) {
-                    // delete insert as it's a no-op.
-                    rewrites.set(iop.instructionIndex, null);
-				}
-			}
-			// Drop any prior replaces contained within
-			List<? extends ReplaceOp> prevReplaces = getKindOfOps(rewrites, ReplaceOp.class, i);
-			for (ReplaceOp prevRop : prevReplaces) {
-				if ( prevRop.index>=rop.index && prevRop.lastIndex <= rop.lastIndex ) {
-                    // delete replace as it's a no-op.
-                    rewrites.set(prevRop.instructionIndex, null);
-					continue;
-				}
-				// throw exception unless disjoint or identical
-				boolean disjoint =
-					prevRop.lastIndex<rop.index || prevRop.index > rop.lastIndex;
-				boolean same =
-					prevRop.index==rop.index && prevRop.lastIndex==rop.lastIndex;
-				// Delete special case of replace (text==null):
-				// D.i-j.u D.x-y.v	| boundaries overlap	combine to max(min)..max(right)
-				if ( prevRop.text==null && rop.text==null && !disjoint ) {
-					//System.out.println("overlapping deletes: "+prevRop+", "+rop);
-					rewrites.set(prevRop.instructionIndex, null); // kill first delete
-					rop.index = Math.min(prevRop.index, rop.index);
-					rop.lastIndex = Math.max(prevRop.lastIndex, rop.lastIndex);
-					System.out.println("new rop "+rop);
-				}
-				else if ( !disjoint && !same ) {
-					throw new IllegalArgumentException("replace op boundaries of "+rop+
-													   " overlap with previous "+prevRop);
-				}
-			}
-		}
-
-		// WALK INSERTS
-		for (int i = 0; i < rewrites.size(); i++) {
-			RewriteOperation op = rewrites.get(i);
-			if ( op==null ) continue;
-			if ( !(op instanceof InsertBeforeOp) ) continue;
-			InsertBeforeOp iop = (InsertBeforeOp)rewrites.get(i);
-			// combine current insert with prior if any at same index
-			List<? extends InsertBeforeOp> prevInserts = getKindOfOps(rewrites, InsertBeforeOp.class, i);
-			for (InsertBeforeOp prevIop : prevInserts) {
-				if ( prevIop.index == iop.index ) { // combine objects
-					// convert to strings...we're in process of toString'ing
-					// whole token buffer so no lazy eval issue with any templates
-					iop.text = catOpText(iop.text,prevIop.text);
-                    // delete redundant prior insert
-                    rewrites.set(prevIop.instructionIndex, null);
-				}
-			}
-			// look for replaces where iop.index is in range; error
-			List<? extends ReplaceOp> prevReplaces = getKindOfOps(rewrites, ReplaceOp.class, i);
-			for (ReplaceOp rop : prevReplaces) {
-				if ( iop.index == rop.index ) {
-					rop.text = catOpText(iop.text,rop.text);
-					rewrites.set(i, null);  // delete current insert
-					continue;
-				}
-				if ( iop.index >= rop.index && iop.index <= rop.lastIndex ) {
-					throw new IllegalArgumentException("insert op "+iop+
-													   " within boundaries of previous "+rop);
-				}
-			}
-		}
-		// System.out.println("rewrites after="+rewrites);
-		Map<Integer, RewriteOperation> m = new HashMap<Integer, RewriteOperation>();
-		for (int i = 0; i < rewrites.size(); i++) {
-			RewriteOperation op = rewrites.get(i);
-			if ( op==null ) continue; // ignore deleted ops
-			if ( m.get(op.index)!=null ) {
-				throw new Error("should only be one op per index");
-			}
-			m.put(op.index, op);
-		}
-		//System.out.println("index to op: "+m);
-		return m;
-	}
-
-	protected String catOpText(Object a, Object b) {
-		String x = "";
-		String y = "";
-		if ( a!=null ) x = a.toString();
-		if ( b!=null ) y = b.toString();
-		return x+y;
-	}
-
-	protected <T extends RewriteOperation> List<? extends T> getKindOfOps(List<? extends RewriteOperation> rewrites, Class<T> kind) {
-		return getKindOfOps(rewrites, kind, rewrites.size());
-	}
-
-    /** Get all operations before an index of a particular kind */
-    protected <T extends RewriteOperation> List<? extends T> getKindOfOps(List<? extends RewriteOperation> rewrites, Class<T> kind, int before) {
-		List<T> ops = new ArrayList<T>();
-		for (int i=0; i<before && i<rewrites.size(); i++) {
-			RewriteOperation op = rewrites.get(i);
-			if ( op==null ) continue; // ignore deleted
-			if ( kind.isInstance(op) ) {
-				ops.add((T)op);
-			}
-		}
-		return ops;
-	}
-
-	public String toDebugString() {
-		return toDebugString(MIN_TOKEN_INDEX, size()-1);
-	}
-
-	public String toDebugString(int start, int end) {
-		StringBuilder buf = new StringBuilder();
-		for (int i=start; i>=MIN_TOKEN_INDEX && i<=end && i<tokens.size(); i++) {
-			buf.append(get(i));
-		}
-		return buf.toString();
-	}
-}

From 1ade15392b9f076a5d3aea0e87dd91b994133a0a Mon Sep 17 00:00:00 2001
From: Sam Harwell <sam@tunnelvisionlabs.com>
Date: Wed, 4 Apr 2012 15:33:39 -0500
Subject: [PATCH 29/34] Add license header

---
 .../antlr/v4/runtime/TokenStreamRewriter.java | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
index 45c2b4e79..8d391898b 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
@@ -1,3 +1,31 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2012 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+    derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
 package org.antlr.v4.runtime;
 
 import org.antlr.v4.runtime.misc.Interval;

From 6a0081d12e77d14eaea81b314909f72c5e511f14 Mon Sep 17 00:00:00 2001
From: Terence Parr <parrt@antlr.org>
Date: Fri, 6 Apr 2012 20:21:11 -0700
Subject: [PATCH 30/34] rm gunit for now

---
 build.xml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/build.xml b/build.xml
index 956cd4c53..dc45098fa 100644
--- a/build.xml
+++ b/build.xml
@@ -87,6 +87,7 @@ build.sysclasspath=ignore
 			</classpath>
 		</java>
 
+		<!--
 		<echo>gunit grammars</echo>
 		<java classname="org.antlr.Tool" fork="true" failonerror="false" maxmemory="300m"
 			dir="${basedir}/gunit/src/org/antlr/v4/gunit">
@@ -100,6 +101,7 @@ build.sysclasspath=ignore
 				<pathelement path="${java.class.path}"/>
 			</classpath>
 		</java>
+		-->
 	</target>
 
     <target name="compile" depends="antlr" description="Compile for generic OS">
@@ -108,7 +110,7 @@ build.sysclasspath=ignore
 		<copy todir="${build.dir}/src" >
 			<fileset dir="${basedir}/tool/src/"/>
 			<fileset dir="${basedir}/runtime/Java/src/"/>
-			<fileset dir="${basedir}/gunit/src/"/>
+			<!-- <fileset dir="${basedir}/gunit/src/"/> -->
 		</copy>
 		<replace dir="${build.dir}/src" token="@version@" value="${version}"/>
         <javac
@@ -161,12 +163,14 @@ build.sysclasspath=ignore
 				<include name="**/*.st"/>
 				<include name="**/*.stg"/>
 			</fileset>
+			<!--
 			<fileset dir="${basedir}/gunit/src/">
 				<include name="**/*.java"/>
 				<include name="**/*.g"/>
 				<include name="**/*.st"/>
 				<include name="**/*.stg"/>
 			</fileset>
+			-->
         </copy>
 
         <copy todir="${install.root.dir}">

From 35aeeb7c5c08498c8f201e175c3848d4fadfb22a Mon Sep 17 00:00:00 2001
From: Terence Parr <parrt@antlr.org>
Date: Sat, 7 Apr 2012 11:27:18 -0700
Subject: [PATCH 31/34] turn on building parse trees by default since we do
 -listener by default.

---
 runtime/Java/src/org/antlr/v4/runtime/Parser.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/Parser.java b/runtime/Java/src/org/antlr/v4/runtime/Parser.java
index 839ab62c3..7b0f200cb 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/Parser.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/Parser.java
@@ -92,7 +92,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
 	 */
 	protected ParserRuleContext<Token> _ctx;
 
-	protected boolean _buildParseTrees;
+	protected boolean _buildParseTrees = true;
 
 	protected TraceListener _tracer;
 

From f4abc84054fec83e69a133681e49cfd434a35a3c Mon Sep 17 00:00:00 2001
From: Terence Parr <parrt@antlr.org>
Date: Tue, 10 Apr 2012 10:51:51 -0700
Subject: [PATCH 32/34] MutableInt's compare method was messed up.

---
 tool/src/org/antlr/v4/misc/MutableInt.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tool/src/org/antlr/v4/misc/MutableInt.java b/tool/src/org/antlr/v4/misc/MutableInt.java
index 9f6fb9409..5a07b2504 100644
--- a/tool/src/org/antlr/v4/misc/MutableInt.java
+++ b/tool/src/org/antlr/v4/misc/MutableInt.java
@@ -4,7 +4,7 @@ public class MutableInt extends Number implements Comparable<Number> {
 	public int v;
 
 	public MutableInt(int v) { this.v = v; }
-	@Override public int compareTo(Number o) { return v; }
+	@Override public int compareTo(Number o) { return v-o.intValue(); }
 	@Override public int intValue() { return v; }
 	@Override public long longValue() { return v; }
 	@Override public float floatValue() { return v; }

From 30f61b17e3d988eb957163ba0ec2a0c76c1f86e0 Mon Sep 17 00:00:00 2001
From: Terence Parr <parrt@antlr.org>
Date: Sun, 15 Apr 2012 18:30:28 -0700
Subject: [PATCH 33/34] comment tweak

---
 .../Java/src/org/antlr/v4/runtime/NoViableAltException.java | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java b/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java
index 661ee961c..a43784ef8 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java
@@ -30,8 +30,10 @@ package org.antlr.v4.runtime;
 
 import org.antlr.v4.runtime.atn.ATNConfigSet;
 
-/** The parser could not decide which path in the decision to take based
- *  upon the remaining input.
+/** Indicates that the parser could not decide which of two or more paths
+ *  to take based upon the remaining input. It tracks the starting token
+ *  of the offending input and also knows where the parser was
+ *  in the various paths when the error. Reported by reportNoViableAlternative()
  */
 public class NoViableAltException extends RecognitionException {
 	/** Which configurations did we try at input.index() that couldn't match input.LT(1)? */

From 6314b7d31bf48ddb199c59958de90db136979049 Mon Sep 17 00:00:00 2001
From: Terence Parr <parrt@antlr.org>
Date: Thu, 26 Apr 2012 11:59:57 -0700
Subject: [PATCH 34/34] -> becomes # for alt labels

---
 tool/MIGRATION.txt                            | 16 +++++++---
 tool/playground/T.g                           | 31 ++++++++++++++-----
 .../analysis/LeftRecursiveRuleAnalyzer.java   |  2 +-
 tool/src/org/antlr/v4/parse/ANTLRParser.g     |  4 +--
 .../org/antlr/v4/test/TestLeftRecursion.java  | 30 +++++++++---------
 .../test/org/antlr/v4/test/TestListeners.java |  4 +--
 6 files changed, 55 insertions(+), 32 deletions(-)

diff --git a/tool/MIGRATION.txt b/tool/MIGRATION.txt
index c292b8ee7..c4aacd8fe 100644
--- a/tool/MIGRATION.txt
+++ b/tool/MIGRATION.txt
@@ -1,17 +1,23 @@
+Parsers
+
+* Full context LL(*) not SLL(*)
+* Adaptive, takes all but indirect left-recursion
+
 Actions/scopes
 
 * no global scopes. no scope[n].
 
 Trees
 
+* no ASTs
+* no tree grammars
+* parse trees created by default
 * moved methods to Trees
-* Tree->AST
-* added parse trees
-
- no -> and other tree construct in tree grammars
 
 Lexers
 
 * Added [Abc] notation
 
-* unicode rule/token names
\ No newline at end of file
+* unicode rule/token names
+
+* -> skip notation
\ No newline at end of file
diff --git a/tool/playground/T.g b/tool/playground/T.g
index 9949f8074..8d9dd33bb 100644
--- a/tool/playground/T.g
+++ b/tool/playground/T.g
@@ -1,8 +1,25 @@
 grammar T;
-s : e ';' ;
-e : e '*' e
-  | ID
-  | INT
-  ;
-INT : '0'..'9'+;
-WS : (' '|'\n') {skip();} ;
+@members {
+  public static class LeafListener extends TBaseListener {
+    public void exitCall(TParser.CallContext ctx) {
+      System.out.printf("%s %s",ctx.e().start.getText(),
+                ctx.eList());
+    }
+    public void exitInt(TParser.IntContext ctx) {
+      System.out.println(ctx.INT().getSymbol().getText());
+    }
+  }
+}
+s
+@init {setBuildParseTree(true);}
+@after {  System.out.println($r.ctx.toStringTree(this));  ParseTreeWalker walker = new ParseTreeWalker();
+  walker.walk(new LeafListener(), $r.ctx);}
+  : r=e ;
+e : e '(' eList ')' # Call
+  | INT             # Int
+  ;     
+eList : e (',' e)* ;
+MULT: '*' ;
+ADD : '+' ;
+INT : [0-9]+ ;
+WS : [ \t\n]+ -> skip ;
diff --git a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java
index bd1ad31aa..782d93886 100644
--- a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java
+++ b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java
@@ -359,7 +359,7 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
 		TokenStream tokens = input.getTokenStream();
 		// find =>
 		for (int i=stop; i>=start; i--) {
-			if ( tokens.get(i).getType()==RARROW ) {
+			if ( tokens.get(i).getType()==POUND ) {
 				altAST.setTokenStopIndex(i-1);
 				return;
 			}
diff --git a/tool/src/org/antlr/v4/parse/ANTLRParser.g b/tool/src/org/antlr/v4/parse/ANTLRParser.g
index 540bfbe47..12d5c0ef5 100644
--- a/tool/src/org/antlr/v4/parse/ANTLRParser.g
+++ b/tool/src/org/antlr/v4/parse/ANTLRParser.g
@@ -535,7 +535,7 @@ ruleAltList
 
 labeledAlt
 	:	alternative
-		(	RARROW! id! {((AltAST)$alternative.tree).altLabel=$id.tree;}
+		(	POUND! id! {((AltAST)$alternative.tree).altLabel=$id.tree;}
 		)?
 	;
 
@@ -655,7 +655,7 @@ lexerCommandName
         :       id
         |       MODE    ->ID[$MODE]
         ;
-        
+
 altList
     :	alternative (OR alternative)* -> alternative+
     ;
diff --git a/tool/test/org/antlr/v4/test/TestLeftRecursion.java b/tool/test/org/antlr/v4/test/TestLeftRecursion.java
index 2eb813f66..5fe3503fb 100644
--- a/tool/test/org/antlr/v4/test/TestLeftRecursion.java
+++ b/tool/test/org/antlr/v4/test/TestLeftRecursion.java
@@ -252,13 +252,13 @@ public class TestLeftRecursion extends BaseTest {
 			"s : q=e {System.out.println($e.v);} ;\n" +
 			"\n" +
 			"e returns [int v]\n" +
-			"  : a=e op='*' b=e {$v = $a.v * $b.v;}  -> mult\n" +
-			"  | a=e '+' b=e {$v = $a.v + $b.v;}     -> add\n" +
-			"  | INT         {$v = $INT.int;}        -> anInt\n" +
-			"  | '(' x=e ')' {$v = $x.v;}            -> parens\n" +
-			"  | x=e '++'    {$v = $x.v+1;}          -> inc\n" +
-			"  | e '--'                              -> dec\n" +
-			"  | ID          {$v = 3;}               -> anID\n" +
+			"  : a=e op='*' b=e {$v = $a.v * $b.v;}  # mult\n" +
+			"  | a=e '+' b=e {$v = $a.v + $b.v;}     # add\n" +
+			"  | INT         {$v = $INT.int;}        # anInt\n" +
+			"  | '(' x=e ')' {$v = $x.v;}            # parens\n" +
+			"  | x=e '++'    {$v = $x.v+1;}          # inc\n" +
+			"  | e '--'                              # dec\n" +
+			"  | ID          {$v = 3;}               # anID\n" +
 			"  ; \n" +
 			"\n" +
 			"ID : 'a'..'z'+ ;\n" +
@@ -304,18 +304,18 @@ public class TestLeftRecursion extends BaseTest {
 			"// START:stat\n" +
 			"prog:   stat ;\n" +
 			"\n" +
-			"stat:   expr NEWLINE                -> printExpr\n" +
-			"    |   ID '=' expr NEWLINE         -> assign\n" +
-			"    |   NEWLINE                     -> blank\n" +
+			"stat:   expr NEWLINE                # printExpr\n" +
+			"    |   ID '=' expr NEWLINE         # assign\n" +
+			"    |   NEWLINE                     # blank\n" +
 			"    ;\n" +
 			"// END:stat\n" +
 			"\n" +
 			"// START:expr\n" +
-			"expr:   expr ('*'|'/') expr      -> MulDiv\n" +
-			"    |   expr ('+'|'-') expr      -> AddSub\n" +
-			"    |   INT                      -> int\n" +
-			"    |   ID                       -> id\n" +
-			"    |   '(' expr ')'             -> parens\n" +
+			"expr:   expr ('*'|'/') expr      # MulDiv\n" +
+			"    |   expr ('+'|'-') expr      # AddSub\n" +
+			"    |   INT                      # int\n" +
+			"    |   ID                       # id\n" +
+			"    |   '(' expr ')'             # parens\n" +
 			"    ;\n" +
 			"// END:expr\n" +
 			"\n" +
diff --git a/tool/test/org/antlr/v4/test/TestListeners.java b/tool/test/org/antlr/v4/test/TestListeners.java
index 0d05a00ca..a683d4aed 100644
--- a/tool/test/org/antlr/v4/test/TestListeners.java
+++ b/tool/test/org/antlr/v4/test/TestListeners.java
@@ -178,8 +178,8 @@ public class TestListeners extends BaseTest {
 			"  walker.walk(new LeafListener(), $r.ctx);" +
 			"}\n" +
 			"  : r=e ;\n" +
-			"e : e '(' eList ')' -> Call\n" +
-			"  | INT             -> Int\n" +
+			"e : e '(' eList ')' # Call\n" +
+			"  | INT             # Int\n" +
 			"  ;     \n" +
 			"eList : e (',' e)* ;\n" +
 			"MULT: '*' ;\n" +