Merge pull request #1445 from parrt/fixes-550-java

Fixes #550 by @lygav, which tweaks and more tests by me.

runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java

@@ -166,6 +166,16 @@ public class TokenStreamRewriter {
 		}
 	}
 
+	/** Distinguish between insert after/before to do the "insert afters"
+	 *  first and then the "insert befores" at same index. Implementation
+	 *  of "insert after" is "insert before index+1".
+	 */
+    class InsertAfterOp extends InsertBeforeOp {
+        public InsertAfterOp(int index, Object text) {
+            super(index+1, text); // insert after is insert before index+1
+        }
+    }
+
 	/** I'm going to try replacing range from x..y with (y-x)+1 ReplaceOp
 	 *  instructions.
 	 */
@@ -255,7 +265,10 @@ public class TokenStreamRewriter {
 
 	public void insertAfter(String programName, int index, Object text) {
 		// to insert after, just insert before next index (even if past end)
-		insertBefore(programName,index+1, text);
+        RewriteOperation op = new InsertAfterOp(index, text);
+        List<RewriteOperation> rewrites = getProgram(programName);
+        op.instructionIndex = rewrites.size();
+        rewrites.add(op);
 	}
 
 	public void insertBefore(Token t, Object text) {
@@ -520,8 +533,6 @@ public class TokenStreamRewriter {
 				// throw exception unless disjoint or identical
 				boolean disjoint =
 					prevRop.lastIndex<rop.index || prevRop.index > rop.lastIndex;
-				boolean same =
-					prevRop.index==rop.index && prevRop.lastIndex==rop.lastIndex;
 				// Delete special case of replace (text==null):
 				// D.i-j.u D.x-y.v	| boundaries overlap	combine to max(min)..max(right)
 				if ( prevRop.text==null && rop.text==null && !disjoint ) {
@@ -531,7 +542,7 @@ public class TokenStreamRewriter {
 					rop.lastIndex = Math.max(prevRop.lastIndex, rop.lastIndex);
 					System.out.println("new rop "+rop);
 				}
-				else if ( !disjoint && !same ) {
+				else if ( !disjoint ) {
 					throw new IllegalArgumentException("replace op boundaries of "+rop+" overlap with previous "+prevRop);
 				}
 			}
@@ -546,7 +557,12 @@ public class TokenStreamRewriter {
 			// combine current insert with prior if any at same index
 			List<? extends InsertBeforeOp> prevInserts = getKindOfOps(rewrites, InsertBeforeOp.class, i);
 			for (InsertBeforeOp prevIop : prevInserts) {
-				if ( prevIop.index == iop.index ) { // combine objects
+				if ( prevIop.index==iop.index ) {
+					if ( InsertAfterOp.class.isInstance(prevIop) ) {
+						iop.text = catOpText(prevIop.text, iop.text);
+						rewrites.set(prevIop.instructionIndex, null);
+					}
+					else if ( InsertBeforeOp.class.isInstance(prevIop) ) { // combine objects
 						// convert to strings...we're in process of toString'ing
 						// whole token buffer so no lazy eval issue with any templates
 						iop.text = catOpText(iop.text, prevIop.text);
@@ -554,6 +570,7 @@ public class TokenStreamRewriter {
 						rewrites.set(prevIop.instructionIndex, null);
 					}
 				}
+			}
 			// look for replaces where iop.index is in range; error
 			List<? extends ReplaceOp> prevReplaces = getKindOfOps(rewrites, ReplaceOp.class, i);
 			for (ReplaceOp rop : prevReplaces) {

tool-testsuite/test/org/antlr/v4/test/tool/TestTokenStreamRewriter.java

@@ -36,7 +36,6 @@ import org.antlr.v4.runtime.TokenStreamRewriter;
 import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.tool.LexerGrammar;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
@@ -889,10 +888,9 @@ public class TestTokenStreamRewriter extends BaseJavaToolTest {
 		assertEquals(expecting, result);
 	}
 
-	// Test for https://github.com/antlr/antlr4/issues/550
+	// Test Fix for https://github.com/antlr/antlr4/issues/550
 	@Test
-	@Ignore
+	public void testDistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder() throws Exception {
-	public void testPreservesOrderOfContiguousInserts() throws Exception {
 		LexerGrammar g = new LexerGrammar(
 											 "lexer grammar T;\n"+
 											 "A : 'a';\n" +
@@ -912,4 +910,52 @@ public class TestTokenStreamRewriter extends BaseJavaToolTest {
 		assertEquals(expecting, result);
 	}
 
+	@Test
+	public void testDistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder2() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar T;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "aa";
+		LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(0, "<p>");
+		tokens.insertBefore(0, "<b>");
+		tokens.insertAfter(0, "</p>");
+		tokens.insertAfter(0, "</b>");
+		tokens.insertBefore(1, "<b>");
+		tokens.insertAfter(1, "</b>");
+		String result = tokens.getText();
+		String expecting = "<b><p>a</p></b><b>a</b>";
+		assertEquals(expecting, result);
+	}
+
+	// Test Fix for https://github.com/antlr/antlr4/issues/550
+	@Test
+	public void testPreservesOrderOfContiguousInserts() throws Exception {
+		LexerGrammar g = new LexerGrammar(
+											 "lexer grammar T;\n"+
+											 "A : 'a';\n" +
+											 "B : 'b';\n" +
+											 "C : 'c';\n");
+		String input = "ab";
+		LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
+		CommonTokenStream stream = new CommonTokenStream(lexEngine);
+		stream.fill();
+		TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
+		tokens.insertBefore(0, "<p>");
+		tokens.insertBefore(0, "<b>");
+		tokens.insertBefore(0, "<div>");
+		tokens.insertAfter(0, "</p>");
+		tokens.insertAfter(0, "</b>");
+		tokens.insertAfter(0, "</div>");
+		tokens.insertBefore(1, "!");
+		String result = tokens.getText();
+		String expecting = "<div><b><p>a</p></b></div>!b";
+		assertEquals(expecting, result);
+	}
+
 }