From c959bf649b65d931ebe5ecd76a372eb3067ef52d Mon Sep 17 00:00:00 2001 From: parrt Date: Sat, 3 Dec 2016 10:25:02 -0800 Subject: [PATCH] Fixes #550 by @lygav, which tweaks and more tests by me. --- .../antlr/v4/runtime/TokenStreamRewriter.java | 45 +++++++++++----- .../v4/test/tool/TestTokenStreamRewriter.java | 54 +++++++++++++++++-- 2 files changed, 81 insertions(+), 18 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java index c9b74c18e..a741d0a8d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java +++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java @@ -120,10 +120,10 @@ public class TokenStreamRewriter { // Define the rewrite operation hierarchy public class RewriteOperation { - /** What index into rewrites List are we? */ - protected int instructionIndex; - /** Token buffer index. */ - protected int index; + /** What index into rewrites List are we? */ + protected int instructionIndex; + /** Token buffer index. */ + protected int index; protected Object text; protected RewriteOperation(int index) { @@ -166,6 +166,16 @@ public class TokenStreamRewriter { } } + /** Distinguish between insert after/before to do the "insert afters" + * first and then the "insert befores" at same index. Implementation + * of "insert after" is "insert before index+1". + */ + class InsertAfterOp extends InsertBeforeOp { + public InsertAfterOp(int index, Object text) { + super(index+1, text); // insert after is insert before index+1 + } + } + /** I'm going to try replacing range from x..y with (y-x)+1 ReplaceOp * instructions. */ @@ -255,7 +265,10 @@ public class TokenStreamRewriter { public void insertAfter(String programName, int index, Object text) { // to insert after, just insert before next index (even if past end) - insertBefore(programName,index+1, text); + RewriteOperation op = new InsertAfterOp(index, text); + List rewrites = getProgram(programName); + op.instructionIndex = rewrites.size(); + rewrites.add(op); } public void insertBefore(Token t, Object text) { @@ -520,8 +533,6 @@ public class TokenStreamRewriter { // throw exception unless disjoint or identical boolean disjoint = prevRop.lastIndex rop.lastIndex; - boolean same = - prevRop.index==rop.index && prevRop.lastIndex==rop.lastIndex; // Delete special case of replace (text==null): // D.i-j.u D.x-y.v | boundaries overlap combine to max(min)..max(right) if ( prevRop.text==null && rop.text==null && !disjoint ) { @@ -531,7 +542,7 @@ public class TokenStreamRewriter { rop.lastIndex = Math.max(prevRop.lastIndex, rop.lastIndex); System.out.println("new rop "+rop); } - else if ( !disjoint && !same ) { + else if ( !disjoint ) { throw new IllegalArgumentException("replace op boundaries of "+rop+" overlap with previous "+prevRop); } } @@ -546,12 +557,18 @@ public class TokenStreamRewriter { // combine current insert with prior if any at same index List prevInserts = getKindOfOps(rewrites, InsertBeforeOp.class, i); for (InsertBeforeOp prevIop : prevInserts) { - if ( prevIop.index == iop.index ) { // combine objects - // convert to strings...we're in process of toString'ing - // whole token buffer so no lazy eval issue with any templates - iop.text = catOpText(iop.text,prevIop.text); - // delete redundant prior insert - rewrites.set(prevIop.instructionIndex, null); + if ( prevIop.index==iop.index ) { + if ( InsertAfterOp.class.isInstance(prevIop) ) { + iop.text = catOpText(prevIop.text, iop.text); + rewrites.set(prevIop.instructionIndex, null); + } + else if ( InsertBeforeOp.class.isInstance(prevIop) ) { // combine objects + // convert to strings...we're in process of toString'ing + // whole token buffer so no lazy eval issue with any templates + iop.text = catOpText(iop.text, prevIop.text); + // delete redundant prior insert + rewrites.set(prevIop.instructionIndex, null); + } } } // look for replaces where iop.index is in range; error diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestTokenStreamRewriter.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestTokenStreamRewriter.java index cbf90a198..24cef881e 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestTokenStreamRewriter.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestTokenStreamRewriter.java @@ -36,7 +36,6 @@ import org.antlr.v4.runtime.TokenStreamRewriter; import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.tool.LexerGrammar; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -889,10 +888,9 @@ public class TestTokenStreamRewriter extends BaseJavaToolTest { assertEquals(expecting, result); } - // Test for https://github.com/antlr/antlr4/issues/550 + // Test Fix for https://github.com/antlr/antlr4/issues/550 @Test - @Ignore - public void testPreservesOrderOfContiguousInserts() throws Exception { + public void testDistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder() throws Exception { LexerGrammar g = new LexerGrammar( "lexer grammar T;\n"+ "A : 'a';\n" + @@ -912,4 +910,52 @@ public class TestTokenStreamRewriter extends BaseJavaToolTest { assertEquals(expecting, result); } + @Test + public void testDistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder2() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar T;\n"+ + "A : 'a';\n" + + "B : 'b';\n" + + "C : 'c';\n"); + String input = "aa"; + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); + CommonTokenStream stream = new CommonTokenStream(lexEngine); + stream.fill(); + TokenStreamRewriter tokens = new TokenStreamRewriter(stream); + tokens.insertBefore(0, "

"); + tokens.insertBefore(0, ""); + tokens.insertAfter(0, "

"); + tokens.insertAfter(0, ""); + tokens.insertBefore(1, ""); + tokens.insertAfter(1, ""); + String result = tokens.getText(); + String expecting = "

a

a"; + assertEquals(expecting, result); + } + + // Test Fix for https://github.com/antlr/antlr4/issues/550 + @Test + public void testPreservesOrderOfContiguousInserts() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar T;\n"+ + "A : 'a';\n" + + "B : 'b';\n" + + "C : 'c';\n"); + String input = "ab"; + LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input)); + CommonTokenStream stream = new CommonTokenStream(lexEngine); + stream.fill(); + TokenStreamRewriter tokens = new TokenStreamRewriter(stream); + tokens.insertBefore(0, "

"); + tokens.insertBefore(0, ""); + tokens.insertBefore(0, "

"); + tokens.insertAfter(0, "

"); + tokens.insertAfter(0, ""); + tokens.insertAfter(0, "
"); + tokens.insertBefore(1, "!"); + String result = tokens.getText(); + String expecting = "

a

!b"; + assertEquals(expecting, result); + } + }