forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Merge pull request #1445 from parrt/fixes-550-java 

Fixes #550 by @lygav, which tweaks and more tests by me.
This commit is contained in:
Terence Parr 2016-12-03 10:25:55 -08:00 committed by GitHub
parent 53b6153060 c959bf649b
commit 30dddbb84d
2 changed files with 81 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
View File

@ -120,10 +120,10 @@ public class TokenStreamRewriter {
// Define the rewrite operation hierarchy // Define the rewrite operation hierarchy
public class RewriteOperation { public class RewriteOperation {
/** What index into rewrites List are we? */ /** What index into rewrites List are we? */
protected int instructionIndex; protected int instructionIndex;
/** Token buffer index. */ /** Token buffer index. */
protected int index; protected int index;
protected Object text; protected Object text;
protected RewriteOperation(int index) { protected RewriteOperation(int index) {
@ -166,6 +166,16 @@ public class TokenStreamRewriter {
} }
} }
/** Distinguish between insert after/before to do the "insert afters"
* first and then the "insert befores" at same index. Implementation
* of "insert after" is "insert before index+1".
*/
class InsertAfterOp extends InsertBeforeOp {
public InsertAfterOp(int index, Object text) {
super(index+1, text); // insert after is insert before index+1
}
}
/** I'm going to try replacing range from x..y with (y-x)+1 ReplaceOp /** I'm going to try replacing range from x..y with (y-x)+1 ReplaceOp
* instructions. * instructions.
*/ */
@ -255,7 +265,10 @@ public class TokenStreamRewriter {
public void insertAfter(String programName, int index, Object text) { public void insertAfter(String programName, int index, Object text) {
// to insert after, just insert before next index (even if past end) // to insert after, just insert before next index (even if past end)
insertBefore(programName,index+1, text); RewriteOperation op = new InsertAfterOp(index, text);
List<RewriteOperation> rewrites = getProgram(programName);
op.instructionIndex = rewrites.size();
rewrites.add(op);
} }
public void insertBefore(Token t, Object text) { public void insertBefore(Token t, Object text) {
@ -520,8 +533,6 @@ public class TokenStreamRewriter {
// throw exception unless disjoint or identical // throw exception unless disjoint or identical
boolean disjoint = boolean disjoint =
prevRop.lastIndex<rop.index || prevRop.index > rop.lastIndex; prevRop.lastIndex<rop.index || prevRop.index > rop.lastIndex;
boolean same =
prevRop.index==rop.index && prevRop.lastIndex==rop.lastIndex;
// Delete special case of replace (text==null): // Delete special case of replace (text==null):
// D.i-j.u D.x-y.v | boundaries overlap combine to max(min)..max(right) // D.i-j.u D.x-y.v | boundaries overlap combine to max(min)..max(right)
if ( prevRop.text==null && rop.text==null && !disjoint ) { if ( prevRop.text==null && rop.text==null && !disjoint ) {
@ -531,7 +542,7 @@ public class TokenStreamRewriter {
rop.lastIndex = Math.max(prevRop.lastIndex, rop.lastIndex); rop.lastIndex = Math.max(prevRop.lastIndex, rop.lastIndex);
System.out.println("new rop "+rop); System.out.println("new rop "+rop);
} }
else if ( !disjoint && !same ) { else if ( !disjoint ) {
throw new IllegalArgumentException("replace op boundaries of "+rop+" overlap with previous "+prevRop); throw new IllegalArgumentException("replace op boundaries of "+rop+" overlap with previous "+prevRop);
} }
} }
@ -546,12 +557,18 @@ public class TokenStreamRewriter {
// combine current insert with prior if any at same index // combine current insert with prior if any at same index
List<? extends InsertBeforeOp> prevInserts = getKindOfOps(rewrites, InsertBeforeOp.class, i); List<? extends InsertBeforeOp> prevInserts = getKindOfOps(rewrites, InsertBeforeOp.class, i);
for (InsertBeforeOp prevIop : prevInserts) { for (InsertBeforeOp prevIop : prevInserts) {
if ( prevIop.index == iop.index ) { // combine objects if ( prevIop.index==iop.index ) {
// convert to strings...we're in process of toString'ing if ( InsertAfterOp.class.isInstance(prevIop) ) {
// whole token buffer so no lazy eval issue with any templates iop.text = catOpText(prevIop.text, iop.text);
iop.text = catOpText(iop.text,prevIop.text); rewrites.set(prevIop.instructionIndex, null);
// delete redundant prior insert }
rewrites.set(prevIop.instructionIndex, null); else if ( InsertBeforeOp.class.isInstance(prevIop) ) { // combine objects
// convert to strings...we're in process of toString'ing
// whole token buffer so no lazy eval issue with any templates
iop.text = catOpText(iop.text, prevIop.text);
// delete redundant prior insert
rewrites.set(prevIop.instructionIndex, null);
}
} }
} }
// look for replaces where iop.index is in range; error // look for replaces where iop.index is in range; error

54
tool-testsuite/test/org/antlr/v4/test/tool/TestTokenStreamRewriter.java
View File

@ -36,7 +36,6 @@ import org.antlr.v4.runtime.TokenStreamRewriter;
import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.tool.LexerGrammar; import org.antlr.v4.tool.LexerGrammar;
import org.junit.Before; import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
@ -889,10 +888,9 @@ public class TestTokenStreamRewriter extends BaseJavaToolTest {
assertEquals(expecting, result); assertEquals(expecting, result);
} }
// Test for https://github.com/antlr/antlr4/issues/550 // Test Fix for https://github.com/antlr/antlr4/issues/550
@Test @Test
@Ignore public void testDistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder() throws Exception {
public void testPreservesOrderOfContiguousInserts() throws Exception {
LexerGrammar g = new LexerGrammar( LexerGrammar g = new LexerGrammar(
"lexer grammar T;\n"+ "lexer grammar T;\n"+
"A : 'a';\n" + "A : 'a';\n" +
@ -912,4 +910,52 @@ public class TestTokenStreamRewriter extends BaseJavaToolTest {
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@Test
public void testDistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder2() throws Exception {
LexerGrammar g = new LexerGrammar(
"lexer grammar T;\n"+
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
String input = "aa";
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
tokens.insertBefore(0, "<p>");
tokens.insertBefore(0, "<b>");
tokens.insertAfter(0, "</p>");
tokens.insertAfter(0, "</b>");
tokens.insertBefore(1, "<b>");
tokens.insertAfter(1, "</b>");
String result = tokens.getText();
String expecting = "<b><p>a</p></b><b>a</b>";
assertEquals(expecting, result);
}
// Test Fix for https://github.com/antlr/antlr4/issues/550
@Test
public void testPreservesOrderOfContiguousInserts() throws Exception {
LexerGrammar g = new LexerGrammar(
"lexer grammar T;\n"+
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
String input = "ab";
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
tokens.insertBefore(0, "<p>");
tokens.insertBefore(0, "<b>");
tokens.insertBefore(0, "<div>");
tokens.insertAfter(0, "</p>");
tokens.insertAfter(0, "</b>");
tokens.insertAfter(0, "</div>");
tokens.insertBefore(1, "!");
String result = tokens.getText();
String expecting = "<div><b><p>a</p></b></div>!b";
assertEquals(expecting, result);
}
} }