forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Fixes #550 by @lygav, which tweaks and more tests by me.

This commit is contained in:
parrt 2016-12-03 10:25:02 -08:00
parent 8ead51d8ff
commit c959bf649b
2 changed files with 81 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
View File

@ -120,10 +120,10 @@ public class TokenStreamRewriter {
// Define the rewrite operation hierarchy
public class RewriteOperation {
/** What index into rewrites List are we? */
protected int instructionIndex;
/** Token buffer index. */
protected int index;
/** What index into rewrites List are we? */
protected int instructionIndex;
/** Token buffer index. */
protected int index;
protected Object text;
protected RewriteOperation(int index) {
@ -166,6 +166,16 @@ public class TokenStreamRewriter {
}
}
/** Distinguish between insert after/before to do the "insert afters"
* first and then the "insert befores" at same index. Implementation
* of "insert after" is "insert before index+1".
*/
class InsertAfterOp extends InsertBeforeOp {
public InsertAfterOp(int index, Object text) {
super(index+1, text); // insert after is insert before index+1
}
}
/** I'm going to try replacing range from x..y with (y-x)+1 ReplaceOp
* instructions.
*/
@ -255,7 +265,10 @@ public class TokenStreamRewriter {
public void insertAfter(String programName, int index, Object text) {
// to insert after, just insert before next index (even if past end)
insertBefore(programName,index+1, text);
RewriteOperation op = new InsertAfterOp(index, text);
List<RewriteOperation> rewrites = getProgram(programName);
op.instructionIndex = rewrites.size();
rewrites.add(op);
}
public void insertBefore(Token t, Object text) {
@ -520,8 +533,6 @@ public class TokenStreamRewriter {
// throw exception unless disjoint or identical
boolean disjoint =
prevRop.lastIndex<rop.index || prevRop.index > rop.lastIndex;
boolean same =
prevRop.index==rop.index && prevRop.lastIndex==rop.lastIndex;
// Delete special case of replace (text==null):
// D.i-j.u D.x-y.v | boundaries overlap combine to max(min)..max(right)
if ( prevRop.text==null && rop.text==null && !disjoint ) {
@ -531,7 +542,7 @@ public class TokenStreamRewriter {
rop.lastIndex = Math.max(prevRop.lastIndex, rop.lastIndex);
System.out.println("new rop "+rop);
}
else if ( !disjoint && !same ) {
else if ( !disjoint ) {
throw new IllegalArgumentException("replace op boundaries of "+rop+" overlap with previous "+prevRop);
}
}
@ -546,12 +557,18 @@ public class TokenStreamRewriter {
// combine current insert with prior if any at same index
List<? extends InsertBeforeOp> prevInserts = getKindOfOps(rewrites, InsertBeforeOp.class, i);
for (InsertBeforeOp prevIop : prevInserts) {
if ( prevIop.index == iop.index ) { // combine objects
// convert to strings...we're in process of toString'ing
// whole token buffer so no lazy eval issue with any templates
iop.text = catOpText(iop.text,prevIop.text);
// delete redundant prior insert
rewrites.set(prevIop.instructionIndex, null);
if ( prevIop.index==iop.index ) {
if ( InsertAfterOp.class.isInstance(prevIop) ) {
iop.text = catOpText(prevIop.text, iop.text);
rewrites.set(prevIop.instructionIndex, null);
}
else if ( InsertBeforeOp.class.isInstance(prevIop) ) { // combine objects
// convert to strings...we're in process of toString'ing
// whole token buffer so no lazy eval issue with any templates
iop.text = catOpText(iop.text, prevIop.text);
// delete redundant prior insert
rewrites.set(prevIop.instructionIndex, null);
}
}
}
// look for replaces where iop.index is in range; error

54
tool-testsuite/test/org/antlr/v4/test/tool/TestTokenStreamRewriter.java
View File

@ -36,7 +36,6 @@ import org.antlr.v4.runtime.TokenStreamRewriter;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.tool.LexerGrammar;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
@ -889,10 +888,9 @@ public class TestTokenStreamRewriter extends BaseJavaToolTest {
assertEquals(expecting, result);
}
// Test for https://github.com/antlr/antlr4/issues/550
// Test Fix for https://github.com/antlr/antlr4/issues/550
@Test
@Ignore
public void testPreservesOrderOfContiguousInserts() throws Exception {
public void testDistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder() throws Exception {
LexerGrammar g = new LexerGrammar(
"lexer grammar T;\n"+
"A : 'a';\n" +
@ -912,4 +910,52 @@ public class TestTokenStreamRewriter extends BaseJavaToolTest {
assertEquals(expecting, result);
}
@Test
public void testDistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder2() throws Exception {
LexerGrammar g = new LexerGrammar(
"lexer grammar T;\n"+
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
String input = "aa";
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
tokens.insertBefore(0, "<p>");
tokens.insertBefore(0, "<b>");
tokens.insertAfter(0, "</p>");
tokens.insertAfter(0, "</b>");
tokens.insertBefore(1, "<b>");
tokens.insertAfter(1, "</b>");
String result = tokens.getText();
String expecting = "<b><p>a</p></b><b>a</b>";
assertEquals(expecting, result);
}
// Test Fix for https://github.com/antlr/antlr4/issues/550
@Test
public void testPreservesOrderOfContiguousInserts() throws Exception {
LexerGrammar g = new LexerGrammar(
"lexer grammar T;\n"+
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
String input = "ab";
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
CommonTokenStream stream = new CommonTokenStream(lexEngine);
stream.fill();
TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
tokens.insertBefore(0, "<p>");
tokens.insertBefore(0, "<b>");
tokens.insertBefore(0, "<div>");
tokens.insertAfter(0, "</p>");
tokens.insertAfter(0, "</b>");
tokens.insertAfter(0, "</div>");
tokens.insertBefore(1, "!");
String result = tokens.getText();
String expecting = "<div><b><p>a</p></b></div>!b";
assertEquals(expecting, result);
}
}