forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Merge pull request #718 from petrbel/master 

TokenStreamRewriter indentation fix
This commit is contained in:
Sam Harwell 2014-09-27 10:00:33 -05:00
parent f143495ab2 2883ee2897
commit 61e23a2032
2 changed files with 48 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contributors.txt
View File

@ -58,4 +58,4 @@ YYYY/MM/DD, github id, Full name, email
2014/03/18, aphyr, Kyle Kingsbury, aphyr@aphyr.com 2014/03/18, aphyr, Kyle Kingsbury, aphyr@aphyr.com
2014/06/07, ericvergnaud, Eric Vergnaud, eric.vergnaud@wanadoo.fr 2014/06/07, ericvergnaud, Eric Vergnaud, eric.vergnaud@wanadoo.fr
2014/07/04, jimidle, Jim Idle, jimi@Idle.ws 2014/07/04, jimidle, Jim Idle, jimi@Idle.ws
2014/09/27, petrbel, Petr Bělohlávek, antlr@petrbel.cz

97
runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java
View File

@ -115,16 +115,16 @@ import java.util.Map;
*/ */
public class TokenStreamRewriter { public class TokenStreamRewriter {
public static final String DEFAULT_PROGRAM_NAME = "default"; public static final String DEFAULT_PROGRAM_NAME = "default";
public static final int PROGRAM_INIT_SIZE = 100; public static final int PROGRAM_INIT_SIZE = 100;
public static final int MIN_TOKEN_INDEX = 0; public static final int MIN_TOKEN_INDEX = 0;
// Define the rewrite operation hierarchy // Define the rewrite operation hierarchy
public class RewriteOperation { public class RewriteOperation {
/** What index into rewrites List are we? */ /** What index into rewrites List are we? */
protected int instructionIndex; protected int instructionIndex;
/** Token buffer index. */ /** Token buffer index. */
protected int index; protected int index;
protected Object text; protected Object text;
protected RewriteOperation(int index) { protected RewriteOperation(int index) {
@ -148,7 +148,7 @@ public class TokenStreamRewriter {
int $index = opName.indexOf('$'); int $index = opName.indexOf('$');
opName = opName.substring($index+1, opName.length()); opName = opName.substring($index+1, opName.length());
return "<"+opName+"@"+tokens.get(index)+ return "<"+opName+"@"+tokens.get(index)+
":\""+text+"\">"; ":\""+text+"\">";
} }
} }
@ -187,10 +187,10 @@ public class TokenStreamRewriter {
public String toString() { public String toString() {
if ( text==null ) { if ( text==null ) {
return "<DeleteOp@"+tokens.get(index)+ return "<DeleteOp@"+tokens.get(index)+
".."+tokens.get(lastIndex)+">"; ".."+tokens.get(lastIndex)+">";
} }
return "<ReplaceOp@"+tokens.get(index)+ return "<ReplaceOp@"+tokens.get(index)+
".."+tokens.get(lastIndex)+":\""+text+"\">"; ".."+tokens.get(lastIndex)+":\""+text+"\">";
} }
} }
@ -224,7 +224,7 @@ public class TokenStreamRewriter {
/** Rollback the instruction stream for a program so that /** Rollback the instruction stream for a program so that
* the indicated instruction (via instructionIndex) is no * the indicated instruction (via instructionIndex) is no
* longer in the stream. UNTESTED! * longer in the stream. UNTESTED!
*/ */
public void rollback(String programName, int instructionIndex) { public void rollback(String programName, int instructionIndex) {
List<RewriteOperation> is = programs.get(programName); List<RewriteOperation> is = programs.get(programName);
@ -274,8 +274,8 @@ public class TokenStreamRewriter {
public void insertBefore(String programName, int index, Object text) { public void insertBefore(String programName, int index, Object text) {
RewriteOperation op = new InsertBeforeOp(index,text); RewriteOperation op = new InsertBeforeOp(index,text);
List<RewriteOperation> rewrites = getProgram(programName); List<RewriteOperation> rewrites = getProgram(programName);
op.instructionIndex = rewrites.size(); op.instructionIndex = rewrites.size();
rewrites.add(op); rewrites.add(op);
} }
public void replace(int index, Object text) { public void replace(int index, Object text) {
@ -300,8 +300,8 @@ public class TokenStreamRewriter {
} }
RewriteOperation op = new ReplaceOp(from, to, text); RewriteOperation op = new ReplaceOp(from, to, text);
List<RewriteOperation> rewrites = getProgram(programName); List<RewriteOperation> rewrites = getProgram(programName);
op.instructionIndex = rewrites.size(); op.instructionIndex = rewrites.size();
rewrites.add(op); rewrites.add(op);
} }
public void replace(String programName, Token from, Token to, @Nullable Object text) { public void replace(String programName, Token from, Token to, @Nullable Object text) {
@ -390,11 +390,11 @@ public class TokenStreamRewriter {
int start = interval.a; int start = interval.a;
int stop = interval.b; int stop = interval.b;
// ensure start/end are in range // ensure start/end are in range
if ( stop>tokens.size()-1 ) stop = tokens.size()-1; if ( stop>tokens.size()-1 ) stop = tokens.size()-1;
if ( start<0 ) start = 0; if ( start<0 ) start = 0;
if ( rewrites==null || rewrites.isEmpty() ) { if ( rewrites==null || rewrites.isEmpty() ) {
return tokens.getText(interval); // no instructions to execute return tokens.getText(interval); // no instructions to execute
} }
StringBuilder buf = new StringBuilder(); StringBuilder buf = new StringBuilder();
@ -402,9 +402,9 @@ public class TokenStreamRewriter {
// First, optimize instruction stream // First, optimize instruction stream
Map<Integer, RewriteOperation> indexToOp = reduceToSingleOperationPerIndex(rewrites); Map<Integer, RewriteOperation> indexToOp = reduceToSingleOperationPerIndex(rewrites);
// Walk buffer, executing instructions and emitting tokens // Walk buffer, executing instructions and emitting tokens
int i = start; int i = start;
while ( i <= stop && i < tokens.size() ) { while ( i <= stop && i < tokens.size() ) {
RewriteOperation op = indexToOp.get(i); RewriteOperation op = indexToOp.get(i);
indexToOp.remove(i); // remove so any left have index size-1 indexToOp.remove(i); // remove so any left have index size-1
Token t = tokens.get(i); Token t = tokens.get(i);
@ -418,22 +418,22 @@ public class TokenStreamRewriter {
} }
} }
// include stuff after end if it's last index in buffer // include stuff after end if it's last index in buffer
// So, if they did an insertAfter(lastValidIndex, "foo"), include // So, if they did an insertAfter(lastValidIndex, "foo"), include
// foo if end==lastValidIndex. // foo if end==lastValidIndex.
if ( stop==tokens.size()-1 ) { if ( stop==tokens.size()-1 ) {
// Scan any remaining operations after last token // Scan any remaining operations after last token
// should be included (they will be inserts). // should be included (they will be inserts).
for (RewriteOperation op : indexToOp.values()) { for (RewriteOperation op : indexToOp.values()) {
if ( op.index >= tokens.size()-1 ) buf.append(op.text); if ( op.index >= tokens.size()-1 ) buf.append(op.text);
} }
} }
return buf.toString(); return buf.toString();
} }
/** We need to combine operations and report invalid operations (like /** We need to combine operations and report invalid operations (like
* overlapping replaces that are not completed nested). Inserts to * overlapping replaces that are not completed nested). Inserts to
* same index need to be combined etc... Here are the cases: * same index need to be combined etc... Here are the cases:
* *
* I.i.u I.j.v leave alone, nonoverlapping * I.i.u I.j.v leave alone, nonoverlapping
* I.i.u I.i.v combine: Iivu * I.i.u I.i.v combine: Iivu
@ -456,25 +456,25 @@ public class TokenStreamRewriter {
* I.i.u = insert u before op @ index i * I.i.u = insert u before op @ index i
* R.x-y.u = replace x-y indexed tokens with u * R.x-y.u = replace x-y indexed tokens with u
* *
* First we need to examine replaces. For any replace op: * First we need to examine replaces. For any replace op:
* *
* 1. wipe out any insertions before op within that range. * 1. wipe out any insertions before op within that range.
* 2. Drop any replace op before that is contained completely within * 2. Drop any replace op before that is contained completely within
* that range. * that range.
* 3. Throw exception upon boundary overlap with any previous replace. * 3. Throw exception upon boundary overlap with any previous replace.
* *
* Then we can deal with inserts: * Then we can deal with inserts:
* *
* 1. for any inserts to same index, combine even if not adjacent. * 1. for any inserts to same index, combine even if not adjacent.
* 2. for any prior replace with same left boundary, combine this * 2. for any prior replace with same left boundary, combine this
* insert with replace and delete this replace. * insert with replace and delete this replace.
* 3. throw exception if index in same range as previous replace * 3. throw exception if index in same range as previous replace
* *
* Don't actually delete; make op null in list. Easier to walk list. * Don't actually delete; make op null in list. Easier to walk list.
* Later we can throw as we add to index &rarr; op map. * Later we can throw as we add to index &rarr; op map.
* *
* Note that I.2 R.2-2 will wipe out I.2 even though, technically, the * Note that I.2 R.2-2 will wipe out I.2 even though, technically, the
* inserted stuff would be before the replace range. But, if you * inserted stuff would be before the replace range. But, if you
* add tokens in front of a method body '{' and then delete the method * add tokens in front of a method body '{' and then delete the method
* body, I think the stuff before the '{' you added should disappear too. * body, I think the stuff before the '{' you added should disappear too.
* *
@ -499,16 +499,16 @@ public class TokenStreamRewriter {
rop.text = iop.text.toString() + (rop.text!=null?rop.text.toString():""); rop.text = iop.text.toString() + (rop.text!=null?rop.text.toString():"");
} }
else if ( iop.index > rop.index && iop.index <= rop.lastIndex ) { else if ( iop.index > rop.index && iop.index <= rop.lastIndex ) {
// delete insert as it's a no-op. // delete insert as it's a no-op.
rewrites.set(iop.instructionIndex, null); rewrites.set(iop.instructionIndex, null);
} }
} }
// Drop any prior replaces contained within // Drop any prior replaces contained within
List<? extends ReplaceOp> prevReplaces = getKindOfOps(rewrites, ReplaceOp.class, i); List<? extends ReplaceOp> prevReplaces = getKindOfOps(rewrites, ReplaceOp.class, i);
for (ReplaceOp prevRop : prevReplaces) { for (ReplaceOp prevRop : prevReplaces) {
if ( prevRop.index>=rop.index && prevRop.lastIndex <= rop.lastIndex ) { if ( prevRop.index>=rop.index && prevRop.lastIndex <= rop.lastIndex ) {
// delete replace as it's a no-op. // delete replace as it's a no-op.
rewrites.set(prevRop.instructionIndex, null); rewrites.set(prevRop.instructionIndex, null);
continue; continue;
} }
// throw exception unless disjoint or identical // throw exception unless disjoint or identical
@ -526,8 +526,7 @@ public class TokenStreamRewriter {
System.out.println("new rop "+rop); System.out.println("new rop "+rop);
} }
else if ( !disjoint && !same ) { else if ( !disjoint && !same ) {
throw new IllegalArgumentException("replace op boundaries of "+rop+ throw new IllegalArgumentException("replace op boundaries of "+rop+" overlap with previous "+prevRop);
" overlap with previous "+prevRop);
} }
} }
} }
@ -545,8 +544,8 @@ public class TokenStreamRewriter {
// convert to strings...we're in process of toString'ing // convert to strings...we're in process of toString'ing
// whole token buffer so no lazy eval issue with any templates // whole token buffer so no lazy eval issue with any templates
iop.text = catOpText(iop.text,prevIop.text); iop.text = catOpText(iop.text,prevIop.text);
// delete redundant prior insert // delete redundant prior insert
rewrites.set(prevIop.instructionIndex, null); rewrites.set(prevIop.instructionIndex, null);
} }
} }
// look for replaces where iop.index is in range; error // look for replaces where iop.index is in range; error
@ -554,12 +553,11 @@ public class TokenStreamRewriter {
for (ReplaceOp rop : prevReplaces) { for (ReplaceOp rop : prevReplaces) {
if ( iop.index == rop.index ) { if ( iop.index == rop.index ) {
rop.text = catOpText(iop.text,rop.text); rop.text = catOpText(iop.text,rop.text);
rewrites.set(i, null); // delete current insert rewrites.set(i, null); // delete current insert
continue; continue;
} }
if ( iop.index >= rop.index && iop.index <= rop.lastIndex ) { if ( iop.index >= rop.index && iop.index <= rop.lastIndex ) {
throw new IllegalArgumentException("insert op "+iop+ throw new IllegalArgumentException("insert op "+iop+" within boundaries of previous "+rop);
" within boundaries of previous "+rop);
} }
} }
} }
@ -585,8 +583,8 @@ public class TokenStreamRewriter {
return x+y; return x+y;
} }
/** Get all operations before an index of a particular kind */ /** Get all operations before an index of a particular kind */
protected <T extends RewriteOperation> List<? extends T> getKindOfOps(List<? extends RewriteOperation> rewrites, Class<T> kind, int before) { protected <T extends RewriteOperation> List<? extends T> getKindOfOps(List<? extends RewriteOperation> rewrites, Class<T> kind, int before) {
List<T> ops = new ArrayList<T>(); List<T> ops = new ArrayList<T>();
for (int i=0; i<before && i<rewrites.size(); i++) { for (int i=0; i<before && i<rewrites.size(); i++) {
RewriteOperation op = rewrites.get(i); RewriteOperation op = rewrites.get(i);
@ -597,5 +595,4 @@ public class TokenStreamRewriter {
} }
return ops; return ops;
} }
} }