forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

fix bug in get hidden channel stuff. rewrote. update test.

This commit is contained in:
Terence Parr 2012-07-07 18:48:48 -07:00
parent ec47251bb2
commit 3b2c0a6177
2 changed files with 73 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java
View File

@ -239,13 +239,16 @@ public class BufferedTokenStream<T extends Token> implements TokenStream {
return getTokens(start,stop, s); return getTokens(start,stop, s);
} }
/** Given a starting index, return the index of the next on-channel /** Given a starting index, return the index of the next token on channel.
* token. Return i if tokens[i] is on channel. * Return i if tokens[i] is on channel. Return -1 if there are no tokens
* on channel between i and EOF.
*/ */
protected int nextTokenOnChannel(int i, int channel) { protected int nextTokenOnChannel(int i, int channel) {
sync(i); sync(i);
Token token = tokens.get(i); Token token = tokens.get(i);
while ( token.getType()!=Token.EOF && token.getChannel()!=channel ) { if ( i>=size() ) return -1;
while ( token.getChannel()!=channel ) {
if ( token.getType()==Token.EOF ) return -1;
i++; i++;
sync(i); sync(i);
token = tokens.get(i); token = tokens.get(i);
@ -253,8 +256,9 @@ public class BufferedTokenStream<T extends Token> implements TokenStream {
return i; return i;
} }
/** Given a starting index, return the index of the previous on-channel /** Given a starting index, return the index of the previous token on channel.
* token. Return i if tokens[i] is on channel. * Return i if tokens[i] is on channel. Return -1 if there are no tokens
* on channel between i and 0.
*/ */
protected int previousTokenOnChannel(int i, int channel) { protected int previousTokenOnChannel(int i, int channel) {
while ( i>=0 && tokens.get(i).getChannel()!=channel ) { while ( i>=0 && tokens.get(i).getChannel()!=channel ) {
@ -263,81 +267,75 @@ public class BufferedTokenStream<T extends Token> implements TokenStream {
return i; return i;
} }
public List<T> getOffChannelTokensToRight(int direction, int tokenIndex, int channel) {
if ( p == -1 ) setup();
if ( tokenIndex<0 || tokenIndex>=tokens.size() ) {
throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1));
}
int nextOnChannel = nextTokenOnChannel(tokenIndex + 1, channel);
if ( nextOnChannel == tokenIndex + 1 ) return null;
// get tokens to right up to next on channel
return tokens.subList(tokenIndex+1, nextOnChannel);
}
/** Collect all tokens on or off specified channel to the right or left of
* the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
* @param direction -1 means to the left, 1 means to the right.
* @param onchannel if true, get all tokens on the channel, else off channel
*/
public List<T> getHiddenTokens(int direction, boolean onchannel, int tokenIndex, int channel) {
if ( p == -1 ) setup();
if ( tokenIndex<0 || tokenIndex>=tokens.size() ) {
throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1));
}
int from;
int to;
if ( direction == -1 ) {
int prevOnChannel = previousTokenOnChannel(tokenIndex - 1, channel);
if ( prevOnChannel == tokenIndex - 1 ) return null;
from = prevOnChannel+1;
to = tokenIndex-1;
}
else {
int nextOnChannel = nextTokenOnChannel(tokenIndex + 1, channel);
if ( nextOnChannel == tokenIndex + 1 ) return null;
from = tokenIndex+1;
to = nextOnChannel;
}
List<T> hidden = new ArrayList<T>();
for (int i=from; i<=to; i++) {
T t = tokens.get(i);
if ( t.getChannel()==channel && onchannel) hidden.add(t);
else if ( t.getChannel()!=channel && !onchannel) hidden.add(t);
}
return hidden;
}
/** Collect all tokens on specified channel to the right of /** Collect all tokens on specified channel to the right of
* the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL or
* EOF. If channel is -1, find any non default channel token.
*/ */
public List<T> getHiddenTokensToRight(int tokenIndex, int channel) { public List<T> getHiddenTokensToRight(int tokenIndex, int channel) {
return getHiddenTokens(1, true, tokenIndex, channel); if ( p == -1 ) setup();
if ( tokenIndex<0 || tokenIndex>=tokens.size() ) {
throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1));
}
int nextOnChannel =
nextTokenOnChannel(tokenIndex + 1, Lexer.DEFAULT_TOKEN_CHANNEL);
int to;
int from = tokenIndex+1;
// if none onchannel to right, nextOnChannel=-1 so set to = last token
if ( nextOnChannel == -1 ) to = size()-1;
else to = nextOnChannel;
return filterForChannel(from, to, channel);
}
/** Collect all hidden tokens (any off-default channel) to the right of
* the current token up until we see a token on DEFAULT_TOKEN_CHANNEL
* of EOF.
*/
public List<T> getHiddenTokensToRight(int tokenIndex) {
return getHiddenTokensToRight(tokenIndex, -1);
} }
/** Collect all tokens on specified channel to the left of /** Collect all tokens on specified channel to the left of
* the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
* If channel is -1, find any non default channel token.
*/ */
public List<T> getHiddenTokensToLeft(int tokenIndex, int channel) { public List<T> getHiddenTokensToLeft(int tokenIndex, int channel) {
return getHiddenTokens(-1, true, tokenIndex, channel); if ( p == -1 ) setup();
} if ( tokenIndex<0 || tokenIndex>=tokens.size() ) {
throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1));
}
/** Collect all hidden tokens (any off-default channel) to the right of int prevOnChannel =
* the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. previousTokenOnChannel(tokenIndex - 1, Lexer.DEFAULT_TOKEN_CHANNEL);
*/ if ( prevOnChannel == tokenIndex - 1 ) return null;
public List<T> getHiddenTokensToRight(int tokenIndex) { // if none onchannel to left, prevOnChannel=-1 then from=0
return getHiddenTokens(1, false, tokenIndex, Lexer.DEFAULT_TOKEN_CHANNEL); int from = prevOnChannel+1;
int to = tokenIndex-1;
return filterForChannel(from, to, channel);
} }
/** Collect all hidden tokens (any off-default channel) to the left of /** Collect all hidden tokens (any off-default channel) to the left of
* the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
*/ */
public List<T> getHiddenTokensToLeft(int tokenIndex) { public List<T> getHiddenTokensToLeft(int tokenIndex) {
return getHiddenTokens(-1, false, tokenIndex, Lexer.DEFAULT_TOKEN_CHANNEL); return getHiddenTokensToLeft(tokenIndex, -1);
}
protected List<T> filterForChannel(int from, int to, int channel) {
List<T> hidden = new ArrayList<T>();
for (int i=from; i<=to; i++) {
T t = tokens.get(i);
if ( channel==-1 ) {
if ( t.getChannel()!= Lexer.DEFAULT_TOKEN_CHANNEL ) hidden.add(t);
}
else {
if ( t.getChannel()==channel ) hidden.add(t);
}
}
if ( hidden.size()==0 ) return null;
return hidden;
} }
@Override @Override

22
tool/test/org/antlr/v4/test/TestCommonTokenStream.java
View File

@ -88,17 +88,17 @@ public class TestCommonTokenStream extends TestBufferedTokenStream {
new TokenSource() { new TokenSource() {
int i = 0; int i = 0;
WritableToken[] tokens = { WritableToken[] tokens = {
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 0
new CommonToken(1,"x"), new CommonToken(1,"x"), // 1
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 2
new CommonToken(1,"="), new CommonToken(1,"="), // 3
new CommonToken(1,"34"), new CommonToken(1,"34"), // 4
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 5
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 6
new CommonToken(1,";"), new CommonToken(1,";"), // 7
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},// 8
new CommonToken(1,"\n") {{channel = Lexer.HIDDEN;}}, new CommonToken(1,"\n") {{channel = Lexer.HIDDEN;}},// 9
new CommonToken(Token.EOF,"") new CommonToken(Token.EOF,"") // 10
}; };
@Override @Override
public Token nextToken() { public Token nextToken() {