forked from jasder/antlr
fix bug in get hidden channel stuff. rewrote. update test.
This commit is contained in:
parent
ec47251bb2
commit
3b2c0a6177
|
@ -239,13 +239,16 @@ public class BufferedTokenStream<T extends Token> implements TokenStream {
|
|||
return getTokens(start,stop, s);
|
||||
}
|
||||
|
||||
/** Given a starting index, return the index of the next on-channel
|
||||
* token. Return i if tokens[i] is on channel.
|
||||
/** Given a starting index, return the index of the next token on channel.
|
||||
* Return i if tokens[i] is on channel. Return -1 if there are no tokens
|
||||
* on channel between i and EOF.
|
||||
*/
|
||||
protected int nextTokenOnChannel(int i, int channel) {
|
||||
sync(i);
|
||||
Token token = tokens.get(i);
|
||||
while ( token.getType()!=Token.EOF && token.getChannel()!=channel ) {
|
||||
if ( i>=size() ) return -1;
|
||||
while ( token.getChannel()!=channel ) {
|
||||
if ( token.getType()==Token.EOF ) return -1;
|
||||
i++;
|
||||
sync(i);
|
||||
token = tokens.get(i);
|
||||
|
@ -253,8 +256,9 @@ public class BufferedTokenStream<T extends Token> implements TokenStream {
|
|||
return i;
|
||||
}
|
||||
|
||||
/** Given a starting index, return the index of the previous on-channel
|
||||
* token. Return i if tokens[i] is on channel.
|
||||
/** Given a starting index, return the index of the previous token on channel.
|
||||
* Return i if tokens[i] is on channel. Return -1 if there are no tokens
|
||||
* on channel between i and 0.
|
||||
*/
|
||||
protected int previousTokenOnChannel(int i, int channel) {
|
||||
while ( i>=0 && tokens.get(i).getChannel()!=channel ) {
|
||||
|
@ -263,81 +267,75 @@ public class BufferedTokenStream<T extends Token> implements TokenStream {
|
|||
return i;
|
||||
}
|
||||
|
||||
public List<T> getOffChannelTokensToRight(int direction, int tokenIndex, int channel) {
|
||||
if ( p == -1 ) setup();
|
||||
if ( tokenIndex<0 || tokenIndex>=tokens.size() ) {
|
||||
throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1));
|
||||
}
|
||||
|
||||
int nextOnChannel = nextTokenOnChannel(tokenIndex + 1, channel);
|
||||
|
||||
if ( nextOnChannel == tokenIndex + 1 ) return null;
|
||||
|
||||
// get tokens to right up to next on channel
|
||||
return tokens.subList(tokenIndex+1, nextOnChannel);
|
||||
}
|
||||
|
||||
/** Collect all tokens on or off specified channel to the right or left of
|
||||
* the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
|
||||
* @param direction -1 means to the left, 1 means to the right.
|
||||
* @param onchannel if true, get all tokens on the channel, else off channel
|
||||
*/
|
||||
public List<T> getHiddenTokens(int direction, boolean onchannel, int tokenIndex, int channel) {
|
||||
if ( p == -1 ) setup();
|
||||
if ( tokenIndex<0 || tokenIndex>=tokens.size() ) {
|
||||
throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1));
|
||||
}
|
||||
|
||||
int from;
|
||||
int to;
|
||||
if ( direction == -1 ) {
|
||||
int prevOnChannel = previousTokenOnChannel(tokenIndex - 1, channel);
|
||||
if ( prevOnChannel == tokenIndex - 1 ) return null;
|
||||
from = prevOnChannel+1;
|
||||
to = tokenIndex-1;
|
||||
}
|
||||
else {
|
||||
int nextOnChannel = nextTokenOnChannel(tokenIndex + 1, channel);
|
||||
if ( nextOnChannel == tokenIndex + 1 ) return null;
|
||||
from = tokenIndex+1;
|
||||
to = nextOnChannel;
|
||||
}
|
||||
|
||||
List<T> hidden = new ArrayList<T>();
|
||||
for (int i=from; i<=to; i++) {
|
||||
T t = tokens.get(i);
|
||||
if ( t.getChannel()==channel && onchannel) hidden.add(t);
|
||||
else if ( t.getChannel()!=channel && !onchannel) hidden.add(t);
|
||||
}
|
||||
return hidden;
|
||||
}
|
||||
|
||||
/** Collect all tokens on specified channel to the right of
|
||||
* the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
|
||||
* the current token up until we see a token on DEFAULT_TOKEN_CHANNEL or
|
||||
* EOF. If channel is -1, find any non default channel token.
|
||||
*/
|
||||
public List<T> getHiddenTokensToRight(int tokenIndex, int channel) {
|
||||
return getHiddenTokens(1, true, tokenIndex, channel);
|
||||
if ( p == -1 ) setup();
|
||||
if ( tokenIndex<0 || tokenIndex>=tokens.size() ) {
|
||||
throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1));
|
||||
}
|
||||
|
||||
int nextOnChannel =
|
||||
nextTokenOnChannel(tokenIndex + 1, Lexer.DEFAULT_TOKEN_CHANNEL);
|
||||
int to;
|
||||
int from = tokenIndex+1;
|
||||
// if none onchannel to right, nextOnChannel=-1 so set to = last token
|
||||
if ( nextOnChannel == -1 ) to = size()-1;
|
||||
else to = nextOnChannel;
|
||||
|
||||
return filterForChannel(from, to, channel);
|
||||
}
|
||||
|
||||
/** Collect all hidden tokens (any off-default channel) to the right of
|
||||
* the current token up until we see a token on DEFAULT_TOKEN_CHANNEL
|
||||
* of EOF.
|
||||
*/
|
||||
public List<T> getHiddenTokensToRight(int tokenIndex) {
|
||||
return getHiddenTokensToRight(tokenIndex, -1);
|
||||
}
|
||||
|
||||
/** Collect all tokens on specified channel to the left of
|
||||
* the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
|
||||
* If channel is -1, find any non default channel token.
|
||||
*/
|
||||
public List<T> getHiddenTokensToLeft(int tokenIndex, int channel) {
|
||||
return getHiddenTokens(-1, true, tokenIndex, channel);
|
||||
if ( p == -1 ) setup();
|
||||
if ( tokenIndex<0 || tokenIndex>=tokens.size() ) {
|
||||
throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1));
|
||||
}
|
||||
|
||||
/** Collect all hidden tokens (any off-default channel) to the right of
|
||||
* the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
|
||||
*/
|
||||
public List<T> getHiddenTokensToRight(int tokenIndex) {
|
||||
return getHiddenTokens(1, false, tokenIndex, Lexer.DEFAULT_TOKEN_CHANNEL);
|
||||
int prevOnChannel =
|
||||
previousTokenOnChannel(tokenIndex - 1, Lexer.DEFAULT_TOKEN_CHANNEL);
|
||||
if ( prevOnChannel == tokenIndex - 1 ) return null;
|
||||
// if none onchannel to left, prevOnChannel=-1 then from=0
|
||||
int from = prevOnChannel+1;
|
||||
int to = tokenIndex-1;
|
||||
|
||||
return filterForChannel(from, to, channel);
|
||||
}
|
||||
|
||||
/** Collect all hidden tokens (any off-default channel) to the left of
|
||||
* the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
|
||||
*/
|
||||
public List<T> getHiddenTokensToLeft(int tokenIndex) {
|
||||
return getHiddenTokens(-1, false, tokenIndex, Lexer.DEFAULT_TOKEN_CHANNEL);
|
||||
return getHiddenTokensToLeft(tokenIndex, -1);
|
||||
}
|
||||
|
||||
protected List<T> filterForChannel(int from, int to, int channel) {
|
||||
List<T> hidden = new ArrayList<T>();
|
||||
for (int i=from; i<=to; i++) {
|
||||
T t = tokens.get(i);
|
||||
if ( channel==-1 ) {
|
||||
if ( t.getChannel()!= Lexer.DEFAULT_TOKEN_CHANNEL ) hidden.add(t);
|
||||
}
|
||||
else {
|
||||
if ( t.getChannel()==channel ) hidden.add(t);
|
||||
}
|
||||
}
|
||||
if ( hidden.size()==0 ) return null;
|
||||
return hidden;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -88,17 +88,17 @@ public class TestCommonTokenStream extends TestBufferedTokenStream {
|
|||
new TokenSource() {
|
||||
int i = 0;
|
||||
WritableToken[] tokens = {
|
||||
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
|
||||
new CommonToken(1,"x"),
|
||||
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
|
||||
new CommonToken(1,"="),
|
||||
new CommonToken(1,"34"),
|
||||
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
|
||||
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
|
||||
new CommonToken(1,";"),
|
||||
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
|
||||
new CommonToken(1,"\n") {{channel = Lexer.HIDDEN;}},
|
||||
new CommonToken(Token.EOF,"")
|
||||
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 0
|
||||
new CommonToken(1,"x"), // 1
|
||||
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 2
|
||||
new CommonToken(1,"="), // 3
|
||||
new CommonToken(1,"34"), // 4
|
||||
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 5
|
||||
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 6
|
||||
new CommonToken(1,";"), // 7
|
||||
new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},// 8
|
||||
new CommonToken(1,"\n") {{channel = Lexer.HIDDEN;}},// 9
|
||||
new CommonToken(Token.EOF,"") // 10
|
||||
};
|
||||
@Override
|
||||
public Token nextToken() {
|
||||
|
|
Loading…
Reference in New Issue