diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index 686c06454..96f8d3754 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -239,13 +239,16 @@ public class BufferedTokenStream implements TokenStream { return getTokens(start,stop, s); } - /** Given a starting index, return the index of the next on-channel - * token. Return i if tokens[i] is on channel. + /** Given a starting index, return the index of the next token on channel. + * Return i if tokens[i] is on channel. Return -1 if there are no tokens + * on channel between i and EOF. */ protected int nextTokenOnChannel(int i, int channel) { sync(i); Token token = tokens.get(i); - while ( token.getType()!=Token.EOF && token.getChannel()!=channel ) { + if ( i>=size() ) return -1; + while ( token.getChannel()!=channel ) { + if ( token.getType()==Token.EOF ) return -1; i++; sync(i); token = tokens.get(i); @@ -253,8 +256,9 @@ public class BufferedTokenStream implements TokenStream { return i; } - /** Given a starting index, return the index of the previous on-channel - * token. Return i if tokens[i] is on channel. + /** Given a starting index, return the index of the previous token on channel. + * Return i if tokens[i] is on channel. Return -1 if there are no tokens + * on channel between i and 0. */ protected int previousTokenOnChannel(int i, int channel) { while ( i>=0 && tokens.get(i).getChannel()!=channel ) { @@ -263,81 +267,75 @@ public class BufferedTokenStream implements TokenStream { return i; } - public List getOffChannelTokensToRight(int direction, int tokenIndex, int channel) { - if ( p == -1 ) setup(); - if ( tokenIndex<0 || tokenIndex>=tokens.size() ) { - throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1)); - } - - int nextOnChannel = nextTokenOnChannel(tokenIndex + 1, channel); - - if ( nextOnChannel == tokenIndex + 1 ) return null; - - // get tokens to right up to next on channel - return tokens.subList(tokenIndex+1, nextOnChannel); - } - - /** Collect all tokens on or off specified channel to the right or left of - * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. - * @param direction -1 means to the left, 1 means to the right. - * @param onchannel if true, get all tokens on the channel, else off channel - */ - public List getHiddenTokens(int direction, boolean onchannel, int tokenIndex, int channel) { - if ( p == -1 ) setup(); - if ( tokenIndex<0 || tokenIndex>=tokens.size() ) { - throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1)); - } - - int from; - int to; - if ( direction == -1 ) { - int prevOnChannel = previousTokenOnChannel(tokenIndex - 1, channel); - if ( prevOnChannel == tokenIndex - 1 ) return null; - from = prevOnChannel+1; - to = tokenIndex-1; - } - else { - int nextOnChannel = nextTokenOnChannel(tokenIndex + 1, channel); - if ( nextOnChannel == tokenIndex + 1 ) return null; - from = tokenIndex+1; - to = nextOnChannel; - } - - List hidden = new ArrayList(); - for (int i=from; i<=to; i++) { - T t = tokens.get(i); - if ( t.getChannel()==channel && onchannel) hidden.add(t); - else if ( t.getChannel()!=channel && !onchannel) hidden.add(t); - } - return hidden; - } - /** Collect all tokens on specified channel to the right of - * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. + * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL or + * EOF. If channel is -1, find any non default channel token. */ public List getHiddenTokensToRight(int tokenIndex, int channel) { - return getHiddenTokens(1, true, tokenIndex, channel); + if ( p == -1 ) setup(); + if ( tokenIndex<0 || tokenIndex>=tokens.size() ) { + throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1)); + } + + int nextOnChannel = + nextTokenOnChannel(tokenIndex + 1, Lexer.DEFAULT_TOKEN_CHANNEL); + int to; + int from = tokenIndex+1; + // if none onchannel to right, nextOnChannel=-1 so set to = last token + if ( nextOnChannel == -1 ) to = size()-1; + else to = nextOnChannel; + + return filterForChannel(from, to, channel); + } + + /** Collect all hidden tokens (any off-default channel) to the right of + * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL + * of EOF. + */ + public List getHiddenTokensToRight(int tokenIndex) { + return getHiddenTokensToRight(tokenIndex, -1); } /** Collect all tokens on specified channel to the left of * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. + * If channel is -1, find any non default channel token. */ public List getHiddenTokensToLeft(int tokenIndex, int channel) { - return getHiddenTokens(-1, true, tokenIndex, channel); - } + if ( p == -1 ) setup(); + if ( tokenIndex<0 || tokenIndex>=tokens.size() ) { + throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1)); + } - /** Collect all hidden tokens (any off-default channel) to the right of - * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. - */ - public List getHiddenTokensToRight(int tokenIndex) { - return getHiddenTokens(1, false, tokenIndex, Lexer.DEFAULT_TOKEN_CHANNEL); + int prevOnChannel = + previousTokenOnChannel(tokenIndex - 1, Lexer.DEFAULT_TOKEN_CHANNEL); + if ( prevOnChannel == tokenIndex - 1 ) return null; + // if none onchannel to left, prevOnChannel=-1 then from=0 + int from = prevOnChannel+1; + int to = tokenIndex-1; + + return filterForChannel(from, to, channel); } /** Collect all hidden tokens (any off-default channel) to the left of * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. */ public List getHiddenTokensToLeft(int tokenIndex) { - return getHiddenTokens(-1, false, tokenIndex, Lexer.DEFAULT_TOKEN_CHANNEL); + return getHiddenTokensToLeft(tokenIndex, -1); + } + + protected List filterForChannel(int from, int to, int channel) { + List hidden = new ArrayList(); + for (int i=from; i<=to; i++) { + T t = tokens.get(i); + if ( channel==-1 ) { + if ( t.getChannel()!= Lexer.DEFAULT_TOKEN_CHANNEL ) hidden.add(t); + } + else { + if ( t.getChannel()==channel ) hidden.add(t); + } + } + if ( hidden.size()==0 ) return null; + return hidden; } @Override diff --git a/tool/test/org/antlr/v4/test/TestCommonTokenStream.java b/tool/test/org/antlr/v4/test/TestCommonTokenStream.java index b73efcd77..31aae118f 100644 --- a/tool/test/org/antlr/v4/test/TestCommonTokenStream.java +++ b/tool/test/org/antlr/v4/test/TestCommonTokenStream.java @@ -88,17 +88,17 @@ public class TestCommonTokenStream extends TestBufferedTokenStream { new TokenSource() { int i = 0; WritableToken[] tokens = { - new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, - new CommonToken(1,"x"), - new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, - new CommonToken(1,"="), - new CommonToken(1,"34"), - new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, - new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, - new CommonToken(1,";"), - new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, - new CommonToken(1,"\n") {{channel = Lexer.HIDDEN;}}, - new CommonToken(Token.EOF,"") + new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 0 + new CommonToken(1,"x"), // 1 + new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 2 + new CommonToken(1,"="), // 3 + new CommonToken(1,"34"), // 4 + new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 5 + new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 6 + new CommonToken(1,";"), // 7 + new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},// 8 + new CommonToken(1,"\n") {{channel = Lexer.HIDDEN;}},// 9 + new CommonToken(Token.EOF,"") // 10 }; @Override public Token nextToken() {