diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index e76d87e70..686c06454 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -263,11 +263,7 @@ public class BufferedTokenStream implements TokenStream { return i; } - /** Return a list of all tokens on channel to right of tokenIndex - * until token not on channel found. Current token not considered. - * channel is typically Lexer.DEFAULT_TOKEN_CHANNEL. - */ - public List getOffChannelTokensToRight(int tokenIndex, int channel) { + public List getOffChannelTokensToRight(int direction, int tokenIndex, int channel) { if ( p == -1 ) setup(); if ( tokenIndex<0 || tokenIndex>=tokens.size() ) { throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1)); @@ -281,30 +277,67 @@ public class BufferedTokenStream implements TokenStream { return tokens.subList(tokenIndex+1, nextOnChannel); } - /** Return a list of all tokens on channel to left of tokenIndex - * until token not on channel found. Current token not considered. - * channel is typically Lexer.DEFAULT_TOKEN_CHANNEL. + /** Collect all tokens on or off specified channel to the right or left of + * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. + * @param direction -1 means to the left, 1 means to the right. + * @param onchannel if true, get all tokens on the channel, else off channel */ - public List getOffChannelTokensToLeft(int tokenIndex, int channel) { + public List getHiddenTokens(int direction, boolean onchannel, int tokenIndex, int channel) { if ( p == -1 ) setup(); if ( tokenIndex<0 || tokenIndex>=tokens.size() ) { throw new IndexOutOfBoundsException(tokenIndex+" not in 0.."+(tokens.size()-1)); } - int prevOnChannel = previousTokenOnChannel(tokenIndex - 1, channel); + int from; + int to; + if ( direction == -1 ) { + int prevOnChannel = previousTokenOnChannel(tokenIndex - 1, channel); + if ( prevOnChannel == tokenIndex - 1 ) return null; + from = prevOnChannel+1; + to = tokenIndex-1; + } + else { + int nextOnChannel = nextTokenOnChannel(tokenIndex + 1, channel); + if ( nextOnChannel == tokenIndex + 1 ) return null; + from = tokenIndex+1; + to = nextOnChannel; + } - if ( prevOnChannel == tokenIndex - 1 ) return null; - - // get tokens to left up starting at prev on channel + 1 to just before tokenIndex - return tokens.subList(prevOnChannel+1, tokenIndex); + List hidden = new ArrayList(); + for (int i=from; i<=to; i++) { + T t = tokens.get(i); + if ( t.getChannel()==channel && onchannel) hidden.add(t); + else if ( t.getChannel()!=channel && !onchannel) hidden.add(t); + } + return hidden; } + /** Collect all tokens on specified channel to the right of + * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. + */ + public List getHiddenTokensToRight(int tokenIndex, int channel) { + return getHiddenTokens(1, true, tokenIndex, channel); + } + + /** Collect all tokens on specified channel to the left of + * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. + */ + public List getHiddenTokensToLeft(int tokenIndex, int channel) { + return getHiddenTokens(-1, true, tokenIndex, channel); + } + + /** Collect all hidden tokens (any off-default channel) to the right of + * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. + */ public List getHiddenTokensToRight(int tokenIndex) { - return getOffChannelTokensToRight(tokenIndex, Lexer.DEFAULT_TOKEN_CHANNEL); + return getHiddenTokens(1, false, tokenIndex, Lexer.DEFAULT_TOKEN_CHANNEL); } + /** Collect all hidden tokens (any off-default channel) to the left of + * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. + */ public List getHiddenTokensToLeft(int tokenIndex) { - return getOffChannelTokensToLeft(tokenIndex, Lexer.DEFAULT_TOKEN_CHANNEL); + return getHiddenTokens(-1, false, tokenIndex, Lexer.DEFAULT_TOKEN_CHANNEL); } @Override diff --git a/tool/test/org/antlr/v4/test/TestCommonTokenStream.java b/tool/test/org/antlr/v4/test/TestCommonTokenStream.java index 6394d9ef7..3190591d6 100644 --- a/tool/test/org/antlr/v4/test/TestCommonTokenStream.java +++ b/tool/test/org/antlr/v4/test/TestCommonTokenStream.java @@ -293,40 +293,40 @@ public class TestCommonTokenStream extends BaseTest { assertEquals(null, tokens.getHiddenTokensToLeft(0)); assertEquals(null, tokens.getHiddenTokensToRight(0)); - assertEquals("[[@0,0:0=' ',<1>,channel=99,0:-1]]", + assertEquals("[[@0,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(1).toString()); - assertEquals("[[@2,0:0=' ',<1>,channel=99,0:-1]]", + assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(1).toString()); assertEquals(null, tokens.getHiddenTokensToLeft(2)); assertEquals(null, tokens.getHiddenTokensToRight(2)); - assertEquals("[[@2,0:0=' ',<1>,channel=99,0:-1]]", + assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(3).toString()); assertEquals(null, tokens.getHiddenTokensToRight(3)); assertEquals(null, tokens.getHiddenTokensToLeft(4)); - assertEquals("[[@5,0:0=' ',<1>,channel=99,0:-1], [@6,0:0=' ',<1>,channel=99,0:-1]]", + assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(4).toString()); assertEquals(null, tokens.getHiddenTokensToLeft(5)); - assertEquals("[[@6,0:0=' ',<1>,channel=99,0:-1]]", + assertEquals("[[@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(5).toString()); - assertEquals("[[@5,0:0=' ',<1>,channel=99,0:-1]]", + assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(6).toString()); assertEquals(null, tokens.getHiddenTokensToRight(6)); - assertEquals("[[@5,0:0=' ',<1>,channel=99,0:-1], [@6,0:0=' ',<1>,channel=99,0:-1]]", + assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(7).toString()); - assertEquals("[[@8,0:0=' ',<1>,channel=99,0:-1], [@9,0:0='\\n',<1>,channel=99,0:-1]]", + assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1], [@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(7).toString()); assertEquals(null, tokens.getHiddenTokensToLeft(8)); - assertEquals("[[@9,0:0='\\n',<1>,channel=99,0:-1]]", + assertEquals("[[@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(8).toString()); - assertEquals("[[@8,0:0=' ',<1>,channel=99,0:-1]]", + assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(9).toString()); assertEquals(null, tokens.getHiddenTokensToRight(9)); }