forked from jasder/antlr
Fix usage of BufferedTokenStream: cannot consume EOF
This commit is contained in:
parent
4832425848
commit
068075f1c4
|
@ -382,7 +382,9 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> {
|
||||||
*/
|
*/
|
||||||
public Token consume() {
|
public Token consume() {
|
||||||
Token o = getCurrentToken();
|
Token o = getCurrentToken();
|
||||||
getInputStream().consume();
|
if (o.getType() != EOF) {
|
||||||
|
getInputStream().consume();
|
||||||
|
}
|
||||||
boolean hasListener = _parseListeners != null && !_parseListeners.isEmpty();
|
boolean hasListener = _parseListeners != null && !_parseListeners.isEmpty();
|
||||||
if (_buildParseTrees || hasListener) {
|
if (_buildParseTrees || hasListener) {
|
||||||
if ( _errHandler.inErrorRecoveryMode(this) ) {
|
if ( _errHandler.inErrorRecoveryMode(this) ) {
|
||||||
|
|
|
@ -145,10 +145,6 @@ public class TestBufferedTokenStream extends BaseTest {
|
||||||
tokens.consume();
|
tokens.consume();
|
||||||
t = tokens.LT(1);
|
t = tokens.LT(1);
|
||||||
}
|
}
|
||||||
tokens.consume();
|
|
||||||
tokens.LT(1); // push it past end
|
|
||||||
tokens.consume();
|
|
||||||
tokens.LT(1);
|
|
||||||
|
|
||||||
String result = tokens.getText();
|
String result = tokens.getText();
|
||||||
String expecting = "x = 3 * 0 + 2 * 0;";
|
String expecting = "x = 3 * 0 + 2 * 0;";
|
||||||
|
|
|
@ -255,12 +255,50 @@ public class TestCommonTokenStream extends TestBufferedTokenStream {
|
||||||
assertEquals(Token.EOF, tokens.LA(1));
|
assertEquals(Token.EOF, tokens.LA(1));
|
||||||
assertEquals(0, tokens.index());
|
assertEquals(0, tokens.index());
|
||||||
assertEquals(1, tokens.size());
|
assertEquals(1, tokens.size());
|
||||||
tokens.consume();
|
}
|
||||||
|
|
||||||
assertEquals(Token.EOF, tokens.LA(1));
|
@Test(expected = IllegalStateException.class)
|
||||||
assertEquals(0, tokens.index());
|
public void testCannotConsumeEOF() throws Exception {
|
||||||
assertEquals(1, tokens.size());
|
TokenSource lexer = new TokenSource() {
|
||||||
tokens.consume();
|
|
||||||
|
@Override
|
||||||
|
public Token nextToken() {
|
||||||
|
return new CommonToken(Token.EOF);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getLine() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getCharPositionInLine() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public CharStream getInputStream() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getSourceName() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public TokenFactory<?> getTokenFactory() {
|
||||||
|
throw new UnsupportedOperationException("Not supported yet.");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setTokenFactory(TokenFactory<?> factory) {
|
||||||
|
throw new UnsupportedOperationException("Not supported yet.");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
CommonTokenStream tokens = new CommonTokenStream(lexer);
|
||||||
|
tokens.fill();
|
||||||
|
|
||||||
assertEquals(Token.EOF, tokens.LA(1));
|
assertEquals(Token.EOF, tokens.LA(1));
|
||||||
assertEquals(0, tokens.index());
|
assertEquals(0, tokens.index());
|
||||||
|
|
Loading…
Reference in New Issue