jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Fix handling of lexer accept positions for zero-length tokens and at EOF

This commit is contained in:
Sam Harwell 2014-10-26 15:44:43 -05:00
parent 68bf457b87
commit e0cf581d40
1 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -200,8 +200,6 @@ public class LexerATNSimulator extends ATNSimulator {
if (ds0.isAcceptState) {
// allow zero-length tokens
captureSimState(prevAccept, input, ds0);
// adjust index since the current input character was not yet consumed
prevAccept.index--;
}
int t = input.LA(1);
@ -239,6 +237,14 @@ public class LexerATNSimulator extends ATNSimulator {
break;
}
// If this is a consumable input element, make sure to consume before
// capturing the accept state so the input index, line, and char
// position accurately reflect the state of the interpreter at the
// end of the token.
if (t != IntStream.EOF) {
consume(input);
}
if (target.isAcceptState) {
captureSimState(prevAccept, input, target);
if (t == IntStream.EOF) {
@ -246,11 +252,7 @@ public class LexerATNSimulator extends ATNSimulator {
}
}
if (t != IntStream.EOF) {
consume(input);
t = input.LA(1);
}
s = target; // flip; current DFA target becomes new src/from state
}
@ -388,9 +390,6 @@ public class LexerATNSimulator extends ATNSimulator {
input.seek(index);
this.line = line;
this.charPositionInLine = charPos;
if (input.LA(1) != IntStream.EOF) {
consume(input);
}
if (lexerActionExecutor != null && recog != null) {
lexerActionExecutor.execute(recog, input, startIndex);