jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Merge pull request #691 from sharwell/fix-688 

Improved support for zero-length lexer tokens
This commit is contained in:
Terence Parr 2014-09-02 13:25:14 -07:00
parent 77a7bda127 e497efaa58
commit 6e581b3be6
3 changed files with 39 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -197,6 +197,13 @@ public class LexerATNSimulator extends ATNSimulator {
System.out.format(Locale.getDefault(), "start state closure=%s\n", ds0.configs);
}
if (ds0.isAcceptState) {
// allow zero-length tokens
captureSimState(prevAccept, input, ds0);
// adjust index since the current input character was not yet consumed
prevAccept.index--;
}
int t = input.LA(1);
@NotNull
DFAState s = ds0; // s is current/from DFA state

4
tool/src/org/antlr/v4/parse/ANTLRParser.g
View File

@ -554,11 +554,11 @@ lexerAlt
( lexerCommands -> ^(LEXER_ALT_ACTION<AltAST> lexerElements lexerCommands)
| -> lexerElements
)
| -> ^(ALT<AltAST> EPSILON) // empty alt
;
lexerElements
: lexerElement+ -> ^(ALT<AltAST> lexerElement+)
: lexerElement+ -> ^(ALT<AltAST> lexerElement+)
| -> ^(ALT<AltAST> EPSILON) // empty alt
;
lexerElement

30
tool/test/org/antlr/v4/test/TestLexerExec.java
View File

@ -657,4 +657,34 @@ public class TestLexerExec extends BaseTest {
"[@1,5:4='<EOF>',<-1>,1:5]\n";
assertEquals(expecting, found);
}
/**
* This is a regression test for antlr/antlr4#687 "Empty zero-length tokens
* cannot have lexer commands" and antlr/antlr4#688 "Lexer cannot match
* zero-length tokens"
* https://github.com/antlr/antlr4/issues/687
* https://github.com/antlr/antlr4/issues/688
*/
@Test public void testZeroLengthToken() throws Exception {
String grammar =
"lexer grammar L;\n"+
"\n" +
"BeginString\n" +
" : '\\'' -> more, pushMode(StringMode)\n" +
" ;\n" +
"\n" +
"mode StringMode;\n" +
"\n" +
" StringMode_X : 'x' -> more;\n" +
" StringMode_Done : -> more, mode(EndStringMode);\n" +
"\n" +
"mode EndStringMode; \n" +
"\n" +
" EndString : '\\'' -> popMode;\n";
String found = execLexer("L.g4", grammar, "L", "'xxx'");
String expecting =
"[@0,0:4=''xxx'',<1>,1:0]\n" +
"[@1,5:4='<EOF>',<-1>,1:5]\n";
assertEquals(expecting, found);
}
}