forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Remove special offset handling for EOFs. 

They can just use the position provided by the lexer.
This commit is contained in:
Michael Peyton Jones 2014-10-14 12:17:37 +01:00
parent 6aa36090dc
commit f3abfb79b7
5 changed files with 16 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
runtime/Java/src/org/antlr/v4/runtime/Lexer.java
View File

@ -274,14 +274,9 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator>
public Token emitEOF() {
int cpos = getCharPositionInLine();
// The character position for EOF is one beyond the position of
// the previous token's last character
if ( _token !=null ) {
int n = _token.getStopIndex() - _token.getStartIndex() + 1;
cpos = _token.getCharPositionInLine()+n;
}
int line = getLine();
Token eof = _factory.create(_tokenFactorySourcePair, Token.EOF, null, Token.DEFAULT_CHANNEL, _input.index(), _input.index()-1,
getLine(), cpos);
line, cpos);
emit(eof);
return eof;
}

4
tool/test/org/antlr/v4/test/TestLexerActions.java
View File

@ -79,7 +79,7 @@ public class TestLexerActions extends BaseTest {
"Hello: Steve\n" +
"\n" +
"[@0,0:11='hello Steve\\n',<1>,1:0]\n" +
"[@1,12:11='<EOF>',<-1>,2:12]\n";
"[@1,12:11='<EOF>',<-1>,2:0]\n";
assertEquals(expecting, found);
}
@ -276,7 +276,7 @@ public class TestLexerActions extends BaseTest {
"[@3,31:31='\\n',<4>,2:18]\n" +
"[@4,32:45='Another line.\\n',<1>,3:0]\n" +
"[@5,46:56='More line.\\n',<1>,4:0]\n" +
"[@6,57:56='<EOF>',<-1>,5:11]\n";
"[@6,57:56='<EOF>',<-1>,5:0]\n";
assertEquals(expecting, found);
}

20
tool/test/org/antlr/v4/test/TestLexerExec.java
View File

@ -147,7 +147,7 @@ public class TestLexerExec extends BaseTest {
String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n");
assertEquals(
"[@0,0:13='//blah\\n//blah\\n',<1>,1:0]\n" +
"[@1,14:13='<EOF>',<-1>,3:14]\n", found);
"[@1,14:13='<EOF>',<-1>,3:0]\n", found);
assertNull(stderrDuringParse);
}
@ -162,7 +162,7 @@ public class TestLexerExec extends BaseTest {
assertEquals(
"[@0,0:6='//blah\\n',<1>,1:0]\n" +
"[@1,7:13='//blah\\n',<1>,2:0]\n" +
"[@2,14:13='<EOF>',<-1>,3:7]\n", found);
"[@2,14:13='<EOF>',<-1>,3:0]\n", found);
assertNull(stderrDuringParse);
}
@ -176,7 +176,7 @@ public class TestLexerExec extends BaseTest {
String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n");
assertEquals(
"[@0,0:13='//blah\\n//blah\\n',<1>,1:0]\n" +
"[@1,14:13='<EOF>',<-1>,3:14]\n", found);
"[@1,14:13='<EOF>',<-1>,3:0]\n", found);
assertNull(stderrDuringParse);
}
@ -191,7 +191,7 @@ public class TestLexerExec extends BaseTest {
assertEquals(
"[@0,0:6='//blah\\n',<1>,1:0]\n" +
"[@1,7:13='//blah\\n',<1>,2:0]\n" +
"[@2,14:13='<EOF>',<-1>,3:7]\n", found);
"[@2,14:13='<EOF>',<-1>,3:0]\n", found);
assertNull(stderrDuringParse);
}
@ -205,7 +205,7 @@ public class TestLexerExec extends BaseTest {
String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n");
assertEquals(
"[@0,0:13='//blah\\n//blah\\n',<1>,1:0]\n" +
"[@1,14:13='<EOF>',<-1>,3:14]\n", found);
"[@1,14:13='<EOF>',<-1>,3:0]\n", found);
assertNull(stderrDuringParse);
}
@ -220,7 +220,7 @@ public class TestLexerExec extends BaseTest {
assertEquals(
"[@0,0:6='//blah\\n',<1>,1:0]\n" +
"[@1,7:13='//blah\\n',<1>,2:0]\n" +
"[@2,14:13='<EOF>',<-1>,3:7]\n", found);
"[@2,14:13='<EOF>',<-1>,3:0]\n", found);
assertNull(stderrDuringParse);
}
@ -237,7 +237,7 @@ public class TestLexerExec extends BaseTest {
"[@1,9:9='\\n',<2>,1:9]\n" +
"[@2,10:34='/* /* */\\n/* /*nested*/ */',<1>,2:0]\n" +
"[@3,35:35='\\n',<2>,3:16]\n" +
"[@4,36:35='<EOF>',<-1>,4:17]\n";
"[@4,36:35='<EOF>',<-1>,4:0]\n";
// stuff on end of comment matches another rule
String found = execLexer("L.g4", grammar, "L",
@ -262,7 +262,7 @@ public class TestLexerExec extends BaseTest {
"[@1,10:10='\\n',<2>,1:10]\n" +
"[@2,11:36='/* /* */x\\n/* /*nested*/ */',<1>,2:0]\n" +
"[@3,38:38='\\n',<2>,3:17]\n" +
"[@4,39:38='<EOF>',<-1>,4:18]\n";
"[@4,39:38='<EOF>',<-1>,4:0]\n";
String found = execLexer("L.g4", grammar, "L",
"/* ick */x\n" +
"/* /* */x\n" +
@ -286,7 +286,7 @@ public class TestLexerExec extends BaseTest {
"[@1,9:9='\\n',<2>,1:9]\n" +
"[@2,10:34='/* /* */\\n/* /*nested*/ */',<1>,2:0]\n" +
"[@3,35:35='\\n',<2>,3:16]\n" +
"[@4,36:35='<EOF>',<-1>,4:17]\n";
"[@4,36:35='<EOF>',<-1>,4:0]\n";
// stuff on end of comment matches another rule
String found = execLexer("L.g4", grammar, "L",
@ -311,7 +311,7 @@ public class TestLexerExec extends BaseTest {
"[@1,10:10='\\n',<2>,1:10]\n" +
"[@2,11:36='/* /* */x\\n/* /*nested*/ */',<1>,2:0]\n" +
"[@3,38:38='\\n',<2>,3:17]\n" +
"[@4,39:38='<EOF>',<-1>,4:18]\n";
"[@4,39:38='<EOF>',<-1>,4:0]\n";
String found = execLexer("L.g4", grammar, "L",
"/* ick */x\n" +
"/* /* */x\n" +

2
tool/test/org/antlr/v4/test/TestSemPredEvalLexer.java
View File

@ -129,7 +129,7 @@ public class TestSemPredEvalLexer extends BaseTest {
"[@3,6:8='def',<1>,2:2]\n" + // ID
"[@4,9:10=' ',<4>,2:5]\n" + // WS
"[@5,11:11='\\n',<3>,2:7]\n" +
"[@6,12:11='<EOF>',<-1>,3:8]\n" +
"[@6,12:11='<EOF>',<-1>,3:0]\n" +
"s0-'\n" +
"'->:s2=>3\n" +
"s0-'a'->:s1=>1\n" +

2
tool/test/org/antlr/v4/test/TestSemPredEvalParser.java
View File

@ -614,7 +614,7 @@ public class TestSemPredEvalParser extends BaseTest {
String found = execParser("T2.g4", grammar, "T2Parser", "T2Lexer", "file",
input, true);
assertEquals("(file (para (paraContent s) \\n \\n) (para (paraContent \\n x \\n)) <EOF>)\n", found);
assertEquals(stderrDuringParse, "line 5:2 mismatched input '<EOF>' expecting '\n'\n");
assertEquals(stderrDuringParse, "line 5:0 mismatched input '<EOF>' expecting '\n'\n");
input = "s\n\n\nx\n\n";
found = execParser("T2.g4", grammar, "T2Parser", "T2Lexer", "file",