From f3abfb79b724643e00868f941653d1290685856a Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Tue, 14 Oct 2014 12:17:37 +0100 Subject: [PATCH] Remove special offset handling for EOFs. They can just use the position provided by the lexer. --- .../Java/src/org/antlr/v4/runtime/Lexer.java | 9 ++------- .../org/antlr/v4/test/TestLexerActions.java | 4 ++-- .../test/org/antlr/v4/test/TestLexerExec.java | 20 +++++++++---------- .../antlr/v4/test/TestSemPredEvalLexer.java | 2 +- .../antlr/v4/test/TestSemPredEvalParser.java | 2 +- 5 files changed, 16 insertions(+), 21 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java index 0584188f1..3f6c4c447 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java @@ -274,14 +274,9 @@ public abstract class Lexer extends Recognizer public Token emitEOF() { int cpos = getCharPositionInLine(); - // The character position for EOF is one beyond the position of - // the previous token's last character - if ( _token !=null ) { - int n = _token.getStopIndex() - _token.getStartIndex() + 1; - cpos = _token.getCharPositionInLine()+n; - } + int line = getLine(); Token eof = _factory.create(_tokenFactorySourcePair, Token.EOF, null, Token.DEFAULT_CHANNEL, _input.index(), _input.index()-1, - getLine(), cpos); + line, cpos); emit(eof); return eof; } diff --git a/tool/test/org/antlr/v4/test/TestLexerActions.java b/tool/test/org/antlr/v4/test/TestLexerActions.java index 68f739b19..1e19a3c05 100644 --- a/tool/test/org/antlr/v4/test/TestLexerActions.java +++ b/tool/test/org/antlr/v4/test/TestLexerActions.java @@ -79,7 +79,7 @@ public class TestLexerActions extends BaseTest { "Hello: Steve\n" + "\n" + "[@0,0:11='hello Steve\\n',<1>,1:0]\n" + - "[@1,12:11='',<-1>,2:12]\n"; + "[@1,12:11='',<-1>,2:0]\n"; assertEquals(expecting, found); } @@ -276,7 +276,7 @@ public class TestLexerActions extends BaseTest { "[@3,31:31='\\n',<4>,2:18]\n" + "[@4,32:45='Another line.\\n',<1>,3:0]\n" + "[@5,46:56='More line.\\n',<1>,4:0]\n" + - "[@6,57:56='',<-1>,5:11]\n"; + "[@6,57:56='',<-1>,5:0]\n"; assertEquals(expecting, found); } diff --git a/tool/test/org/antlr/v4/test/TestLexerExec.java b/tool/test/org/antlr/v4/test/TestLexerExec.java index 88a060734..c010ce24d 100644 --- a/tool/test/org/antlr/v4/test/TestLexerExec.java +++ b/tool/test/org/antlr/v4/test/TestLexerExec.java @@ -147,7 +147,7 @@ public class TestLexerExec extends BaseTest { String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n"); assertEquals( "[@0,0:13='//blah\\n//blah\\n',<1>,1:0]\n" + - "[@1,14:13='',<-1>,3:14]\n", found); + "[@1,14:13='',<-1>,3:0]\n", found); assertNull(stderrDuringParse); } @@ -162,7 +162,7 @@ public class TestLexerExec extends BaseTest { assertEquals( "[@0,0:6='//blah\\n',<1>,1:0]\n" + "[@1,7:13='//blah\\n',<1>,2:0]\n" + - "[@2,14:13='',<-1>,3:7]\n", found); + "[@2,14:13='',<-1>,3:0]\n", found); assertNull(stderrDuringParse); } @@ -176,7 +176,7 @@ public class TestLexerExec extends BaseTest { String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n"); assertEquals( "[@0,0:13='//blah\\n//blah\\n',<1>,1:0]\n" + - "[@1,14:13='',<-1>,3:14]\n", found); + "[@1,14:13='',<-1>,3:0]\n", found); assertNull(stderrDuringParse); } @@ -191,7 +191,7 @@ public class TestLexerExec extends BaseTest { assertEquals( "[@0,0:6='//blah\\n',<1>,1:0]\n" + "[@1,7:13='//blah\\n',<1>,2:0]\n" + - "[@2,14:13='',<-1>,3:7]\n", found); + "[@2,14:13='',<-1>,3:0]\n", found); assertNull(stderrDuringParse); } @@ -205,7 +205,7 @@ public class TestLexerExec extends BaseTest { String found = execLexer("L.g4", grammar, "L", "//blah\n//blah\n"); assertEquals( "[@0,0:13='//blah\\n//blah\\n',<1>,1:0]\n" + - "[@1,14:13='',<-1>,3:14]\n", found); + "[@1,14:13='',<-1>,3:0]\n", found); assertNull(stderrDuringParse); } @@ -220,7 +220,7 @@ public class TestLexerExec extends BaseTest { assertEquals( "[@0,0:6='//blah\\n',<1>,1:0]\n" + "[@1,7:13='//blah\\n',<1>,2:0]\n" + - "[@2,14:13='',<-1>,3:7]\n", found); + "[@2,14:13='',<-1>,3:0]\n", found); assertNull(stderrDuringParse); } @@ -237,7 +237,7 @@ public class TestLexerExec extends BaseTest { "[@1,9:9='\\n',<2>,1:9]\n" + "[@2,10:34='/* /* */\\n/* /*nested*/ */',<1>,2:0]\n" + "[@3,35:35='\\n',<2>,3:16]\n" + - "[@4,36:35='',<-1>,4:17]\n"; + "[@4,36:35='',<-1>,4:0]\n"; // stuff on end of comment matches another rule String found = execLexer("L.g4", grammar, "L", @@ -262,7 +262,7 @@ public class TestLexerExec extends BaseTest { "[@1,10:10='\\n',<2>,1:10]\n" + "[@2,11:36='/* /* */x\\n/* /*nested*/ */',<1>,2:0]\n" + "[@3,38:38='\\n',<2>,3:17]\n" + - "[@4,39:38='',<-1>,4:18]\n"; + "[@4,39:38='',<-1>,4:0]\n"; String found = execLexer("L.g4", grammar, "L", "/* ick */x\n" + "/* /* */x\n" + @@ -286,7 +286,7 @@ public class TestLexerExec extends BaseTest { "[@1,9:9='\\n',<2>,1:9]\n" + "[@2,10:34='/* /* */\\n/* /*nested*/ */',<1>,2:0]\n" + "[@3,35:35='\\n',<2>,3:16]\n" + - "[@4,36:35='',<-1>,4:17]\n"; + "[@4,36:35='',<-1>,4:0]\n"; // stuff on end of comment matches another rule String found = execLexer("L.g4", grammar, "L", @@ -311,7 +311,7 @@ public class TestLexerExec extends BaseTest { "[@1,10:10='\\n',<2>,1:10]\n" + "[@2,11:36='/* /* */x\\n/* /*nested*/ */',<1>,2:0]\n" + "[@3,38:38='\\n',<2>,3:17]\n" + - "[@4,39:38='',<-1>,4:18]\n"; + "[@4,39:38='',<-1>,4:0]\n"; String found = execLexer("L.g4", grammar, "L", "/* ick */x\n" + "/* /* */x\n" + diff --git a/tool/test/org/antlr/v4/test/TestSemPredEvalLexer.java b/tool/test/org/antlr/v4/test/TestSemPredEvalLexer.java index 95934d397..2f5032439 100644 --- a/tool/test/org/antlr/v4/test/TestSemPredEvalLexer.java +++ b/tool/test/org/antlr/v4/test/TestSemPredEvalLexer.java @@ -129,7 +129,7 @@ public class TestSemPredEvalLexer extends BaseTest { "[@3,6:8='def',<1>,2:2]\n" + // ID "[@4,9:10=' ',<4>,2:5]\n" + // WS "[@5,11:11='\\n',<3>,2:7]\n" + - "[@6,12:11='',<-1>,3:8]\n" + + "[@6,12:11='',<-1>,3:0]\n" + "s0-'\n" + "'->:s2=>3\n" + "s0-'a'->:s1=>1\n" + diff --git a/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java b/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java index ed347a8ab..f9305a73c 100644 --- a/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java +++ b/tool/test/org/antlr/v4/test/TestSemPredEvalParser.java @@ -614,7 +614,7 @@ public class TestSemPredEvalParser extends BaseTest { String found = execParser("T2.g4", grammar, "T2Parser", "T2Lexer", "file", input, true); assertEquals("(file (para (paraContent s) \\n \\n) (para (paraContent \\n x \\n)) )\n", found); - assertEquals(stderrDuringParse, "line 5:2 mismatched input '' expecting '\n'\n"); + assertEquals(stderrDuringParse, "line 5:0 mismatched input '' expecting '\n'\n"); input = "s\n\n\nx\n\n"; found = execParser("T2.g4", grammar, "T2Parser", "T2Lexer", "file",