Merge branch 'lexer-eof' of github.com:sharwell/antlr4

runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java

@@ -144,6 +144,11 @@ public class ANTLRInputStream implements CharStream {
 
     @Override
     public void consume() {
+		if (p >= n) {
+			assert LA(1) == CharStream.EOF;
+			throw new IllegalStateException("cannot consume EOF");
+		}
+
 		//System.out.println("prev p="+p+", c="+(char)data[p]);
         if ( p < n ) {
             p++;

runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java

@@ -103,9 +103,6 @@ public abstract class ATNSimulator {
 		atn.grammarType = toInt(data[p++]);
 		atn.maxTokenType = toInt(data[p++]);
 
-		// Set up target of all EOF edges emanating from rule stop states
-		ATNState eofTarget = new ATNState();
-
 		//
 		// STATES
 		//
@@ -237,14 +234,6 @@ public abstract class ATNSimulator {
 			}
 		}
 
-		// If no edges out of stop state, add EOF transition
-		for (RuleStopState ruleStopState : atn.ruleToStopState) {
-			if ( ruleStopState.getNumberOfTransitions()==0 ) {
-				Transition t = new AtomTransition(eofTarget, Token.EOF);
-				ruleStopState.addTransition(t);
-			}
-		}
-
 		for (ATNState state : atn.states) {
 			if (state instanceof BlockStartState) {
 				// we need to know the end state to set its start state

runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java

@@ -265,10 +265,16 @@ public class LexerATNSimulator extends ATNSimulator {
 
 			if (target.isAcceptState) {
 				captureSimState(prevAccept, input, target);
+				if (t == IntStream.EOF) {
+					break;
+				}
+			}
+
+			if (t != IntStream.EOF) {
+				consume(input);
+				t = input.LA(1);
 			}
 
-			consume(input);
-			t = input.LA(1);
 			s = target; // flip; current DFA target becomes new src/from state
 		}
 
@@ -342,7 +348,9 @@ public class LexerATNSimulator extends ATNSimulator {
 		input.seek(index);
 		this.line = line;
 		this.charPositionInLine = charPos;
-		consume(input);
+		if (input.LA(1) != IntStream.EOF) {
+			consume(input);
+		}
 	}
 
 	@Nullable

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -7,7 +7,7 @@ javaTypeInitMap ::= [
 	"byte":"0",
 	"short":"0",
 	"char":"0",
-	default:"null" // anything other than an atomic type
+	default:"null" // anything other than a primitive type is an object
 ]
 
 // args must be <object-model-object>, <fields-resulting-in-STs>

tool/test/org/antlr/v4/test/TestSemPredEvalLexer.java

@@ -130,4 +130,22 @@ public class TestSemPredEvalLexer extends BaseTest {
 			"[@4,12:11='<EOF>',<-1>,3:0]\n";
 		assertEquals(expecting, found);
 	}
+
+	@Test public void testPredicatedKeywords() {
+		String grammar =
+			"lexer grammar A;" +
+			"ENUM : [a-z]+ {getText().equals(\"enum\")}? {System.out.println(\"enum!\");} ;\n" +
+			"ID   : [a-z]+ {System.out.println(\"ID \"+getText());} ;\n" +
+			"WS   : [ \n] -> skip ;";
+		String found = execLexer("A.g4", grammar, "A", "enum enu a");
+		String expecting =
+			"enum!\n" +
+			"ID enu\n" +
+			"ID a\n" +
+			"[@0,0:3='enum',<1>,1:0]\n" +
+			"[@1,5:7='enu',<2>,1:5]\n" +
+			"[@2,9:9='a',<2>,1:9]\n" +
+			"[@3,10:9='<EOF>',<-1>,1:10]\n";
+		assertEquals(expecting, found);
+	}
 }