From b6653293df131531c95ab1a4f38017361675818f Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 3 Nov 2012 14:07:47 -0500 Subject: [PATCH 1/5] Tweak comment --- .../resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index 597e8d8d7..8a045c69e 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -7,7 +7,7 @@ javaTypeInitMap ::= [ "byte":"0", "short":"0", "char":"0", - default:"null" // anything other than an atomic type + default:"null" // anything other than a primitive type is an object ] // args must be , From cf5bf7c3844f40cff1911e3e24df9a569797bfd3 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 3 Nov 2012 15:14:50 -0500 Subject: [PATCH 2/5] Add testPredicatedKeywords --- .../antlr/v4/test/TestSemPredEvalLexer.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tool/test/org/antlr/v4/test/TestSemPredEvalLexer.java b/tool/test/org/antlr/v4/test/TestSemPredEvalLexer.java index 8c736afa1..c1814fb42 100644 --- a/tool/test/org/antlr/v4/test/TestSemPredEvalLexer.java +++ b/tool/test/org/antlr/v4/test/TestSemPredEvalLexer.java @@ -130,4 +130,22 @@ public class TestSemPredEvalLexer extends BaseTest { "[@4,12:11='',<-1>,3:0]\n"; assertEquals(expecting, found); } + + @Test public void testPredicatedKeywords() { + String grammar = + "lexer grammar A;" + + "ENUM : [a-z]+ {getText().equals(\"enum\")}? {System.out.println(\"enum!\");} ;\n" + + "ID : [a-z]+ {System.out.println(\"ID \"+getText());} ;\n" + + "WS : [ \n] -> skip ;"; + String found = execLexer("A.g4", grammar, "A", "enum enu a"); + String expecting = + "enum!\n" + + "ID enu\n" + + "ID a\n" + + "[@0,0:3='enum',<1>,1:0]\n" + + "[@1,5:7='enu',<2>,1:5]\n" + + "[@2,9:9='a',<2>,1:9]\n" + + "[@3,10:9='',<-1>,1:10]\n"; + assertEquals(expecting, found); + } } From c9890e83050483ddbaa7a78db4ff90bbba7db3fa Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 3 Nov 2012 15:38:32 -0500 Subject: [PATCH 3/5] Throw IllegalStateException if the user tries to consume EOF --- runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java b/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java index de9258849..5408aedce 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java @@ -144,6 +144,11 @@ public class ANTLRInputStream implements CharStream { @Override public void consume() { + if (p >= n) { + assert LA(1) == CharStream.EOF; + throw new IllegalStateException("cannot consume EOF"); + } + //System.out.println("prev p="+p+", c="+(char)data[p]); if ( p < n ) { p++; From fbaac8194b1a2d8f7cb612961ee333e949ef6d68 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 3 Nov 2012 15:16:29 -0500 Subject: [PATCH 4/5] Avoid calling consume() at EOF --- .../antlr/v4/runtime/atn/LexerATNSimulator.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index 0b24707d0..5782c729e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -265,10 +265,16 @@ public class LexerATNSimulator extends ATNSimulator { if (target.isAcceptState) { captureSimState(prevAccept, input, target); + if (t == IntStream.EOF) { + break; + } + } + + if (t != IntStream.EOF) { + consume(input); + t = input.LA(1); } - consume(input); - t = input.LA(1); s = target; // flip; current DFA target becomes new src/from state } @@ -342,7 +348,9 @@ public class LexerATNSimulator extends ATNSimulator { input.seek(index); this.line = line; this.charPositionInLine = charPos; - consume(input); + if (input.LA(1) != IntStream.EOF) { + consume(input); + } } @Nullable From 28e29e39dc8370db27a1693a0d217ad4820d084a Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Sat, 3 Nov 2012 15:44:11 -0500 Subject: [PATCH 5/5] Remove code to explicitly add EOF transitions to rule stop states with no outgoing edges --- .../src/org/antlr/v4/runtime/atn/ATNSimulator.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java index 224cad6f6..39a3c3833 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java @@ -103,9 +103,6 @@ public abstract class ATNSimulator { atn.grammarType = toInt(data[p++]); atn.maxTokenType = toInt(data[p++]); - // Set up target of all EOF edges emanating from rule stop states - ATNState eofTarget = new ATNState(); - // // STATES // @@ -237,14 +234,6 @@ public abstract class ATNSimulator { } } - // If no edges out of stop state, add EOF transition - for (RuleStopState ruleStopState : atn.ruleToStopState) { - if ( ruleStopState.getNumberOfTransitions()==0 ) { - Transition t = new AtomTransition(eofTarget, Token.EOF); - ruleStopState.addTransition(t); - } - } - for (ATNState state : atn.states) { if (state instanceof BlockStartState) { // we need to know the end state to set its start state