Merge pull request #503 from sharwell/fix-469

Fix 469

runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java

@@ -36,8 +36,11 @@ import org.antlr.v4.runtime.misc.Nullable;
 import org.antlr.v4.runtime.misc.ObjectEqualityComparator;
 
 public class LexerATNConfig extends ATNConfig {
-	/** Capture lexer actions we traverse. */
+	/**
-	public LexerActionExecutor lexerActionExecutor;
+	 * This is the backing field for {@link #getLexerActionExecutor}.
+	 */
+	@Nullable
+	private final LexerActionExecutor lexerActionExecutor;
 
 	private final boolean passedThroughNonGreedyDecision;
 
@@ -47,12 +50,13 @@ public class LexerATNConfig extends ATNConfig {
 	{
 		super(state, alt, context, SemanticContext.NONE);
 		this.passedThroughNonGreedyDecision = false;
+		this.lexerActionExecutor = null;
 	}
 
 	public LexerATNConfig(@NotNull ATNState state,
 						  int alt,
 						  @Nullable PredictionContext context,
-						  LexerActionExecutor lexerActionExecutor)
+						  @Nullable LexerActionExecutor lexerActionExecutor)
 	{
 		super(state, alt, context, SemanticContext.NONE);
 		this.lexerActionExecutor = lexerActionExecutor;
@@ -66,7 +70,7 @@ public class LexerATNConfig extends ATNConfig {
 	}
 
 	public LexerATNConfig(@NotNull LexerATNConfig c, @NotNull ATNState state,
-						  LexerActionExecutor lexerActionExecutor)
+						  @Nullable LexerActionExecutor lexerActionExecutor)
 	{
 		super(c, state, c.context, c.semanticContext);
 		this.lexerActionExecutor = lexerActionExecutor;
@@ -80,6 +84,15 @@ public class LexerATNConfig extends ATNConfig {
 		this.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state);
 	}
 
+	/**
+	 * Gets the {@link LexerActionExecutor} capable of executing the embedded
+	 * action(s) for the current configuration.
+	 */
+	@Nullable
+	public final LexerActionExecutor getLexerActionExecutor() {
+		return lexerActionExecutor;
+	}
+
 	public final boolean hasPassedThroughNonGreedyDecision() {
 		return passedThroughNonGreedyDecision;
 	}

runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java

@@ -346,7 +346,7 @@ public class LexerATNSimulator extends ATNSimulator {
 				Transition trans = c.state.transition(ti);
 				ATNState target = getReachableTarget(trans, t);
 				if ( target!=null ) {
-					LexerActionExecutor lexerActionExecutor = ((LexerATNConfig)c).lexerActionExecutor;
+					LexerActionExecutor lexerActionExecutor = ((LexerATNConfig)c).getLexerActionExecutor();
 					if (lexerActionExecutor != null) {
 						lexerActionExecutor = lexerActionExecutor.fixOffsetBeforeMatch(input.index() - startIndex);
 					}
@@ -446,7 +446,7 @@ public class LexerATNSimulator extends ATNSimulator {
 					if (config.context.getReturnState(i) != PredictionContext.EMPTY_RETURN_STATE) {
 						PredictionContext newContext = config.context.getParent(i); // "pop" return state
 						ATNState returnState = atn.states.get(config.context.getReturnState(i));
-						LexerATNConfig c = new LexerATNConfig(returnState, config.alt, newContext);
+						LexerATNConfig c = new LexerATNConfig(config, returnState, newContext);
 						currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState, speculative);
 					}
 				}
@@ -537,7 +537,7 @@ public class LexerATNSimulator extends ATNSimulator {
 					// getEpsilonTarget to return two configurations, so
 					// additional modifications are needed before we can support
 					// the split operation.
-					LexerActionExecutor lexerActionExecutor = LexerActionExecutor.append(config.lexerActionExecutor, atn.lexerActions[((ActionTransition)t).actionIndex]);
+					LexerActionExecutor lexerActionExecutor = LexerActionExecutor.append(config.getLexerActionExecutor(), atn.lexerActions[((ActionTransition)t).actionIndex]);
 					c = new LexerATNConfig(config, t.target, lexerActionExecutor);
 					break;
 				}
@@ -684,7 +684,7 @@ public class LexerATNSimulator extends ATNSimulator {
 
 		if ( firstConfigWithRuleStopState!=null ) {
 			proposed.isAcceptState = true;
-			proposed.lexerActionExecutor = ((LexerATNConfig)firstConfigWithRuleStopState).lexerActionExecutor;
+			proposed.lexerActionExecutor = ((LexerATNConfig)firstConfigWithRuleStopState).getLexerActionExecutor();
 			proposed.prediction = atn.ruleToTokenType[firstConfigWithRuleStopState.state.ruleIndex];
 		}
 

tool/test/org/antlr/v4/test/TestLexerActions.java

@@ -37,6 +37,52 @@ public class TestLexerActions extends BaseTest {
 		assertEquals(expecting, found);
 	}
 
+	/**
+	 * This is a regressing test for antlr/antlr4#469 "Not all internal lexer
+	 * rule actions are executed".
+	 * https://github.com/antlr/antlr4/issues/469
+	 */
+	@Test public void testEvalMultipleActions() throws Exception {
+		String grammar =
+			"lexer grammar L;\n" +
+			"\n" +
+			"@lexer::members\n" +
+			"{\n" +
+			"class Marker\n" +
+			"{\n" +
+			"   Marker (Lexer lexer) { this.lexer = lexer; }\n" +
+			"\n" +
+			"   public String getText ()\n" +
+			"   {\n" +
+			"      return lexer._input.getText (new Interval (start_index, stop_index));\n" +
+			"   }\n" +
+			"\n" +
+			"   public void start ()  { start_index = lexer._input.index (); System.out.println (\"Start:\" + start_index);}\n" +
+			"   public void stop () { stop_index = lexer._input.index (); System.out.println (\"Stop:\" + stop_index);}\n" +
+			"\n" +
+			"   private int start_index = 0;\n" +
+			"   private int stop_index = 0;\n" +
+			"   private Lexer lexer;\n" +
+			"}\n" +
+			"\n" +
+			"Marker m_name = new Marker (this);\n" +
+			"}\n" +
+			"\n" +
+			"HELLO: 'hello' WS { m_name.start (); } NAME { m_name.stop (); } '\\n' { System.out.println (\"Hello: \" + m_name.getText ()); };\n" +
+			"NAME: ('a'..'z' | 'A'..'Z')+ ('\\n')?;\n" +
+			"\n" +
+			"fragment WS: [ \\r\\t\\n]+ ;\n";
+		String found = execLexer("L.g4", grammar, "L", "hello Steve\n");
+		String expecting =
+			"Start:6\n" +
+			"Stop:11\n" +
+			"Hello: Steve\n" +
+			"\n" +
+			"[@0,0:11='hello Steve\\n',<1>,1:0]\n" +
+			"[@1,12:11='<EOF>',<-1>,2:12]\n";
+		assertEquals(expecting, found);
+	}
+
 	@Test public void test2ActionsIn1Rule() throws Exception {
 		String grammar =
 			"lexer grammar L;\n"+