Merge branch 'master' into main

runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java

@@ -102,6 +102,7 @@ public abstract class ATNSimulator {
 		// STATES
 		//
 		List<Pair<LoopEndState, Integer>> loopBackStateNumbers = new ArrayList<Pair<LoopEndState, Integer>>();
+		List<Pair<BlockStartState, Integer>> endStateNumbers = new ArrayList<Pair<BlockStartState, Integer>>();
 		int nstates = toInt(data[p++]);
 		for (int i=1; i<=nstates; i++) {
 			int stype = toInt(data[p++]);
@@ -116,14 +117,22 @@ public abstract class ATNSimulator {
 				int loopBackStateNumber = toInt(data[p++]);
 				loopBackStateNumbers.add(new Pair<LoopEndState, Integer>((LoopEndState)s, loopBackStateNumber));
 			}
+			else if (s instanceof BlockStartState) {
+				int endStateNumber = toInt(data[p++]);
+				endStateNumbers.add(new Pair<BlockStartState, Integer>((BlockStartState)s, endStateNumber));
+			}
 			atn.addState(s);
 		}
 
-		// delay the assignment of loop back states until we know all the state instances have been initialized
+		// delay the assignment of loop back and end states until we know all the state instances have been initialized
 		for (Pair<LoopEndState, Integer> pair : loopBackStateNumbers) {
 			pair.a.loopBackState = atn.states.get(pair.b);
 		}
 
+		for (Pair<BlockStartState, Integer> pair : endStateNumbers) {
+			pair.a.endState = (BlockEndState)atn.states.get(pair.b);
+		}
+
 		//
 		// RULES
 		//
@@ -145,6 +154,17 @@ public abstract class ATNSimulator {
 			}
 		}
 
+		atn.ruleToStopState = new RuleStopState[nrules];
+		for (ATNState state : atn.states) {
+			if (!(state instanceof RuleStopState)) {
+				continue;
+			}
+
+			RuleStopState stopState = (RuleStopState)state;
+			atn.ruleToStopState[state.ruleIndex] = stopState;
+			atn.ruleToStartState[state.ruleIndex].stopState = stopState;
+		}
+
 		//
 		// MODES
 		//
@@ -190,6 +210,41 @@ public abstract class ATNSimulator {
 			p += 6;
 		}
 
+		if (atn.grammarType == ATN.LEXER) {
+			for (ATNState state : atn.states) {
+				for (int i = 0; i < state.getNumberOfTransitions(); i++) {
+					Transition t = state.transition(i);
+					if (!(t instanceof RuleTransition)) {
+						continue;
+					}
+
+					RuleTransition ruleTransition = (RuleTransition)t;
+					atn.ruleToStopState[ruleTransition.target.ruleIndex].addTransition(new EpsilonTransition(ruleTransition.followState));
+				}
+			}
+		}
+
+		for (ATNState state : atn.states) {
+			if (state instanceof PlusLoopbackState) {
+				PlusLoopbackState loopbackState = (PlusLoopbackState)state;
+				for (int i = 0; i < loopbackState.getNumberOfTransitions(); i++) {
+					ATNState target = loopbackState.transition(i).target;
+					if (target instanceof PlusBlockStartState) {
+						((PlusBlockStartState)target).loopBackState = loopbackState;
+					}
+				}
+			}
+			else if (state instanceof StarLoopbackState) {
+				StarLoopbackState loopbackState = (StarLoopbackState)state;
+				for (int i = 0; i < loopbackState.getNumberOfTransitions(); i++) {
+					ATNState target = loopbackState.transition(i).target;
+					if (target instanceof StarLoopEntryState) {
+						((StarLoopEntryState)target).loopBackState = loopbackState;
+					}
+				}
+			}
+		}
+
 		//
 		// DECISIONS
 		//
@@ -203,9 +258,56 @@ public abstract class ATNSimulator {
 			decState.isGreedy = isGreedy==1;
 		}
 
+		verifyATN(atn);
 		return atn;
 	}
 
+	private static void verifyATN(ATN atn) {
+		// verify assumptions
+		for (ATNState state : atn.states) {
+			if (state == null) {
+				continue;
+			}
+
+			if (state instanceof PlusBlockStartState) {
+				if (((PlusBlockStartState)state).loopBackState == null) {
+					throw new IllegalStateException();
+				}
+			}
+
+			if (state instanceof StarLoopEntryState) {
+				if (((StarLoopEntryState)state).loopBackState == null) {
+					throw new IllegalStateException();
+				}
+			}
+
+			if (state instanceof LoopEndState) {
+				if (((LoopEndState)state).loopBackState == null) {
+					throw new IllegalStateException();
+				}
+			}
+
+			if (state instanceof RuleStartState) {
+				if (((RuleStartState)state).stopState == null) {
+					throw new IllegalStateException();
+				}
+			}
+
+			if (state instanceof BlockStartState) {
+				if (((BlockStartState)state).endState == null) {
+					throw new IllegalStateException();
+				}
+			}
+
+			if (state instanceof DecisionState) {
+				DecisionState decisionState = (DecisionState)state;
+				if (decisionState.getNumberOfTransitions() > 1 && decisionState.decision < 0) {
+					throw new IllegalStateException();
+				}
+			}
+		}
+	}
+
 	public static int toInt(char c) {
 		return c==65535 ? -1 : c;
 	}

runtime/Java/src/org/antlr/v4/runtime/atn/TokensStartState.java

@@ -30,5 +30,5 @@
 package org.antlr.v4.runtime.atn;
 
 /** The Tokens rule start state linking to each lexer rule start state */
-public class TokensStartState extends BlockStartState {
+public class TokensStartState extends DecisionState {
 }

tool/src/org/antlr/v4/automata/ATNSerializer.java

@@ -100,7 +100,12 @@ public class ATNSerializer {
 			}
 			data.add(s.getStateType());
 			data.add(s.ruleIndex);
-			if ( s.getStateType() == ATNState.LOOP_END ) data.add(((LoopEndState)s).loopBackState.stateNumber);
+			if ( s.getStateType() == ATNState.LOOP_END ) {
+				data.add(((LoopEndState)s).loopBackState.stateNumber);
+			}
+			else if ( s instanceof BlockStartState ) {
+				data.add(((BlockStartState)s).endState.stateNumber);
+			}
 			nedges += s.getNumberOfTransitions();
 			for (int i=0; i<s.getNumberOfTransitions(); i++) {
 				Transition t = s.transition(i);
@@ -223,6 +228,10 @@ public class ATNSerializer {
 				int loopBackStateNumber = ATNSimulator.toInt(data[p++]);
 				arg = " "+loopBackStateNumber;
 			}
+			else if ( stype == ATNState.PLUS_BLOCK_START || stype == ATNState.STAR_BLOCK_START ) {
+				int endStateNumber = ATNSimulator.toInt(data[p++]);
+				arg = " "+endStateNumber;
+			}
 			buf.append(i - 1).append(":")
 				.append(ATNState.serializationNames.get(stype)).append(" ")
 				.append(ruleIndex).append(arg).append("\n");