v4: Fix Lexer.reset (and as a side effect fixes Lexer.setInputStream)

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9396]

runtime/Java/src/org/antlr/v4/runtime/Lexer.java

@@ -109,6 +109,14 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator>
 		tokenStartCharPositionInLine = -1;
 		tokenStartLine = -1;
 		text = null;
+
+		hitEOF = false;
+		mode = Lexer.DEFAULT_MODE;
+		if (modeStack != null) {
+			modeStack.clear();
+		}
+
+		_interp.reset();
 	}
 
 	/** Return a token from this source; i.e., match a token on the char

runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java

@@ -55,19 +55,38 @@ public class LexerATNSimulator extends ATNSimulator {
 	 *  simulation. If the ATN simulation fails, we need the DFA to fall
 	 *  back to its previously accepted state, if any. If the ATN succeeds,
 	 *  then the ATN does the accept and the DFA simulator that invoked it
-	 *  can simply return thepredicted token type.
+	 *  can simply return the predicted token type.
 	 */
 	protected static class ExecState {
-		int marker = -1;
 		int index = -1;
 		int line = 0;
 		int charPos = -1;
+
+		void reset() {
+			index = -1;
+			line = 0;
+			charPos = -1;
+		}
 	}
+
 	protected static class DFAExecState extends ExecState {
-		DFAState state = null;
+		DFAState state;
+
+		@Override
+		void reset() {
+			super.reset();
+			state = null;
+		}
 	}
+
 	protected static class ATNExecState extends ExecState {
-		ATNConfig config = null;
+		ATNConfig config;
+
+		@Override
+		void reset() {
+			super.reset();
+			config = null;
+		}
 	}
 
 	@Nullable
@@ -131,18 +150,35 @@ public class LexerATNSimulator extends ATNSimulator {
 		}
 	}
 
+	public void reset() {
+		dfaPrevAccept.reset();
+		atnPrevAccept.reset();
+		textIndex = -1;
+		startIndex = -1;
+		line = 1;
+		charPositionInLine = 0;
+		mode = Lexer.DEFAULT_MODE;
+	}
+
 	// only called from test code from outside
 	public int matchATN(@NotNull CharStream input) {
-		textIndex = -1;
+		int mark = input.mark();
-		startIndex = input.index();
+		try {
-		ATNState startState = atn.modeToStartState.get(mode);
+			textIndex = -1;
-		if ( debug ) System.out.println("mode "+ mode +" start: "+startState);
+			startIndex = input.index();
-		OrderedHashSet<ATNConfig> s0_closure = computeStartState(input, startState);
+			ATNState startState = atn.modeToStartState.get(mode);
-		int old_mode = mode;
+			if ( debug ) System.out.println("mode "+ mode +" start: "+startState);
-		dfa[mode].s0 = addDFAState(s0_closure);
+			OrderedHashSet<ATNConfig> s0_closure = computeStartState(input, startState);
-		int predict = exec(input, s0_closure);
+			int old_mode = mode;
-		if ( debug ) System.out.println("DFA after matchATN: "+dfa[old_mode].toLexerString());
+			dfa[mode].s0 = addDFAState(s0_closure);
-		return predict;
+			int predict = exec(input, s0_closure);
+			if ( debug ) System.out.println("DFA after matchATN: "+dfa[old_mode].toLexerString());
+			return predict;
+		} finally {
+			if (mark >= 0) {
+				input.release(mark);
+			}
+		}
 	}
 
 	protected int exec(@NotNull CharStream input, @NotNull DFAState s0) {
@@ -151,8 +187,7 @@ public class LexerATNSimulator extends ATNSimulator {
 		//System.out.println("DFA start of execDFA: "+dfa[mode].toLexerString());
 		textIndex = -1;
 		startIndex = input.index();
-		resetPrevAccept(dfaPrevAccept);
+		dfaPrevAccept.reset();
-		dfaPrevAccept.state = null;
 		LexerNoViableAltException atnException = null;
 		DFAState s = s0;
 		int t = input.LA(1);
@@ -213,8 +248,7 @@ public class LexerATNSimulator extends ATNSimulator {
 
 		@NotNull
 		OrderedHashSet<ATNConfig> reach = new OrderedHashSet<ATNConfig>();
-		resetPrevAccept(atnPrevAccept);
+		atnPrevAccept.reset();
-		atnPrevAccept.config = null;
 
 		int t = input.LA(1);
 
@@ -319,7 +353,6 @@ public class LexerATNSimulator extends ATNSimulator {
 
 		// seek to after last char in token
 		input.seek(prevAccept.index);
-		input.release(prevAccept.marker);
 		line = prevAccept.line;
 		charPositionInLine = prevAccept.charPos;
 		consume(input);
@@ -472,19 +505,11 @@ public class LexerATNSimulator extends ATNSimulator {
 	}
 
 	protected void markAcceptState(@NotNull ExecState state, @NotNull CharStream input) {
-		state.marker = input.mark();
 		state.index = input.index();
 		state.line = line;
 		state.charPos = charPositionInLine;
 	}
 
-	protected void resetPrevAccept(@NotNull ExecState prevAccept) {
-		prevAccept.marker = -1;
-		prevAccept.index = -1;
-		prevAccept.line = 0;
-		prevAccept.charPos = -1;
-	}
-
 	protected void addDFAEdge(@NotNull OrderedHashSet<ATNConfig> p,
 							  int t,
 							  @NotNull OrderedHashSet<ATNConfig> q)