useError recovery said notThe set that follows a loop for sync()

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9264]

runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java

@@ -82,16 +82,22 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
 	 */
 	public Object match(int ttype) throws RecognitionException {
 //		System.out.println("match "+((TokenStream)input).LT(1)+" vs expected "+ttype);
-		Object matchedSymbol = getCurrentInputSymbol();
+		Object currentSymbol = getCurrentInputSymbol();
 		if ( getInputStream().LA(1)==ttype ) {
 			_errHandler.endErrorCondition(this);
 			consume();
 		}
 		else {
-			matchedSymbol = _errHandler.recoverInline(this);
+			currentSymbol = _errHandler.recoverInline(this);
+			if ( buildParseTrees && currentSymbol instanceof Token &&
+			     ((Token)currentSymbol).getTokenIndex()==-1 )
+			{
+				// we must have conjured up a new token during single token insertion
+				// if it's not the current symbol
+				_ctx.addErrorNode((Token)currentSymbol);
+			}
 		}
-//		if ( buildParseTrees ) _ctx.addChild((Token)matchedSymbol);
+		return currentSymbol;
-		return matchedSymbol;
 	}
 
 	/** Track the RuleContext objects during the parse and hook them up

runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java

@@ -173,8 +173,9 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 //			System.err.println("at loop back: "+s.getClass().getSimpleName());
 			reportUnwantedToken(recognizer);
 			consumeUntil(recognizer, expecting);
+//			consumeUntil(recognizer, getErrorRecoverySet(recognizer));
 		}
-		// do nothing if we can identify the exact kind of ATN state
+		// do nothing if we can't identify the exact kind of ATN state
 	}
 
 	public void reportNoViableAlternative(BaseRecognizer recognizer,
@@ -266,24 +267,6 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 	public Object recoverInline(BaseRecognizer recognizer)
 		throws RecognitionException
 	{
-		// if next token is what we are looking for then "delete" this token
-//		int nextTokenType = recognizer.getInputStream().LA(2);
-//		IntervalSet expecting = getExpectedTokens(recognizer);
-//		if ( expecting.contains(nextTokenType) ) {
-//			reportUnwantedToken(recognizer);
-//			/*
-//			System.err.println("recoverFromMismatchedToken deleting "+
-//							   ((TokenStream)recognizer.getInputStream()).LT(1)+
-//							   " since "+((TokenStream)recognizer.getInputStream()).LT(2)+
-//							   " is what we want");
-//			*/
-//			recognizer.consume(); // simply delete extra token
-//			// we want to return the token we're actually matching
-//			Object matchedSymbol = recognizer.getCurrentInputSymbol();
-//			endErrorCondition(recognizer);  // we know next token is correct
-//			recognizer.consume(); // move past ttype token as if all were ok
-//			return matchedSymbol;
-//		}
 		// SINGLE TOKEN DELETION
 		Object matchedSymbol = singleTokenDeletion(recognizer);
 		if ( matchedSymbol!=null ) return matchedSymbol;
@@ -369,6 +352,7 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 		t.charPositionInLine = current.getCharPositionInLine();
 		t.channel = Token.DEFAULT_CHANNEL;
 		t.source = current.getTokenSource();
+		t.index = -1; // indicate we conjured this up because it has no index
 		return t;
 	}
 

runtime/Java/src/org/antlr/v4/runtime/Token.java

@@ -84,6 +84,9 @@ public interface Token {
 	/** An index from 0..n-1 of the token object in the input stream.
 	 *  This must be valid in order to print token streams,
 	 *  use TokenRewriteStream, and generally deal with ASTs.
+	 *
+	 *  Return -1 to indicate that this token was conjured up since
+	 *  it doesn't have a valid index.
 	 */
 	int getTokenIndex();