diff --git a/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java b/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java index e52f401d0..3d30ada59 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java @@ -82,16 +82,22 @@ public abstract class BaseRecognizer extends Recognizer { */ public Object match(int ttype) throws RecognitionException { // System.out.println("match "+((TokenStream)input).LT(1)+" vs expected "+ttype); - Object matchedSymbol = getCurrentInputSymbol(); + Object currentSymbol = getCurrentInputSymbol(); if ( getInputStream().LA(1)==ttype ) { _errHandler.endErrorCondition(this); consume(); } else { - matchedSymbol = _errHandler.recoverInline(this); + currentSymbol = _errHandler.recoverInline(this); + if ( buildParseTrees && currentSymbol instanceof Token && + ((Token)currentSymbol).getTokenIndex()==-1 ) + { + // we must have conjured up a new token during single token insertion + // if it's not the current symbol + _ctx.addErrorNode((Token)currentSymbol); + } } -// if ( buildParseTrees ) _ctx.addChild((Token)matchedSymbol); - return matchedSymbol; + return currentSymbol; } /** Track the RuleContext objects during the parse and hook them up diff --git a/runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java b/runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java index d0cd79bda..62ebf0fc5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java +++ b/runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java @@ -173,8 +173,9 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { // System.err.println("at loop back: "+s.getClass().getSimpleName()); reportUnwantedToken(recognizer); consumeUntil(recognizer, expecting); +// consumeUntil(recognizer, getErrorRecoverySet(recognizer)); } - // do nothing if we can identify the exact kind of ATN state + // do nothing if we can't identify the exact kind of ATN state } public void reportNoViableAlternative(BaseRecognizer recognizer, @@ -266,24 +267,6 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { public Object recoverInline(BaseRecognizer recognizer) throws RecognitionException { - // if next token is what we are looking for then "delete" this token -// int nextTokenType = recognizer.getInputStream().LA(2); -// IntervalSet expecting = getExpectedTokens(recognizer); -// if ( expecting.contains(nextTokenType) ) { -// reportUnwantedToken(recognizer); -// /* -// System.err.println("recoverFromMismatchedToken deleting "+ -// ((TokenStream)recognizer.getInputStream()).LT(1)+ -// " since "+((TokenStream)recognizer.getInputStream()).LT(2)+ -// " is what we want"); -// */ -// recognizer.consume(); // simply delete extra token -// // we want to return the token we're actually matching -// Object matchedSymbol = recognizer.getCurrentInputSymbol(); -// endErrorCondition(recognizer); // we know next token is correct -// recognizer.consume(); // move past ttype token as if all were ok -// return matchedSymbol; -// } // SINGLE TOKEN DELETION Object matchedSymbol = singleTokenDeletion(recognizer); if ( matchedSymbol!=null ) return matchedSymbol; @@ -369,6 +352,7 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { t.charPositionInLine = current.getCharPositionInLine(); t.channel = Token.DEFAULT_CHANNEL; t.source = current.getTokenSource(); + t.index = -1; // indicate we conjured this up because it has no index return t; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/Token.java b/runtime/Java/src/org/antlr/v4/runtime/Token.java index 6127b8bd9..750cc52d8 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Token.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Token.java @@ -84,6 +84,9 @@ public interface Token { /** An index from 0..n-1 of the token object in the input stream. * This must be valid in order to print token streams, * use TokenRewriteStream, and generally deal with ASTs. + * + * Return -1 to indicate that this token was conjured up since + * it doesn't have a valid index. */ int getTokenIndex();