got sync in

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9101]

runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java

@@ -52,7 +52,12 @@ public interface ANTLRErrorStrategy {
 	 */
 	void recover(BaseRecognizer recognizer);
 
-	/** Implement Jim Idle's magic sync mechanism in closures and optional
+	/** Make sure that the current lookahead symbol is consistent with
+	 *  what were expecting at this point in the ATN. You can call this
+	 *  anytime but ANTLR only generates code to check before loops
+	 *  and each iteration.
+	 *
+	 *  Implements Jim Idle's magic sync mechanism in closures and optional
 	 *  subrules. E.g.,
 	 *
 	 * 		a : sync ( stuff sync )* ;
@@ -80,5 +85,5 @@ public interface ANTLRErrorStrategy {
 	 *  when it matches a valid token (indicating no longer in recovery mode)
 	 *  and from its own reset method.
 	 */
-	void reset();
+	void endErrorCondition();
 }

runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java

@@ -58,7 +58,7 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
 	 *  matched a token.  Prevents generation of more than one error message
 	 *  per error.
 	 */
-	protected boolean errorRecovery = false;
+//	protected boolean errorRecovery = false;
 
 	/** Did the recognizer encounter a syntax error?  Track how many. */
 	protected int syntaxErrors = 0;
@@ -70,8 +70,8 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
 	/** reset the parser's state */
 	public void reset() {
 		if ( getInputStream()!=null ) getInputStream().seek(0);
-		_errHandler.reset();
+		_errHandler.endErrorCondition();
-		errorRecovery = false;
+//		errorRecovery = false;
 		_ctx = null;
 	}
 
@@ -92,8 +92,8 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
 		Object matchedSymbol = getCurrentInputSymbol();
 		if ( getInputStream().LA(1)==ttype ) {
 			getInputStream().consume();
-			errorRecovery = false;
+//			errorRecovery = false;
-			_errHandler.reset();
+			_errHandler.endErrorCondition();
 			if ( buildParseTrees ) _ctx.addChild((Token)matchedSymbol);
 			return matchedSymbol;
 		}
@@ -102,8 +102,8 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
 
 	/** Match the wildcard: in a symbol */
 	public void matchAny() {
-		errorRecovery = false;
+//		errorRecovery = false;
-		_errHandler.reset();
+		_errHandler.endErrorCondition();
 		getInputStream().consume();
 	}
 
@@ -143,32 +143,6 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
 		return traceATNStates;
 	}
 
-	/** Report a recognition problem.
-	 *
-	 *  This method sets errorRecovery to indicate the parser is recovering
-	 *  not parsing.  Once in recovery mode, no errors are generated.
-	 *  To get out of recovery mode, the parser must successfully match
-	 *  a token (after a resync).  So it will go:
-	 *
-	 * 		1. error occurs
-	 * 		2. enter recovery mode, report error
-	 * 		3. consume until token found in resynch set
-	 * 		4. try to resume parsing
-	 * 		5. next match() will reset errorRecovery mode
-	 */
-	public void reportError(RecognitionException e) {
-		// if we've already reported an error and have not matched a token
-		// yet successfully, don't report any errors.
-		if ( errorRecovery ) {
-			//System.err.print("[SPURIOUS] ");
-			return;
-		}
-		syntaxErrors++; // don't count spurious
-		errorRecovery = true;
-
-		notifyListeners(e.offendingToken, e.getMessage(), e);
-	}
-
 	/** Get number of recognition errors (lexer, parser, tree parser).  Each
 	 *  recognizer tracks its own number.  So parser and lexer each have
 	 *  separate count.  Does not count the spurious errors found between

runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java

@@ -23,8 +23,12 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 
 	protected IntervalSet lastErrorStates;
 
+	protected void beginErrorCondition() {
+		errorRecovery = true;
+	}
+
 	@Override
-	public void reset() {
+	public void endErrorCondition() {
 		errorRecovery = false;
 		lastErrorStates = null;
 		lastErrorIndex = -1;
@@ -35,6 +39,11 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 							RecognitionException e)
 		throws RecognitionException
 	{
+		// if we've already reported an error and have not matched a token
+		// yet successfully, don't report any errors.
+		if ( errorRecovery ) return; // don't count spurious errors
+		recognizer.syntaxErrors++;
+		beginErrorCondition();
 		if ( e instanceof NoViableAltException ) {
 			reportNoViableAlternative(recognizer, (NoViableAltException) e);
 		}
@@ -69,22 +78,36 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 		lastErrorIndex = recognizer.getInputStream().index();
 		if ( lastErrorStates==null ) lastErrorStates = new IntervalSet();
 		lastErrorStates.add(recognizer._ctx.s);
-		IntervalSet followSet = computeErrorRecoverySet(recognizer);
+		IntervalSet followSet = getErrorRecoverySet(recognizer);
 		consumeUntil(recognizer, followSet);
 	}
 
+	/** Make sure that the current lookahead symbol is consistent with
+	 *  what were expecting at this point in the ATN.
+ 	 */
 	@Override
 	public void sync(BaseRecognizer recognizer) {
+		System.out.println("sync");
+		// TODO: CACHE THESE RESULTS!!
+		IntervalSet expecting = getExpectedTokens(recognizer);
+		// TODO: subclass this class for treeparsers
+		TokenStream tokens = (TokenStream)recognizer.getInputStream();
+		Token la = tokens.LT(1);
+		if ( expecting.contains(la.getType()) ) {
+			endErrorCondition();
+			return;
+		}
+		recoverInline(recognizer);
 	}
 
 	public void reportNoViableAlternative(BaseRecognizer recognizer,
 										  NoViableAltException e)
 		throws RecognitionException
 	{
-		if ( recognizer.errorRecovery ) return;
+		// TODO: subclass this class for treeparsers
-		trackError(recognizer);
+		TokenStream tokens = (TokenStream)recognizer.getInputStream();
-
+		String input = tokens.toString(e.startToken, e.offendingToken);
-		String msg = "no viable alternative at input "+getTokenErrorDisplay(e.offendingToken);
+		String msg = "no viable alternative at input "+escapeWSAndQuote(input);
 		recognizer.notifyListeners(e.offendingToken, msg, e);
 	}
 
@@ -92,9 +115,6 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 									InputMismatchException e)
 		throws RecognitionException
 	{
-		if ( recognizer.errorRecovery ) return;
-		trackError(recognizer);
-
 		String msg = "mismatched input "+getTokenErrorDisplay(e.offendingToken)+
 		" expecting "+e.getExpectedTokens().toString(recognizer.getTokenNames());
 		recognizer.notifyListeners(e.offendingToken, msg, e);
@@ -104,9 +124,6 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 									  FailedPredicateException e)
 		throws RecognitionException
 	{
-		if ( recognizer.errorRecovery ) return;
-		trackError(recognizer);
-
 		String ruleName = recognizer.getRuleNames()[recognizer._ctx.getRuleIndex()];
 		String msg = "rule "+ruleName+" failed predicate: {"+
 		e.predicateText+"}?";
@@ -114,11 +131,11 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 	}
 
 	public void reportUnwantedToken(BaseRecognizer recognizer) {
-		if ( recognizer.errorRecovery ) return;
+		if ( errorRecovery ) return;
-		trackError(recognizer);
+		recognizer.syntaxErrors++;
+		beginErrorCondition();
 
 		Token t = (Token)recognizer.getCurrentInputSymbol();
-
 		String tokenName = getTokenErrorDisplay(t);
 		IntervalSet expecting = getExpectedTokens(recognizer);
 		String msg = "extraneous input "+tokenName+" expecting "+
@@ -127,8 +144,9 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 	}
 
 	public void reportMissingToken(BaseRecognizer recognizer) {
-		if ( recognizer.errorRecovery ) return;
+		if ( errorRecovery ) return;
-		trackError(recognizer);
+		recognizer.syntaxErrors++;
+		beginErrorCondition();
 
 		Token t = (Token)recognizer.getCurrentInputSymbol();
 		IntervalSet expecting = getExpectedTokens(recognizer);
@@ -268,6 +286,10 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 				s = "<"+t.getType()+">";
 			}
 		}
+		return escapeWSAndQuote(s);
+	}
+
+	protected String escapeWSAndQuote(String s) {
 		s = s.replaceAll("\n","\\\\n");
 		s = s.replaceAll("\r","\\\\r");
 		s = s.replaceAll("\t","\\\\t");
@@ -366,7 +388,7 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 	 *  Like Grosch I implement context-sensitive FOLLOW sets that are combined
 	 *  at run-time upon error to avoid overhead during parsing.
 	 */
-	protected IntervalSet computeErrorRecoverySet(BaseRecognizer recognizer) {
+	protected IntervalSet getErrorRecoverySet(BaseRecognizer recognizer) {
 		ATN atn = recognizer._interp.atn;
 		RuleContext ctx = recognizer._ctx;
 		IntervalSet recoverSet = new IntervalSet();
@@ -393,9 +415,9 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 		}
 	}
 
-	protected void trackError(BaseRecognizer recognizer) {
+//	protected void trackError(BaseRecognizer recognizer) {
-		recognizer.syntaxErrors++;
+//		recognizer.syntaxErrors++;
-		recognizer.errorRecovery = true;
+//		recognizer.errorRecovery = true;
-	}
+//	}
 
 }

runtime/Java/src/org/antlr/v4/runtime/tree/TreeParser.java

@@ -106,8 +106,7 @@ public class TreeParser extends BaseRecognizer {
 	 *  corresponding UP node.
 	 */
 	public void matchAny(IntStream ignore) { // ignore stream, copy of input
-		errorRecovery = false;
+		_errHandler.endErrorCondition();
-//		failed = false;
 		Object look = _input.LT(1);
 		if ( _input.getTreeAdaptor().getChildCount(look)==0 ) {
 			_input.consume(); // not subtree, consume 1 node and return

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -218,6 +218,7 @@ CodeBlockForAlt(c, locals, preamble, ops) ::= <<
 >>
 
 LL1AltBlock(choice, preamble, alts, error) ::= <<
+setState(<choice.stateNumber>);
 <if(choice.label)><labelref(choice.label)> = _input.LT(1);<endif>
 <preamble; separator="\n">
 switch ( _input.LA(1) ) {
@@ -230,6 +231,7 @@ switch ( _input.LA(1) ) {
 >>
 
 LL1OptionalBlock(choice, alts, error) ::= <<
+setState(<choice.stateNumber>);
 switch ( _input.LA(1) ) {
 	<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
 	<alt>
@@ -240,6 +242,7 @@ switch ( _input.LA(1) ) {
 >>
 
 LL1OptionalBlockSingleAlt(choice, expr, alts, preamble, error, followExpr) ::= <<
+setState(<choice.stateNumber>);
 <preamble; separator="\n">
 if ( <expr> ) {
     <alts; separator="\n">
@@ -247,7 +250,9 @@ if ( <expr> ) {
 <!else if ( !(<followExpr>) ) <error>!>
 >>
 
-LL1StarBlock(choice, alts, sync) ::= <<
+LL1StarBlock(choice, alts) ::= <<
+setState(<choice.stateNumber>);
+_errHandler.sync(this);
 <choice.loopLabel>:
 while (true) {
 	switch ( _input.LA(1) ) {
@@ -257,21 +262,26 @@ while (true) {
 		<cases(choice.exitLook)>
 		    break <choice.loopLabel>;
 	}
-    //<sync>
+	setState(<choice.stateNumber>);
+    _errHandler.sync(this);
 }
 >>
 
-LL1StarBlockSingleAlt(choice, loopExpr, alts, preamble, iteration, sync) ::= <<
+LL1StarBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= <<
+setState(<choice.stateNumber>);
+_errHandler.sync(this);
 <preamble; separator="\n">
 while ( <loopExpr> ) {
     <alts; separator="\n">
+	setState(<choice.stateNumber>);
+	_errHandler.sync(this);
     <iteration>
-    //<sync>
 }
 >>
 
-LL1PlusBlock(choice, alts, iteration, loopExpr, sync, error, iterationSync) ::= <<
+LL1PlusBlock(choice, alts, iteration, loopExpr, error) ::= <<
-//<sync>
+setState(<choice.stateNumber>);
+_errHandler.sync(this);
 do {
 	switch ( _input.LA(1) ) {
 		<choice.altLook,alts:{look,alt| <cases(look)>
@@ -280,26 +290,28 @@ do {
         default :
             <error>
 	}
+	setState(<choice.stateNumber>);
+	_errHandler.sync(this);
     <iteration>
-    //<iterationSync>
 } while ( <loopExpr> );
 >>
 
-LL1PlusBlockSingleAlt(choice, loopExpr, alts, preamble, iteration,
+LL1PlusBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= <<
-                      sync, iterationSync) ::=
+setState(<choice.stateNumber>);
-<<
+_errHandler.sync(this);
-//<sync>
 <preamble; separator="\n">
 do {
     <alts; separator="\n">
+	setState(<choice.stateNumber>);
+	_errHandler.sync(this);
     <iteration>
-//    <iterationSync>
 } while ( <loopExpr> );
 >>
 
 // LL(*) stuff
 
 AltBlock(choice, preamble, alts, error) ::= <<
+setState(<choice.stateNumber>);
 <if(choice.label)><labelref(choice.label)> = _input.LT(1);<endif>
 <preamble; separator="\n">
 switch ( _interp.adaptivePredict(_input,<choice.decision>,_ctx) ) {
@@ -311,6 +323,7 @@ case <i>:
 >>
 
 OptionalBlock(choice, alts, error) ::= <<
+setState(<choice.stateNumber>);
 switch ( _interp.adaptivePredict(_input,<choice.decision>,_ctx) ) {
 	<alts:{alt |
 case <i>:
@@ -322,6 +335,8 @@ case <i>:
 // TODO: we we need uniqueID? a single _alt might work
 
 StarBlock(choice, alts, sync) ::= <<
+setState(<choice.stateNumber>);
+_errHandler.sync(this);
 int _alt<choice.uniqueID> = _interp.adaptivePredict(_input,<choice.decision>,_ctx);
 while ( _alt<choice.uniqueID>!=<choice.exitAlt> && _alt<choice.uniqueID>!=-1 ) {
 	switch ( _alt<choice.uniqueID> ) {
@@ -330,11 +345,15 @@ case <i>:
 	<alt>
 	break;}; separator="\n">
 	}
+	setState(<choice.stateNumber>);
+	_errHandler.sync(this);
 	_alt<choice.uniqueID> = _interp.adaptivePredict(_input,<choice.decision>,_ctx);
 }
 >>
 
 PlusBlock(choice, alts, error) ::= <<
+setState(<choice.stateNumber>);
+_errHandler.sync(this);
 int _alt<choice.uniqueID> = _interp.adaptivePredict(_input,<choice.decision>,_ctx);
 do {
 	switch ( _alt<choice.uniqueID> ) {
@@ -345,6 +364,8 @@ case <i>:
 	    default :
 		    <error>
 	}
+	setState(<choice.stateNumber>);
+	_errHandler.sync(this);
 	_alt<choice.uniqueID> = _interp.adaptivePredict(_input,<choice.decision>,_ctx);
 } while ( _alt<choice.uniqueID>!=<choice.exitAlt> && _alt<choice.uniqueID>!=-1 );
 >>

tool/src/org/antlr/v4/codegen/model/LL1Loop.java

@@ -39,7 +39,6 @@ import java.util.*;
 public abstract class LL1Loop extends Choice {
 	@ModelElement public OutputModelObject loopExpr;
 	@ModelElement public List<SrcOp> iteration;
-	@ModelElement public Sync sync;
 
 	public LL1Loop(OutputModelFactory factory,
 				   GrammarAST blkAST,

tool/src/org/antlr/v4/codegen/model/LL1PlusBlock.java

@@ -41,7 +41,6 @@ public class LL1PlusBlock extends LL1Loop {
 	/** Token names for each alt 0..n-1 */
 	public List<String[]> altLook;
 
-	@ModelElement public Sync iterationSync;
 	public String loopLabel;
 	public String loopCounterVar;
 	public String[] exitLook;
@@ -73,10 +72,5 @@ public class LL1PlusBlock extends LL1Loop {
 		IntervalSet exitLookSet = altLookSets[altLookSets.length-1];
 		this.exitLook = gen.target.getTokenTypesAsTargetLabels(g,
 															   exitLookSet.toArray());
-
-		//IntervalSet iterationExpected = (IntervalSet)loopBackLook.or(exitLookSet);
-//		this.sync = new Sync(factory, plusRoot, loopBackLook, decision, "enter");
-//		this.iterationSync = new Sync(factory, plusRoot, iterationExpected, decision, "iter");
-//		this.earlyExitError = new ThrowEarlyExitException(factory, plusRoot, null);
 	}
 }

tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java

@@ -38,21 +38,14 @@ import java.util.List;
 
 /** */
 public class LL1PlusBlockSingleAlt extends LL1Loop {
-	@ModelElement public Sync iterationSync;
-
 	public LL1PlusBlockSingleAlt(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlockForAlt> alts) {
 		super(factory, blkAST, alts);
 
 		PlusBlockStartState plus = (PlusBlockStartState)blkAST.atnState;
 		this.decision = plus.loopBackState.decision;
 		IntervalSet[] altLookSets = factory.getGrammar().decisionLOOK.get(decision);
-		IntervalSet exitLook = altLookSets[altLookSets.length-1];
 
 		IntervalSet loopBackLook = altLookSets[1];
 		loopExpr = addCodeForLoopLookaheadTempVar(loopBackLook);
-
-		this.sync = new Sync(factory, blkAST, loopBackLook, decision, "enter");
-		IntervalSet iterationExpected = (IntervalSet) loopBackLook.or(exitLook);
-		iterationSync = new Sync(factory, blkAST, iterationExpected, decision, "iter");
 	}
 }

tool/src/org/antlr/v4/codegen/model/LL1StarBlock.java

@@ -46,7 +46,6 @@ public class LL1StarBlock extends LL1Loop {
 	public LL1StarBlock(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlockForAlt> alts) {
 		super(factory, blkAST, alts);
 
-//		StarBlockStartState blkStart = (StarBlockStartState)blkAST.atnState;
 		StarLoopEntryState star = (StarLoopEntryState)blkAST.atnState;
 		this.decision = star.decision;
 
@@ -61,7 +60,5 @@ public class LL1StarBlock extends LL1Loop {
 
 		this.exitLook =
 			factory.getGenerator().target.getTokenTypesAsTargetLabels(factory.getGrammar(), lastLook.toArray());
-
-//		this.sync = new Sync(factory, blkAST, expecting, decision, "iter");
 	}
 }

tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java

@@ -48,8 +48,5 @@ public class LL1StarBlockSingleAlt extends LL1Loop {
 		IntervalSet enterLook = altLookSets[1];
 		IntervalSet exitLook = altLookSets[2];
 		loopExpr = addCodeForLoopLookaheadTempVar(enterLook);
-
-		IntervalSet enterExpecting = (IntervalSet)exitLook.or(enterLook);
-		this.sync = new Sync(factory, starRoot, enterExpecting, decision, "iter");
 	}
 }

tool/src/org/antlr/v4/codegen/model/StarBlock.java

@@ -44,7 +44,6 @@ public class StarBlock extends Loop {
 	{
 		super(factory, blkOrEbnfRootAST, alts);
 		loopLabel = factory.getGenerator().target.getLoopLabel(blkOrEbnfRootAST);
-//		BlockStartState star = (BlockStartState)blkOrEbnfRootAST.atnState;
 		StarLoopEntryState star = (StarLoopEntryState)blkOrEbnfRootAST.atnState;
 		decision = star.decision;
 		exitAlt = alts.size()+1;

tool/test/org/antlr/v4/test/TestParseErrors.java

@@ -108,7 +108,7 @@ public class TestParseErrors extends BaseTest {
 			";\n" +
 			"q : 'e' ;\n";
 		String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "ae", false);
-		String expecting = "line 1:1 no viable alternative at input 'e'\n";
+		String expecting = "line 1:1 no viable alternative at input 'ae'\n";
 		String result = stderrDuringParse;
 		assertEquals(expecting, result);
 	}
@@ -122,7 +122,7 @@ public class TestParseErrors extends BaseTest {
 			"q : 'e' ;\n";
 		System.out.println(grammar);
 		String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "abe", false);
-		String expecting = "line 1:2 no viable alternative at input 'e'\n";
+		String expecting = "line 1:2 no viable alternative at input 'abe'\n";
 		String result = stderrDuringParse;
 		assertEquals(expecting, result);
 	}
@@ -135,11 +135,22 @@ public class TestParseErrors extends BaseTest {
 			";\n" +
 			"q : 'e' ;\n";
 		String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "aaae", false);
-		String expecting = "line 1:3 no viable alternative at input 'e'\n";
+		String expecting = "line 1:3 no viable alternative at input 'aaae'\n";
 		String result = stderrDuringParse;
 		assertEquals(expecting, result);
 	}
 
+	@Test public void testSingleTokenDeletionBeforeLoop() throws Exception {
+		String grammar =
+			"grammar T;\n" +
+			"a : 'a' 'b'*;";
+		String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "aabc", false);
+		String expecting = "line 1:1 extraneous input 'a' expecting {<EOF>, 'b'}\n";
+		String result = stderrDuringParse;
+		assertEquals(expecting, result);
+	}
+
+
 	@Test public void testLL1ErrorInfo() throws Exception {
 		String grammar =
 			"grammar T;\n" +