refactored sync/recoverInLine. figured out exactly what sync does at start of alt. setState goes to loopBack now in loops

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9177]

runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java

@@ -190,7 +190,7 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
 	 *  If the parser is creating parse trees, the current symbol
 	 *  would also be added as a child to the current context (node).
 	 */
-	protected Object consume() {
+	public Object consume() {
 		Object o = getCurrentInputSymbol();
 		getInputStream().consume();
 		if ( buildParseTrees ) {
@@ -227,6 +227,10 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
 		return null;
 	}
 
+	public ParserRuleContext getContext() {
+		return _ctx;
+	}
+
 	public boolean inContext(String context) {
 		// TODO: useful in parser?
 		return false;

runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java

@@ -129,27 +129,52 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 
 	/** Make sure that the current lookahead symbol is consistent with
 	 *  what were expecting at this point in the ATN.
-	 *  sync() differs fundamentally from the recoverInline() method.
+	 *
-	 *  In this case, we throw out a token that's not in the set of what
+	 *  At the start of a sub rule upon error, sync() performs single
-	 *  were expecting at this point. recoverInline() only deletes this
+	 *  token deletion, if possible. If it can't do that, it bails
-	 *  token if LT(2) (token after the current token) is what were expecting;
+	 *  on the current rule and uses the default error recovery,
-	 *  i.e., we have an extra token sitting on the input stream.  sync()
+	 *  which consumes until the resynchronization set of the current rule.
-	 *  simply consumes until it finds something that can start whatever
+	 *
-	 *  follows the call to sync().
+	 *  If the sub rule is optional, ()? or ()* or optional alternative,
+	 *  then the expected set includes what follows the subrule.
+	 *
+	 *  During loop iteration, it consumes until it sees a token that can
+	 *  start a sub rule or what follows loop. Yes, that is pretty aggressive.
+	 *  We opt to stay in the loop as long as possible.
  	 */
 	@Override
 	public void sync(BaseRecognizer recognizer) {
+		ATNState s = recognizer._interp.atn.states.get(recognizer._ctx.s);
+//		System.err.println("sync @ "+s.stateNumber+"="+s.getClass().getSimpleName());
 		// If already recovering, don't try to sync
 		if ( errorRecoveryMode ) return;
+
 		// TODO: CACHE THESE RESULTS!!
 		IntervalSet expecting = getExpectedTokens(recognizer);
+//		System.err.println("sync expecting: "+expecting);
+
 		// TODO: subclass this class for treeparsers
 		TokenStream tokens = (TokenStream)recognizer.getInputStream();
 		Token la = tokens.LT(1);
 		// Return but don't end recovery. only do that upon valid token match
 		if ( la.getType()==Token.EOF || expecting.contains(la.getType()) ) return;
-		reportUnwantedToken(recognizer);
+
-		consumeUntil(recognizer, expecting);
+		if ( s instanceof PlusBlockStartState ||
+			 s instanceof StarLoopEntryState ||
+			 s instanceof BlockStartState )
+		{
+			// report error and recover if possible
+			if ( singleTokenDeletion(recognizer)!=null ) return;
+			throw new InputMismatchException(recognizer);
+		}
+		if ( s instanceof PlusLoopbackState ||
+			 s instanceof StarLoopbackState )
+		{
+//			System.err.println("at loop back: "+s.getClass().getSimpleName());
+			reportUnwantedToken(recognizer);
+			consumeUntil(recognizer, expecting);
+		}
+		// do nothing if we can identify the exact kind of ATN state
 	}
 
 	public void reportNoViableAlternative(BaseRecognizer recognizer,
@@ -241,10 +266,55 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 	public Object recoverInline(BaseRecognizer recognizer)
 		throws RecognitionException
 	{
-		Object currentSymbol = recognizer.getCurrentInputSymbol();
-
-		// SINGLE TOKEN DELETION
 		// if next token is what we are looking for then "delete" this token
+//		int nextTokenType = recognizer.getInputStream().LA(2);
+//		IntervalSet expecting = getExpectedTokens(recognizer);
+//		if ( expecting.contains(nextTokenType) ) {
+//			reportUnwantedToken(recognizer);
+//			/*
+//			System.err.println("recoverFromMismatchedToken deleting "+
+//							   ((TokenStream)recognizer.getInputStream()).LT(1)+
+//							   " since "+((TokenStream)recognizer.getInputStream()).LT(2)+
+//							   " is what we want");
+//			*/
+//			recognizer.consume(); // simply delete extra token
+//			// we want to return the token we're actually matching
+//			Object matchedSymbol = recognizer.getCurrentInputSymbol();
+//			endErrorCondition(recognizer);  // we know next token is correct
+//			recognizer.consume(); // move past ttype token as if all were ok
+//			return matchedSymbol;
+//		}
+		// SINGLE TOKEN DELETION
+		Object matchedSymbol = singleTokenDeletion(recognizer);
+		if ( matchedSymbol!=null ) return matchedSymbol;
+
+		// SINGLE TOKEN INSERTION
+		if ( singleTokenInsertion(recognizer) ) {
+			return getMissingSymbol(recognizer);
+		}
+
+		// even that didn't work; must throw the exception
+		throw new InputMismatchException(recognizer);
+	}
+
+	// if next token is what we are looking for then "delete" this token
+	public boolean singleTokenInsertion(BaseRecognizer recognizer) {
+		Object currentSymbol = recognizer.getCurrentInputSymbol();
+		// if current token is consistent with what could come after current
+		// ATN state, then we know we're missing a token; error recovery
+		// is free to conjure up and insert the missing token
+		ATNState currentState = recognizer._interp.atn.states.get(recognizer._ctx.s);
+		ATNState next = currentState.transition(0).target;
+		IntervalSet expectingAtLL2 = recognizer._interp.atn.nextTokens(next, recognizer._ctx);
+//		System.out.println("LT(2) set="+expectingAtLL2.toString(recognizer.getTokenNames()));
+		if ( expectingAtLL2.contains(((Token)currentSymbol).getType()) ) {
+			reportMissingToken(recognizer);
+			return true;
+		}
+		return false;
+	}
+
+	public Object singleTokenDeletion(BaseRecognizer recognizer) {
 		int nextTokenType = recognizer.getInputStream().LA(2);
 		IntervalSet expecting = getExpectedTokens(recognizer);
 		if ( expecting.contains(nextTokenType) ) {
@@ -256,29 +326,13 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 							   " is what we want");
 			*/
 			recognizer.consume(); // simply delete extra token
-//			recognizer.getInputStream().consume(); // simply delete extra token
 			// we want to return the token we're actually matching
 			Object matchedSymbol = recognizer.getCurrentInputSymbol();
 			endErrorCondition(recognizer);  // we know next token is correct
 			recognizer.consume(); // move past ttype token as if all were ok
-//			recognizer.getInputStream().consume(); // move past ttype token as if all were ok
 			return matchedSymbol;
 		}
-
+		return null;
-		// SINGLE TOKEN INSERTION
-		// if current token is consistent with what could come after current
-		// ATN state, then we know we're missing a token; error recovery
-		// is free to conjure up and insert the missing token
-		ATNState currentState = recognizer._interp.atn.states.get(recognizer._ctx.s);
-		ATNState next = currentState.transition(0).target;
-		IntervalSet expectingAtLL2 = recognizer._interp.atn.nextTokens(next, recognizer._ctx);
-//		System.out.println("LT(2) set="+expectingAtLL2.toString(recognizer.getTokenNames()));
-		if ( expectingAtLL2.contains(((Token)currentSymbol).getType()) ) {
-			reportMissingToken(recognizer);
-			return getMissingSymbol(recognizer);
-		}
-		// even that didn't work; must throw the exception
-		throw new InputMismatchException(recognizer);
 	}
 
 	/** Conjure up a missing token during error recovery.
@@ -461,7 +515,7 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
 
 	/** Consume tokens until one matches the given token set */
 	public void consumeUntil(BaseRecognizer recognizer, IntervalSet set) {
-//		System.out.println("consumeUntil("+set.toString(recognizer.getTokenNames())+")");
+//		System.err.println("consumeUntil("+set.toString(recognizer.getTokenNames())+")");
 		int ttype = recognizer.getInputStream().LA(1);
 		while (ttype != Token.EOF && !set.contains(ttype) ) {
 			//System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]);

runtime/Java/src/org/antlr/v4/runtime/atn/ATNState.java

@@ -35,6 +35,14 @@ public class ATNState {
 	public static final int INITIAL_NUM_TRANSITIONS = 4;
 
 	// constants for serialization
+//	public static enum ATNStateType {
+//		BASIC(1), RULE_START(2), BLOCK_START(3), PLUS_BLOCK_START(4),
+//		STAR_BLOCK_START(5), TOKEN_START(6), RULE_STOP(7), BLOCK_END(8),
+//		STAR_LOOP_BACK(9), STAR_LOOP_ENTRY(10), PLUS_LOOP_BACK(11);
+//		public int type;
+//		ATNStateType(int type) { this.type = type; }
+//		public int intValue() { return type; }
+//	}
 	public static final int BASIC = 1;
 	public static final int RULE_START = 2;
 	public static final int BLOCK_START = 3;
@@ -83,6 +91,8 @@ public class ATNState {
 
 	public int ruleIndex; // at runtime, we don't have Rule objects
 
+	public int epsilonOnlyTransitions = -1;
+
 	/** Which ATN are we in? */
 	public ATN atn = null;
 
@@ -123,7 +133,9 @@ public class ATNState {
 		transitions.set(i, e);
 	}
 
-	protected int epsilonOnlyTransitions = -1;
+	public int getStateType() {
+		return serializationTypes.get(this.getClass());
+	}
 
 	//lexer atn sim: getEpTar: 13.2%
 	// ruleCtx.equals 10%

runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopEntryState.java

@@ -30,6 +30,7 @@
 package org.antlr.v4.runtime.atn;
 
 public class StarLoopEntryState extends DecisionState {
+	public StarLoopbackState loopBackState;
 	@Override
 	public boolean onlyHasEpsilonTransitions() { return true; }
 }

runtime/Java/src/org/antlr/v4/runtime/tree/gui/PostScriptDocument.java

@@ -70,26 +70,26 @@ public class PostScriptDocument {
 		ps.append("%!PS-Adobe-3.0 EPSF-3.0\n");
 		ps.append("%%BoundingBox: (atend)\n");
 		lineWidth(0.3);
-		ps.append("%\n");
+//		ps.append("%\n");
-		ps.append("% x y rarrow\n");
+//		ps.append("% x y rarrow\n");
-		ps.append("%\n");
+//		ps.append("%\n");
-		ps.append("/rarrow {\n");
+//		ps.append("/rarrow {\n");
-		ps.append("  moveto\n");
+//		ps.append("  moveto\n");
-		ps.append("  -3 +2 rlineto\n");
+//		ps.append("  -3 +2 rlineto\n");
-		ps.append("  0 -4 rlineto\n");
+//		ps.append("  0 -4 rlineto\n");
-		ps.append("  3 +2 rlineto\n");
+//		ps.append("  3 +2 rlineto\n");
-		ps.append("  fill\n");
+//		ps.append("  fill\n");
-		ps.append("} def\n");
+//		ps.append("} def\n");
-		ps.append("%\n");
+//		ps.append("%\n");
-		ps.append("% x y darrow\n");
+//		ps.append("% x y darrow\n");
-		ps.append("%\n");
+//		ps.append("%\n");
-		ps.append("/darrow {\n");
+//		ps.append("/darrow {\n");
-		ps.append("  moveto\n");
+//		ps.append("  moveto\n");
-		ps.append("  -2 +3 rlineto\n");
+//		ps.append("  -2 +3 rlineto\n");
-		ps.append("  4 0 rlineto\n");
+//		ps.append("  4 0 rlineto\n");
-		ps.append("  -2 -3 rlineto\n");
+//		ps.append("  -2 -3 rlineto\n");
-		ps.append("  fill\n");
+//		ps.append("  fill\n");
-		ps.append("} def\n");
+//		ps.append("} def\n");
 	}
 
 	// Courier, Helvetica, Times, ... should be available
@@ -144,6 +144,22 @@ public class PostScriptDocument {
 	}
 
 	public void text(String s, double x, double y) {
+		StringBuilder buf = new StringBuilder();
+		// escape \, (, ): \\,  \(,  \)
+		for (char c : s.toCharArray()) {
+			switch ( c ) {
+				case '\\' :
+				case '(' :
+				case ')' :
+					buf.append('\\');
+					buf.append(c);
+					break;
+				default :
+					buf.append(c);
+					break;
+			}
+		}
+		s = buf.toString();
 		move(x,y);
 		ps.append(String.format("(%s) show\n", s));
 		stroke();

runtime/Java/src/org/antlr/v4/runtime/tree/gui/TreePostScriptGenerator.java

@@ -1,6 +1,6 @@
 /*
  [The "BSD license"]
- Copyright (c) 2011 Terence Parr
+ Copyright (c) 2011 T2rence Parr
  All rights reserved.
 
  Redistribution and use in source and binary forms, with or without
@@ -36,6 +36,7 @@ import org.antlr.v4.runtime.tree.*;
 
 import java.awt.*;
 import java.awt.geom.Rectangle2D;
+import java.io.IOException;
 
 public class TreePostScriptGenerator {
 	public class VariableExtentProvide implements NodeExtentProvider<Tree> {
@@ -48,16 +49,18 @@ public class TreePostScriptGenerator {
 		@Override
 		public double getHeight(Tree tree) {
 			String s = getText(tree);
-			double h = doc.getLineHeight() + nodeHeightPadding*2;
+			double h =
+				doc.getLineHeight() + nodeHeightPaddingAbove + nodeHeightPaddingBelow;
 			String[] lines = s.split("\n");
 			return h * lines.length;
 		}
 	}
 
-	protected double gapBetweenLevels = 30;
+	protected double gapBetweenLevels = 10;
-	protected double gapBetweenNodes = 10;
+	protected double gapBetweenNodes = 7;
 	protected int nodeWidthPadding = 1;  // added to left/right
-	protected int nodeHeightPadding = 1; // added above/below
+	protected int nodeHeightPaddingAbove = 2;
+	protected int nodeHeightPaddingBelow = 5;
 
 	protected Tree root;
 	protected TreeTextProvider treeTextProvider;
@@ -99,12 +102,12 @@ public class TreePostScriptGenerator {
 	protected void generateEdges(Tree parent) {
 		if (!getTree().isLeaf(parent)) {
 			Rectangle2D.Double parentBounds = getBoundsOfNode(parent);
-			System.out.println("%% parent("+getText(parent)+")="+parentBounds);
+//			System.out.println("%% parent("+getText(parent)+")="+parentBounds);
 			double x1 = parentBounds.getCenterX();
 			double y1 = parentBounds.y;
 			for (Tree child : getChildren(parent)) {
 				Rectangle2D.Double childBounds = getBoundsOfNode(child);
-				System.out.println("%% child("+getText(child)+")="+childBounds);
+//				System.out.println("%% child("+getText(child)+")="+childBounds);
 				double x2 = childBounds.getCenterX();
 				double y2 = childBounds.getMaxY();
 				doc.line(x1, y1, x2, y2);
@@ -120,7 +123,7 @@ public class TreePostScriptGenerator {
 		// for debugging, turn this on to see boundingbox of nodes
 		// doc.rect(box.x, box.y, box.width, box.height);
 		double x = box.x+nodeWidthPadding;
-		double y = box.y+nodeHeightPadding;
+		double y = box.y+nodeHeightPaddingBelow;
 		for (int i = 0; i < lines.length; i++) {
 			doc.text(lines[i], x, y);
 			y += doc.getFontSize();
@@ -151,19 +154,36 @@ public class TreePostScriptGenerator {
 		this.treeTextProvider = treeTextProvider;
 	}
 
-	public static void main(String[] args) {
+	public static void main(String[] args) throws IOException {
-		CommonAST t = new CommonAST(new CommonToken(1, "+"));
+		CommonAST t = new CommonAST(new CommonToken(1, "s"));
-		CommonAST a = new CommonAST(new CommonToken(1, "expression"));
+		CommonAST ifstat = new CommonAST(new CommonToken(1, "ifstat"));
-		CommonAST f = new CommonAST(new CommonToken(1, "3000"));
+		CommonAST iff = new CommonAST(new CommonToken(1, "if"));
-		CommonAST b = new CommonAST(new CommonToken(1, "*"));
+		CommonAST b = new CommonAST(new CommonToken(1, "("));
-		CommonAST c = new CommonAST(new CommonToken(1, "42"));
+		CommonAST c = new CommonAST(new CommonToken(1, "expr"));
-		CommonAST d = new CommonAST(new CommonToken(1, "105"));
+		CommonAST d = new CommonAST(new CommonToken(1, ")"));
-		t.addChild(a);
+		CommonAST e = new CommonAST(new CommonToken(1, "assign"));
-		a.addChild(f);
+		CommonAST f = new CommonAST(new CommonToken(1, "34"));
-		t.addChild(b);
+		CommonAST g = new CommonAST(new CommonToken(1, "a"));
-		b.addChild(c);
+		CommonAST h = new CommonAST(new CommonToken(1, "="));
-		b.addChild(d);
+		CommonAST i = new CommonAST(new CommonToken(1, "expr"));
-		TreePostScriptGenerator psgen = new TreePostScriptGenerator(null, t, "CourierNew", 11);
+		CommonAST j = new CommonAST(new CommonToken(1, ";"));
-		System.out.println(psgen.getPS());
+		CommonAST k = new CommonAST(new CommonToken(1, "b"));
+		t.addChild(ifstat);
+		ifstat.addChild(iff);
+		ifstat.addChild(b);
+		ifstat.addChild(c);
+		ifstat.addChild(d);
+		ifstat.addChild(e);
+		c.addChild(f);
+		e.addChild(g);
+		e.addChild(h);
+		e.addChild(i);
+		e.addChild(j);
+		i.addChild(k);
+		Trees.writePS(t, null,
+					  "/Users/parrt/antlr/code/antlr4/main/tool/playground/t.eps",
+					  "Arial", 11);
+//		TreePostScriptGenerator psgen = new TreePostScriptGenerator(null, t, "CourierNew", 11);
+//		System.out.println(psgen.getPS());
 	}
 }

runtime/Java/src/org/antlr/v4/runtime/tree/gui/TreeViewer.java

@@ -82,10 +82,10 @@ public class TreeViewer extends JComponent {
 	protected int fontSize = 12;
 	protected Font font = new Font(fontName, fontStyle, fontSize);
 
-	protected double gapBetweenLevels = 30;
+	protected double gapBetweenLevels = 12;
-	protected double gapBetweenNodes = 10;
+	protected double gapBetweenNodes = 7;
 	protected int nodeWidthPadding = 2;  // added to left/right
-	protected int nodeHeightPadding = 1; // added above/below
+	protected int nodeHeightPadding = 4; // added above/below
 	protected int arcSize = 0;           // make an arc in node outline?
 
 	protected Color boxColor = Color.white;

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -265,7 +265,7 @@ while (true) {
 		<cases(choice.exitLook)>
 			break <choice.loopLabel>;
 	}
-	setState(<choice.stateNumber>);
+	setState(<choice.loopBackStateNumber>);
 	_errHandler.sync(this);
 }
 >>
@@ -276,7 +276,7 @@ _errHandler.sync(this);
 <preamble; separator="\n">
 while ( <loopExpr> ) {
 	<alts; separator="\n">
-	setState(<choice.stateNumber>);
+	setState(<choice.loopBackStateNumber>);
 	_errHandler.sync(this);
 	<iteration>
 }
@@ -350,7 +350,7 @@ case <i>:
 	<alt>
 	break;}; separator="\n">
 	}
-	setState(<choice.stateNumber>);
+	setState(<choice.loopBackStateNumber>);
 	_errHandler.sync(this);
 	_alt<choice.uniqueID> = _interp.adaptivePredict(_input,<choice.decision>,_ctx);
 }

tool/src/org/antlr/v4/automata/ATNSerializer.java

@@ -79,7 +79,7 @@ public class ATNSerializer {
 		int nedges = 0;
 		// dump states, count edges and collect sets while doing so
 		for (ATNState s : atn.states) {
-			data.add(ATNState.serializationTypes.get(s.getClass()));
+			data.add(s.getStateType());
 			data.add(s.ruleIndex);
 			nedges += s.getNumberOfTransitions();
 			for (int i=0; i<s.getNumberOfTransitions(); i++) {

tool/src/org/antlr/v4/automata/ParserATNFactory.java

@@ -30,26 +30,18 @@
 package org.antlr.v4.automata;
 
 
-import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.*;
-import org.antlr.runtime.Token;
+import org.antlr.runtime.tree.*;
-import org.antlr.runtime.tree.CommonTreeNodeStream;
-import org.antlr.runtime.tree.Tree;
 import org.antlr.v4.misc.CharSupport;
-import org.antlr.v4.parse.ANTLRParser;
+import org.antlr.v4.parse.*;
-import org.antlr.v4.parse.ATNBuilder;
-import org.antlr.v4.parse.GrammarASTAdaptor;
 import org.antlr.v4.runtime.atn.*;
 import org.antlr.v4.runtime.misc.IntervalSet;
 import org.antlr.v4.semantics.UseDefAnalyzer;
-import org.antlr.v4.tool.ErrorManager;
+import org.antlr.v4.tool.*;
-import org.antlr.v4.tool.ErrorType;
-import org.antlr.v4.tool.Grammar;
-import org.antlr.v4.tool.Rule;
 import org.antlr.v4.tool.ast.*;
 
 import java.lang.reflect.Constructor;
-import java.util.Collection;
+import java.util.*;
-import java.util.List;
 
 /** ATN construction routines triggered by ATNBuilder.g.
  *
@@ -364,9 +356,9 @@ public class ParserATNFactory implements ATNFactory {
 		PlusLoopbackState loop = (PlusLoopbackState)newState(PlusLoopbackState.class, plusAST);
 		atn.defineDecisionState(loop);
 		ATNState end = newState(ATNState.class, plusAST);
+		blkStart.loopBackState = loop;
 
 		plusAST.atnState = blkStart;
-		blkStart.loopBackState = loop;
 		epsilon(blkEnd, loop);		// blk can see loop back
 
 		BlockAST blkAST = (BlockAST)plusAST.getChild(0);
@@ -404,6 +396,7 @@ public class ParserATNFactory implements ATNFactory {
 		atn.defineDecisionState(entry);
 		ATNState end = newState(ATNState.class, starAST);
 		StarLoopbackState loop = (StarLoopbackState)newState(StarLoopbackState.class, starAST);
+		entry.loopBackState = loop;
 
 		BlockAST blkAST = (BlockAST)starAST.getChild(0);
 		entry.isGreedy = isGreedy(blkAST);

tool/src/org/antlr/v4/codegen/model/LL1Loop.java

@@ -33,8 +33,7 @@ import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.runtime.misc.IntervalSet;
 import org.antlr.v4.tool.ast.GrammarAST;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.List;
 
 /** */
 public abstract class LL1Loop extends Choice {
@@ -42,6 +41,7 @@ public abstract class LL1Loop extends Choice {
 	 *  is super.stateNumber
 	 */
 	public int blockStartStateNumber;
+	public int loopBackStateNumber;
 
 	@ModelElement public OutputModelObject loopExpr;
 	@ModelElement public List<SrcOp> iteration;

tool/src/org/antlr/v4/codegen/model/LL1StarBlock.java

@@ -50,6 +50,8 @@ public class LL1StarBlock extends LL1Loop {
 		blockStartStateNumber =
 			starRootAST.atnState.transition(0).target.stateNumber;
 
+		loopBackStateNumber = star.loopBackState.stateNumber;
+
 		this.decision = star.decision;
 
 		/** Lookahead for each alt 1..n */

tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java

@@ -41,8 +41,8 @@ public class LL1StarBlockSingleAlt extends LL1Loop {
 	public LL1StarBlockSingleAlt(OutputModelFactory factory, GrammarAST starRoot, List<CodeBlockForAlt> alts) {
 		super(factory, starRoot, alts);
 
-//		StarBlockStartState star = (StarBlockStartState)starRoot.atnState;
 		StarLoopEntryState star = (StarLoopEntryState)starRoot.atnState;
+		loopBackStateNumber = star.loopBackState.stateNumber;
 		this.decision = star.decision;
 		IntervalSet[] altLookSets = factory.getGrammar().decisionLOOK.get(decision);
 		IntervalSet enterLook = altLookSets[1];

tool/src/org/antlr/v4/codegen/model/Loop.java

@@ -35,6 +35,7 @@ import org.antlr.v4.tool.ast.GrammarAST;
 import java.util.List;
 
 public class Loop extends Choice {
+	public int loopBackStateNumber;
 	public int exitAlt;
 	public Loop(OutputModelFactory factory,
 				GrammarAST blkOrEbnfRootAST,

tool/src/org/antlr/v4/codegen/model/PlusBlock.java

@@ -30,8 +30,7 @@
 package org.antlr.v4.codegen.model;
 
 import org.antlr.v4.codegen.OutputModelFactory;
-import org.antlr.v4.runtime.atn.PlusBlockStartState;
+import org.antlr.v4.runtime.atn.*;
-import org.antlr.v4.runtime.atn.PlusLoopbackState;
 import org.antlr.v4.tool.ast.GrammarAST;
 
 import java.util.List;
@@ -45,6 +44,7 @@ public class PlusBlock extends Loop {
 	{
 		super(factory, ebnfRootAST, alts);
 		PlusLoopbackState loop = ((PlusBlockStartState)ebnfRootAST.atnState).loopBackState;
+		loopBackStateNumber = loop.stateNumber;
 		stateNumber = loop.stateNumber;
 		this.error = new ThrowNoViableAlt(factory, ebnfRootAST, null);
 		decision = loop.decision;

tool/src/org/antlr/v4/codegen/model/StarBlock.java

@@ -45,6 +45,7 @@ public class StarBlock extends Loop {
 		super(factory, blkOrEbnfRootAST, alts);
 		loopLabel = factory.getGenerator().target.getLoopLabel(blkOrEbnfRootAST);
 		StarLoopEntryState star = (StarLoopEntryState)blkOrEbnfRootAST.atnState;
+		loopBackStateNumber = star.loopBackState.stateNumber;
 		decision = star.decision;
 		exitAlt = alts.size()+1;
 	}

tool/test/org/antlr/v4/test/TestParseErrors.java

@@ -151,11 +151,14 @@ public class TestParseErrors extends BaseTest {
 	}
 
 	@Test public void testMultiTokenDeletionBeforeLoop() throws Exception {
+		// can only delete 1 before loop
 		String grammar =
 			"grammar T;\n" +
 			"a : 'a' 'b'* 'c';";
 		String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "aacabc", false);
-		String expecting = "line 1:1 extraneous input 'a' expecting {'b', 'c'}\n";
+		String expecting =
+			"line 1:1 extraneous input 'a' expecting {'b', 'c'}\n" +
+			"line 1:3 mismatched input 'a' expecting 'c'\n";
 		String result = stderrDuringParse;
 		assertEquals(expecting, result);
 	}
@@ -182,6 +185,53 @@ public class TestParseErrors extends BaseTest {
 		assertEquals(expecting, result);
 	}
 
+	// ------
+
+	@Test public void testSingleTokenDeletionBeforeLoop2() throws Exception {
+		String grammar =
+			"grammar T;\n" +
+			"a : 'a' ('b'|'z'{;})*;";
+		String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "aabc", false);
+		String expecting = "line 1:1 extraneous input 'a' expecting {<EOF>, 'b', 'z'}\n";
+		String result = stderrDuringParse;
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testMultiTokenDeletionBeforeLoop2() throws Exception {
+		// can only delete 1 before loop
+		String grammar =
+			"grammar T;\n" +
+			"a : 'a' ('b'|'z'{;})* 'c';";
+		String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "aacabc", false);
+		String expecting =
+			"line 1:1 extraneous input 'a' expecting {'b', 'z', 'c'}\n" +
+			"line 1:3 mismatched input 'a' expecting 'c'\n";
+		String result = stderrDuringParse;
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testSingleTokenDeletionDuringLoop2() throws Exception {
+		String grammar =
+			"grammar T;\n" +
+			"a : 'a' ('b'|'z'{;})* 'c' ;";
+		String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "ababbc", false);
+		String expecting = "line 1:2 extraneous input 'a' expecting {'b', 'z', 'c'}\n";
+		String result = stderrDuringParse;
+		assertEquals(expecting, result);
+	}
+
+	@Test public void testMultiTokenDeletionDuringLoop2() throws Exception {
+		String grammar =
+			"grammar T;\n" +
+			"a : 'a' ('b'|'z'{;})* 'c' ;";
+		String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "abaaababc", false);
+		String expecting =
+				"line 1:2 extraneous input 'a' expecting {'b', 'z', 'c'}\n" +
+				"line 1:6 extraneous input 'a' expecting {'b', 'z', 'c'}\n";
+		String result = stderrDuringParse;
+		assertEquals(expecting, result);
+	}
+
 	@Test public void testLL1ErrorInfo() throws Exception {
 		String grammar =
 			"grammar T;\n" +