refactored sync/recoverInLine. figured out exactly what sync does at start of alt. setState goes to loopBack now in loops

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9177]
This commit is contained in:
parrt 2011-10-23 11:07:07 -08:00
parent 4920bb2c9e
commit 11b61d979d
17 changed files with 255 additions and 101 deletions

View File

@ -190,7 +190,7 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
* If the parser is creating parse trees, the current symbol * If the parser is creating parse trees, the current symbol
* would also be added as a child to the current context (node). * would also be added as a child to the current context (node).
*/ */
protected Object consume() { public Object consume() {
Object o = getCurrentInputSymbol(); Object o = getCurrentInputSymbol();
getInputStream().consume(); getInputStream().consume();
if ( buildParseTrees ) { if ( buildParseTrees ) {
@ -227,6 +227,10 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
return null; return null;
} }
public ParserRuleContext getContext() {
return _ctx;
}
public boolean inContext(String context) { public boolean inContext(String context) {
// TODO: useful in parser? // TODO: useful in parser?
return false; return false;

View File

@ -129,28 +129,53 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
/** Make sure that the current lookahead symbol is consistent with /** Make sure that the current lookahead symbol is consistent with
* what were expecting at this point in the ATN. * what were expecting at this point in the ATN.
* sync() differs fundamentally from the recoverInline() method. *
* In this case, we throw out a token that's not in the set of what * At the start of a sub rule upon error, sync() performs single
* were expecting at this point. recoverInline() only deletes this * token deletion, if possible. If it can't do that, it bails
* token if LT(2) (token after the current token) is what were expecting; * on the current rule and uses the default error recovery,
* i.e., we have an extra token sitting on the input stream. sync() * which consumes until the resynchronization set of the current rule.
* simply consumes until it finds something that can start whatever *
* follows the call to sync(). * If the sub rule is optional, ()? or ()* or optional alternative,
* then the expected set includes what follows the subrule.
*
* During loop iteration, it consumes until it sees a token that can
* start a sub rule or what follows loop. Yes, that is pretty aggressive.
* We opt to stay in the loop as long as possible.
*/ */
@Override @Override
public void sync(BaseRecognizer recognizer) { public void sync(BaseRecognizer recognizer) {
ATNState s = recognizer._interp.atn.states.get(recognizer._ctx.s);
// System.err.println("sync @ "+s.stateNumber+"="+s.getClass().getSimpleName());
// If already recovering, don't try to sync // If already recovering, don't try to sync
if ( errorRecoveryMode ) return; if ( errorRecoveryMode ) return;
// TODO: CACHE THESE RESULTS!! // TODO: CACHE THESE RESULTS!!
IntervalSet expecting = getExpectedTokens(recognizer); IntervalSet expecting = getExpectedTokens(recognizer);
// System.err.println("sync expecting: "+expecting);
// TODO: subclass this class for treeparsers // TODO: subclass this class for treeparsers
TokenStream tokens = (TokenStream)recognizer.getInputStream(); TokenStream tokens = (TokenStream)recognizer.getInputStream();
Token la = tokens.LT(1); Token la = tokens.LT(1);
// Return but don't end recovery. only do that upon valid token match // Return but don't end recovery. only do that upon valid token match
if ( la.getType()==Token.EOF || expecting.contains(la.getType()) ) return; if ( la.getType()==Token.EOF || expecting.contains(la.getType()) ) return;
if ( s instanceof PlusBlockStartState ||
s instanceof StarLoopEntryState ||
s instanceof BlockStartState )
{
// report error and recover if possible
if ( singleTokenDeletion(recognizer)!=null ) return;
throw new InputMismatchException(recognizer);
}
if ( s instanceof PlusLoopbackState ||
s instanceof StarLoopbackState )
{
// System.err.println("at loop back: "+s.getClass().getSimpleName());
reportUnwantedToken(recognizer); reportUnwantedToken(recognizer);
consumeUntil(recognizer, expecting); consumeUntil(recognizer, expecting);
} }
// do nothing if we can identify the exact kind of ATN state
}
public void reportNoViableAlternative(BaseRecognizer recognizer, public void reportNoViableAlternative(BaseRecognizer recognizer,
NoViableAltException e) NoViableAltException e)
@ -241,10 +266,55 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
public Object recoverInline(BaseRecognizer recognizer) public Object recoverInline(BaseRecognizer recognizer)
throws RecognitionException throws RecognitionException
{ {
Object currentSymbol = recognizer.getCurrentInputSymbol();
// SINGLE TOKEN DELETION
// if next token is what we are looking for then "delete" this token // if next token is what we are looking for then "delete" this token
// int nextTokenType = recognizer.getInputStream().LA(2);
// IntervalSet expecting = getExpectedTokens(recognizer);
// if ( expecting.contains(nextTokenType) ) {
// reportUnwantedToken(recognizer);
// /*
// System.err.println("recoverFromMismatchedToken deleting "+
// ((TokenStream)recognizer.getInputStream()).LT(1)+
// " since "+((TokenStream)recognizer.getInputStream()).LT(2)+
// " is what we want");
// */
// recognizer.consume(); // simply delete extra token
// // we want to return the token we're actually matching
// Object matchedSymbol = recognizer.getCurrentInputSymbol();
// endErrorCondition(recognizer); // we know next token is correct
// recognizer.consume(); // move past ttype token as if all were ok
// return matchedSymbol;
// }
// SINGLE TOKEN DELETION
Object matchedSymbol = singleTokenDeletion(recognizer);
if ( matchedSymbol!=null ) return matchedSymbol;
// SINGLE TOKEN INSERTION
if ( singleTokenInsertion(recognizer) ) {
return getMissingSymbol(recognizer);
}
// even that didn't work; must throw the exception
throw new InputMismatchException(recognizer);
}
// if next token is what we are looking for then "delete" this token
public boolean singleTokenInsertion(BaseRecognizer recognizer) {
Object currentSymbol = recognizer.getCurrentInputSymbol();
// if current token is consistent with what could come after current
// ATN state, then we know we're missing a token; error recovery
// is free to conjure up and insert the missing token
ATNState currentState = recognizer._interp.atn.states.get(recognizer._ctx.s);
ATNState next = currentState.transition(0).target;
IntervalSet expectingAtLL2 = recognizer._interp.atn.nextTokens(next, recognizer._ctx);
// System.out.println("LT(2) set="+expectingAtLL2.toString(recognizer.getTokenNames()));
if ( expectingAtLL2.contains(((Token)currentSymbol).getType()) ) {
reportMissingToken(recognizer);
return true;
}
return false;
}
public Object singleTokenDeletion(BaseRecognizer recognizer) {
int nextTokenType = recognizer.getInputStream().LA(2); int nextTokenType = recognizer.getInputStream().LA(2);
IntervalSet expecting = getExpectedTokens(recognizer); IntervalSet expecting = getExpectedTokens(recognizer);
if ( expecting.contains(nextTokenType) ) { if ( expecting.contains(nextTokenType) ) {
@ -256,29 +326,13 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
" is what we want"); " is what we want");
*/ */
recognizer.consume(); // simply delete extra token recognizer.consume(); // simply delete extra token
// recognizer.getInputStream().consume(); // simply delete extra token
// we want to return the token we're actually matching // we want to return the token we're actually matching
Object matchedSymbol = recognizer.getCurrentInputSymbol(); Object matchedSymbol = recognizer.getCurrentInputSymbol();
endErrorCondition(recognizer); // we know next token is correct endErrorCondition(recognizer); // we know next token is correct
recognizer.consume(); // move past ttype token as if all were ok recognizer.consume(); // move past ttype token as if all were ok
// recognizer.getInputStream().consume(); // move past ttype token as if all were ok
return matchedSymbol; return matchedSymbol;
} }
return null;
// SINGLE TOKEN INSERTION
// if current token is consistent with what could come after current
// ATN state, then we know we're missing a token; error recovery
// is free to conjure up and insert the missing token
ATNState currentState = recognizer._interp.atn.states.get(recognizer._ctx.s);
ATNState next = currentState.transition(0).target;
IntervalSet expectingAtLL2 = recognizer._interp.atn.nextTokens(next, recognizer._ctx);
// System.out.println("LT(2) set="+expectingAtLL2.toString(recognizer.getTokenNames()));
if ( expectingAtLL2.contains(((Token)currentSymbol).getType()) ) {
reportMissingToken(recognizer);
return getMissingSymbol(recognizer);
}
// even that didn't work; must throw the exception
throw new InputMismatchException(recognizer);
} }
/** Conjure up a missing token during error recovery. /** Conjure up a missing token during error recovery.
@ -461,7 +515,7 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
/** Consume tokens until one matches the given token set */ /** Consume tokens until one matches the given token set */
public void consumeUntil(BaseRecognizer recognizer, IntervalSet set) { public void consumeUntil(BaseRecognizer recognizer, IntervalSet set) {
// System.out.println("consumeUntil("+set.toString(recognizer.getTokenNames())+")"); // System.err.println("consumeUntil("+set.toString(recognizer.getTokenNames())+")");
int ttype = recognizer.getInputStream().LA(1); int ttype = recognizer.getInputStream().LA(1);
while (ttype != Token.EOF && !set.contains(ttype) ) { while (ttype != Token.EOF && !set.contains(ttype) ) {
//System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]); //System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]);

View File

@ -35,6 +35,14 @@ public class ATNState {
public static final int INITIAL_NUM_TRANSITIONS = 4; public static final int INITIAL_NUM_TRANSITIONS = 4;
// constants for serialization // constants for serialization
// public static enum ATNStateType {
// BASIC(1), RULE_START(2), BLOCK_START(3), PLUS_BLOCK_START(4),
// STAR_BLOCK_START(5), TOKEN_START(6), RULE_STOP(7), BLOCK_END(8),
// STAR_LOOP_BACK(9), STAR_LOOP_ENTRY(10), PLUS_LOOP_BACK(11);
// public int type;
// ATNStateType(int type) { this.type = type; }
// public int intValue() { return type; }
// }
public static final int BASIC = 1; public static final int BASIC = 1;
public static final int RULE_START = 2; public static final int RULE_START = 2;
public static final int BLOCK_START = 3; public static final int BLOCK_START = 3;
@ -83,6 +91,8 @@ public class ATNState {
public int ruleIndex; // at runtime, we don't have Rule objects public int ruleIndex; // at runtime, we don't have Rule objects
public int epsilonOnlyTransitions = -1;
/** Which ATN are we in? */ /** Which ATN are we in? */
public ATN atn = null; public ATN atn = null;
@ -123,7 +133,9 @@ public class ATNState {
transitions.set(i, e); transitions.set(i, e);
} }
protected int epsilonOnlyTransitions = -1; public int getStateType() {
return serializationTypes.get(this.getClass());
}
//lexer atn sim: getEpTar: 13.2% //lexer atn sim: getEpTar: 13.2%
// ruleCtx.equals 10% // ruleCtx.equals 10%

View File

@ -30,6 +30,7 @@
package org.antlr.v4.runtime.atn; package org.antlr.v4.runtime.atn;
public class StarLoopEntryState extends DecisionState { public class StarLoopEntryState extends DecisionState {
public StarLoopbackState loopBackState;
@Override @Override
public boolean onlyHasEpsilonTransitions() { return true; } public boolean onlyHasEpsilonTransitions() { return true; }
} }

View File

@ -70,26 +70,26 @@ public class PostScriptDocument {
ps.append("%!PS-Adobe-3.0 EPSF-3.0\n"); ps.append("%!PS-Adobe-3.0 EPSF-3.0\n");
ps.append("%%BoundingBox: (atend)\n"); ps.append("%%BoundingBox: (atend)\n");
lineWidth(0.3); lineWidth(0.3);
ps.append("%\n"); // ps.append("%\n");
ps.append("% x y rarrow\n"); // ps.append("% x y rarrow\n");
ps.append("%\n"); // ps.append("%\n");
ps.append("/rarrow {\n"); // ps.append("/rarrow {\n");
ps.append(" moveto\n"); // ps.append(" moveto\n");
ps.append(" -3 +2 rlineto\n"); // ps.append(" -3 +2 rlineto\n");
ps.append(" 0 -4 rlineto\n"); // ps.append(" 0 -4 rlineto\n");
ps.append(" 3 +2 rlineto\n"); // ps.append(" 3 +2 rlineto\n");
ps.append(" fill\n"); // ps.append(" fill\n");
ps.append("} def\n"); // ps.append("} def\n");
ps.append("%\n"); // ps.append("%\n");
ps.append("% x y darrow\n"); // ps.append("% x y darrow\n");
ps.append("%\n"); // ps.append("%\n");
ps.append("/darrow {\n"); // ps.append("/darrow {\n");
ps.append(" moveto\n"); // ps.append(" moveto\n");
ps.append(" -2 +3 rlineto\n"); // ps.append(" -2 +3 rlineto\n");
ps.append(" 4 0 rlineto\n"); // ps.append(" 4 0 rlineto\n");
ps.append(" -2 -3 rlineto\n"); // ps.append(" -2 -3 rlineto\n");
ps.append(" fill\n"); // ps.append(" fill\n");
ps.append("} def\n"); // ps.append("} def\n");
} }
// Courier, Helvetica, Times, ... should be available // Courier, Helvetica, Times, ... should be available
@ -144,6 +144,22 @@ public class PostScriptDocument {
} }
public void text(String s, double x, double y) { public void text(String s, double x, double y) {
StringBuilder buf = new StringBuilder();
// escape \, (, ): \\, \(, \)
for (char c : s.toCharArray()) {
switch ( c ) {
case '\\' :
case '(' :
case ')' :
buf.append('\\');
buf.append(c);
break;
default :
buf.append(c);
break;
}
}
s = buf.toString();
move(x,y); move(x,y);
ps.append(String.format("(%s) show\n", s)); ps.append(String.format("(%s) show\n", s));
stroke(); stroke();

View File

@ -1,6 +1,6 @@
/* /*
[The "BSD license"] [The "BSD license"]
Copyright (c) 2011 Terence Parr Copyright (c) 2011 T2rence Parr
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
@ -36,6 +36,7 @@ import org.antlr.v4.runtime.tree.*;
import java.awt.*; import java.awt.*;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.io.IOException;
public class TreePostScriptGenerator { public class TreePostScriptGenerator {
public class VariableExtentProvide implements NodeExtentProvider<Tree> { public class VariableExtentProvide implements NodeExtentProvider<Tree> {
@ -48,16 +49,18 @@ public class TreePostScriptGenerator {
@Override @Override
public double getHeight(Tree tree) { public double getHeight(Tree tree) {
String s = getText(tree); String s = getText(tree);
double h = doc.getLineHeight() + nodeHeightPadding*2; double h =
doc.getLineHeight() + nodeHeightPaddingAbove + nodeHeightPaddingBelow;
String[] lines = s.split("\n"); String[] lines = s.split("\n");
return h * lines.length; return h * lines.length;
} }
} }
protected double gapBetweenLevels = 30; protected double gapBetweenLevels = 10;
protected double gapBetweenNodes = 10; protected double gapBetweenNodes = 7;
protected int nodeWidthPadding = 1; // added to left/right protected int nodeWidthPadding = 1; // added to left/right
protected int nodeHeightPadding = 1; // added above/below protected int nodeHeightPaddingAbove = 2;
protected int nodeHeightPaddingBelow = 5;
protected Tree root; protected Tree root;
protected TreeTextProvider treeTextProvider; protected TreeTextProvider treeTextProvider;
@ -99,12 +102,12 @@ public class TreePostScriptGenerator {
protected void generateEdges(Tree parent) { protected void generateEdges(Tree parent) {
if (!getTree().isLeaf(parent)) { if (!getTree().isLeaf(parent)) {
Rectangle2D.Double parentBounds = getBoundsOfNode(parent); Rectangle2D.Double parentBounds = getBoundsOfNode(parent);
System.out.println("%% parent("+getText(parent)+")="+parentBounds); // System.out.println("%% parent("+getText(parent)+")="+parentBounds);
double x1 = parentBounds.getCenterX(); double x1 = parentBounds.getCenterX();
double y1 = parentBounds.y; double y1 = parentBounds.y;
for (Tree child : getChildren(parent)) { for (Tree child : getChildren(parent)) {
Rectangle2D.Double childBounds = getBoundsOfNode(child); Rectangle2D.Double childBounds = getBoundsOfNode(child);
System.out.println("%% child("+getText(child)+")="+childBounds); // System.out.println("%% child("+getText(child)+")="+childBounds);
double x2 = childBounds.getCenterX(); double x2 = childBounds.getCenterX();
double y2 = childBounds.getMaxY(); double y2 = childBounds.getMaxY();
doc.line(x1, y1, x2, y2); doc.line(x1, y1, x2, y2);
@ -120,7 +123,7 @@ public class TreePostScriptGenerator {
// for debugging, turn this on to see boundingbox of nodes // for debugging, turn this on to see boundingbox of nodes
// doc.rect(box.x, box.y, box.width, box.height); // doc.rect(box.x, box.y, box.width, box.height);
double x = box.x+nodeWidthPadding; double x = box.x+nodeWidthPadding;
double y = box.y+nodeHeightPadding; double y = box.y+nodeHeightPaddingBelow;
for (int i = 0; i < lines.length; i++) { for (int i = 0; i < lines.length; i++) {
doc.text(lines[i], x, y); doc.text(lines[i], x, y);
y += doc.getFontSize(); y += doc.getFontSize();
@ -151,19 +154,36 @@ public class TreePostScriptGenerator {
this.treeTextProvider = treeTextProvider; this.treeTextProvider = treeTextProvider;
} }
public static void main(String[] args) { public static void main(String[] args) throws IOException {
CommonAST t = new CommonAST(new CommonToken(1, "+")); CommonAST t = new CommonAST(new CommonToken(1, "s"));
CommonAST a = new CommonAST(new CommonToken(1, "expression")); CommonAST ifstat = new CommonAST(new CommonToken(1, "ifstat"));
CommonAST f = new CommonAST(new CommonToken(1, "3000")); CommonAST iff = new CommonAST(new CommonToken(1, "if"));
CommonAST b = new CommonAST(new CommonToken(1, "*")); CommonAST b = new CommonAST(new CommonToken(1, "("));
CommonAST c = new CommonAST(new CommonToken(1, "42")); CommonAST c = new CommonAST(new CommonToken(1, "expr"));
CommonAST d = new CommonAST(new CommonToken(1, "105")); CommonAST d = new CommonAST(new CommonToken(1, ")"));
t.addChild(a); CommonAST e = new CommonAST(new CommonToken(1, "assign"));
a.addChild(f); CommonAST f = new CommonAST(new CommonToken(1, "34"));
t.addChild(b); CommonAST g = new CommonAST(new CommonToken(1, "a"));
b.addChild(c); CommonAST h = new CommonAST(new CommonToken(1, "="));
b.addChild(d); CommonAST i = new CommonAST(new CommonToken(1, "expr"));
TreePostScriptGenerator psgen = new TreePostScriptGenerator(null, t, "CourierNew", 11); CommonAST j = new CommonAST(new CommonToken(1, ";"));
System.out.println(psgen.getPS()); CommonAST k = new CommonAST(new CommonToken(1, "b"));
t.addChild(ifstat);
ifstat.addChild(iff);
ifstat.addChild(b);
ifstat.addChild(c);
ifstat.addChild(d);
ifstat.addChild(e);
c.addChild(f);
e.addChild(g);
e.addChild(h);
e.addChild(i);
e.addChild(j);
i.addChild(k);
Trees.writePS(t, null,
"/Users/parrt/antlr/code/antlr4/main/tool/playground/t.eps",
"Arial", 11);
// TreePostScriptGenerator psgen = new TreePostScriptGenerator(null, t, "CourierNew", 11);
// System.out.println(psgen.getPS());
} }
} }

View File

@ -82,10 +82,10 @@ public class TreeViewer extends JComponent {
protected int fontSize = 12; protected int fontSize = 12;
protected Font font = new Font(fontName, fontStyle, fontSize); protected Font font = new Font(fontName, fontStyle, fontSize);
protected double gapBetweenLevels = 30; protected double gapBetweenLevels = 12;
protected double gapBetweenNodes = 10; protected double gapBetweenNodes = 7;
protected int nodeWidthPadding = 2; // added to left/right protected int nodeWidthPadding = 2; // added to left/right
protected int nodeHeightPadding = 1; // added above/below protected int nodeHeightPadding = 4; // added above/below
protected int arcSize = 0; // make an arc in node outline? protected int arcSize = 0; // make an arc in node outline?
protected Color boxColor = Color.white; protected Color boxColor = Color.white;

View File

@ -265,7 +265,7 @@ while (true) {
<cases(choice.exitLook)> <cases(choice.exitLook)>
break <choice.loopLabel>; break <choice.loopLabel>;
} }
setState(<choice.stateNumber>); setState(<choice.loopBackStateNumber>);
_errHandler.sync(this); _errHandler.sync(this);
} }
>> >>
@ -276,7 +276,7 @@ _errHandler.sync(this);
<preamble; separator="\n"> <preamble; separator="\n">
while ( <loopExpr> ) { while ( <loopExpr> ) {
<alts; separator="\n"> <alts; separator="\n">
setState(<choice.stateNumber>); setState(<choice.loopBackStateNumber>);
_errHandler.sync(this); _errHandler.sync(this);
<iteration> <iteration>
} }
@ -350,7 +350,7 @@ case <i>:
<alt> <alt>
break;}; separator="\n"> break;}; separator="\n">
} }
setState(<choice.stateNumber>); setState(<choice.loopBackStateNumber>);
_errHandler.sync(this); _errHandler.sync(this);
_alt<choice.uniqueID> = _interp.adaptivePredict(_input,<choice.decision>,_ctx); _alt<choice.uniqueID> = _interp.adaptivePredict(_input,<choice.decision>,_ctx);
} }

View File

@ -79,7 +79,7 @@ public class ATNSerializer {
int nedges = 0; int nedges = 0;
// dump states, count edges and collect sets while doing so // dump states, count edges and collect sets while doing so
for (ATNState s : atn.states) { for (ATNState s : atn.states) {
data.add(ATNState.serializationTypes.get(s.getClass())); data.add(s.getStateType());
data.add(s.ruleIndex); data.add(s.ruleIndex);
nedges += s.getNumberOfTransitions(); nedges += s.getNumberOfTransitions();
for (int i=0; i<s.getNumberOfTransitions(); i++) { for (int i=0; i<s.getNumberOfTransitions(); i++) {

View File

@ -30,26 +30,18 @@
package org.antlr.v4.automata; package org.antlr.v4.automata;
import org.antlr.runtime.RecognitionException; import org.antlr.runtime.*;
import org.antlr.runtime.Token; import org.antlr.runtime.tree.*;
import org.antlr.runtime.tree.CommonTreeNodeStream;
import org.antlr.runtime.tree.Tree;
import org.antlr.v4.misc.CharSupport; import org.antlr.v4.misc.CharSupport;
import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.parse.*;
import org.antlr.v4.parse.ATNBuilder;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.misc.IntervalSet; import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.semantics.UseDefAnalyzer; import org.antlr.v4.semantics.UseDefAnalyzer;
import org.antlr.v4.tool.ErrorManager; import org.antlr.v4.tool.*;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.*; import org.antlr.v4.tool.ast.*;
import java.lang.reflect.Constructor; import java.lang.reflect.Constructor;
import java.util.Collection; import java.util.*;
import java.util.List;
/** ATN construction routines triggered by ATNBuilder.g. /** ATN construction routines triggered by ATNBuilder.g.
* *
@ -364,9 +356,9 @@ public class ParserATNFactory implements ATNFactory {
PlusLoopbackState loop = (PlusLoopbackState)newState(PlusLoopbackState.class, plusAST); PlusLoopbackState loop = (PlusLoopbackState)newState(PlusLoopbackState.class, plusAST);
atn.defineDecisionState(loop); atn.defineDecisionState(loop);
ATNState end = newState(ATNState.class, plusAST); ATNState end = newState(ATNState.class, plusAST);
blkStart.loopBackState = loop;
plusAST.atnState = blkStart; plusAST.atnState = blkStart;
blkStart.loopBackState = loop;
epsilon(blkEnd, loop); // blk can see loop back epsilon(blkEnd, loop); // blk can see loop back
BlockAST blkAST = (BlockAST)plusAST.getChild(0); BlockAST blkAST = (BlockAST)plusAST.getChild(0);
@ -404,6 +396,7 @@ public class ParserATNFactory implements ATNFactory {
atn.defineDecisionState(entry); atn.defineDecisionState(entry);
ATNState end = newState(ATNState.class, starAST); ATNState end = newState(ATNState.class, starAST);
StarLoopbackState loop = (StarLoopbackState)newState(StarLoopbackState.class, starAST); StarLoopbackState loop = (StarLoopbackState)newState(StarLoopbackState.class, starAST);
entry.loopBackState = loop;
BlockAST blkAST = (BlockAST)starAST.getChild(0); BlockAST blkAST = (BlockAST)starAST.getChild(0);
entry.isGreedy = isGreedy(blkAST); entry.isGreedy = isGreedy(blkAST);

View File

@ -33,8 +33,7 @@ import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.runtime.misc.IntervalSet; import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.tool.ast.GrammarAST; import org.antlr.v4.tool.ast.GrammarAST;
import java.util.ArrayList; import java.util.*;
import java.util.List;
/** */ /** */
public abstract class LL1Loop extends Choice { public abstract class LL1Loop extends Choice {
@ -42,6 +41,7 @@ public abstract class LL1Loop extends Choice {
* is super.stateNumber * is super.stateNumber
*/ */
public int blockStartStateNumber; public int blockStartStateNumber;
public int loopBackStateNumber;
@ModelElement public OutputModelObject loopExpr; @ModelElement public OutputModelObject loopExpr;
@ModelElement public List<SrcOp> iteration; @ModelElement public List<SrcOp> iteration;

View File

@ -50,6 +50,8 @@ public class LL1StarBlock extends LL1Loop {
blockStartStateNumber = blockStartStateNumber =
starRootAST.atnState.transition(0).target.stateNumber; starRootAST.atnState.transition(0).target.stateNumber;
loopBackStateNumber = star.loopBackState.stateNumber;
this.decision = star.decision; this.decision = star.decision;
/** Lookahead for each alt 1..n */ /** Lookahead for each alt 1..n */

View File

@ -41,8 +41,8 @@ public class LL1StarBlockSingleAlt extends LL1Loop {
public LL1StarBlockSingleAlt(OutputModelFactory factory, GrammarAST starRoot, List<CodeBlockForAlt> alts) { public LL1StarBlockSingleAlt(OutputModelFactory factory, GrammarAST starRoot, List<CodeBlockForAlt> alts) {
super(factory, starRoot, alts); super(factory, starRoot, alts);
// StarBlockStartState star = (StarBlockStartState)starRoot.atnState;
StarLoopEntryState star = (StarLoopEntryState)starRoot.atnState; StarLoopEntryState star = (StarLoopEntryState)starRoot.atnState;
loopBackStateNumber = star.loopBackState.stateNumber;
this.decision = star.decision; this.decision = star.decision;
IntervalSet[] altLookSets = factory.getGrammar().decisionLOOK.get(decision); IntervalSet[] altLookSets = factory.getGrammar().decisionLOOK.get(decision);
IntervalSet enterLook = altLookSets[1]; IntervalSet enterLook = altLookSets[1];

View File

@ -35,6 +35,7 @@ import org.antlr.v4.tool.ast.GrammarAST;
import java.util.List; import java.util.List;
public class Loop extends Choice { public class Loop extends Choice {
public int loopBackStateNumber;
public int exitAlt; public int exitAlt;
public Loop(OutputModelFactory factory, public Loop(OutputModelFactory factory,
GrammarAST blkOrEbnfRootAST, GrammarAST blkOrEbnfRootAST,

View File

@ -30,8 +30,7 @@
package org.antlr.v4.codegen.model; package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.OutputModelFactory; import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.runtime.atn.PlusBlockStartState; import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.PlusLoopbackState;
import org.antlr.v4.tool.ast.GrammarAST; import org.antlr.v4.tool.ast.GrammarAST;
import java.util.List; import java.util.List;
@ -45,6 +44,7 @@ public class PlusBlock extends Loop {
{ {
super(factory, ebnfRootAST, alts); super(factory, ebnfRootAST, alts);
PlusLoopbackState loop = ((PlusBlockStartState)ebnfRootAST.atnState).loopBackState; PlusLoopbackState loop = ((PlusBlockStartState)ebnfRootAST.atnState).loopBackState;
loopBackStateNumber = loop.stateNumber;
stateNumber = loop.stateNumber; stateNumber = loop.stateNumber;
this.error = new ThrowNoViableAlt(factory, ebnfRootAST, null); this.error = new ThrowNoViableAlt(factory, ebnfRootAST, null);
decision = loop.decision; decision = loop.decision;

View File

@ -45,6 +45,7 @@ public class StarBlock extends Loop {
super(factory, blkOrEbnfRootAST, alts); super(factory, blkOrEbnfRootAST, alts);
loopLabel = factory.getGenerator().target.getLoopLabel(blkOrEbnfRootAST); loopLabel = factory.getGenerator().target.getLoopLabel(blkOrEbnfRootAST);
StarLoopEntryState star = (StarLoopEntryState)blkOrEbnfRootAST.atnState; StarLoopEntryState star = (StarLoopEntryState)blkOrEbnfRootAST.atnState;
loopBackStateNumber = star.loopBackState.stateNumber;
decision = star.decision; decision = star.decision;
exitAlt = alts.size()+1; exitAlt = alts.size()+1;
} }

View File

@ -151,11 +151,14 @@ public class TestParseErrors extends BaseTest {
} }
@Test public void testMultiTokenDeletionBeforeLoop() throws Exception { @Test public void testMultiTokenDeletionBeforeLoop() throws Exception {
// can only delete 1 before loop
String grammar = String grammar =
"grammar T;\n" + "grammar T;\n" +
"a : 'a' 'b'* 'c';"; "a : 'a' 'b'* 'c';";
String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "aacabc", false); String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "aacabc", false);
String expecting = "line 1:1 extraneous input 'a' expecting {'b', 'c'}\n"; String expecting =
"line 1:1 extraneous input 'a' expecting {'b', 'c'}\n" +
"line 1:3 mismatched input 'a' expecting 'c'\n";
String result = stderrDuringParse; String result = stderrDuringParse;
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -182,6 +185,53 @@ public class TestParseErrors extends BaseTest {
assertEquals(expecting, result); assertEquals(expecting, result);
} }
// ------
@Test public void testSingleTokenDeletionBeforeLoop2() throws Exception {
String grammar =
"grammar T;\n" +
"a : 'a' ('b'|'z'{;})*;";
String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "aabc", false);
String expecting = "line 1:1 extraneous input 'a' expecting {<EOF>, 'b', 'z'}\n";
String result = stderrDuringParse;
assertEquals(expecting, result);
}
@Test public void testMultiTokenDeletionBeforeLoop2() throws Exception {
// can only delete 1 before loop
String grammar =
"grammar T;\n" +
"a : 'a' ('b'|'z'{;})* 'c';";
String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "aacabc", false);
String expecting =
"line 1:1 extraneous input 'a' expecting {'b', 'z', 'c'}\n" +
"line 1:3 mismatched input 'a' expecting 'c'\n";
String result = stderrDuringParse;
assertEquals(expecting, result);
}
@Test public void testSingleTokenDeletionDuringLoop2() throws Exception {
String grammar =
"grammar T;\n" +
"a : 'a' ('b'|'z'{;})* 'c' ;";
String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "ababbc", false);
String expecting = "line 1:2 extraneous input 'a' expecting {'b', 'z', 'c'}\n";
String result = stderrDuringParse;
assertEquals(expecting, result);
}
@Test public void testMultiTokenDeletionDuringLoop2() throws Exception {
String grammar =
"grammar T;\n" +
"a : 'a' ('b'|'z'{;})* 'c' ;";
String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "abaaababc", false);
String expecting =
"line 1:2 extraneous input 'a' expecting {'b', 'z', 'c'}\n" +
"line 1:6 extraneous input 'a' expecting {'b', 'z', 'c'}\n";
String result = stderrDuringParse;
assertEquals(expecting, result);
}
@Test public void testLL1ErrorInfo() throws Exception { @Test public void testLL1ErrorInfo() throws Exception {
String grammar = String grammar =
"grammar T;\n" + "grammar T;\n" +