From 8508dd6da0efd30d0165caf18474aaf4ea44d9e0 Mon Sep 17 00:00:00 2001 From: parrt Date: Sun, 2 Oct 2011 20:31:26 -0800 Subject: [PATCH] got sync in [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9101] --- .../antlr/v4/runtime/ANTLRErrorStrategy.java | 9 ++- .../org/antlr/v4/runtime/BaseRecognizer.java | 40 ++--------- .../v4/runtime/DefaultANTLRErrorStrategy.java | 68 ++++++++++++------- .../org/antlr/v4/runtime/tree/TreeParser.java | 3 +- .../v4/tool/templates/codegen/Java/Java.stg | 45 ++++++++---- .../org/antlr/v4/codegen/model/LL1Loop.java | 1 - .../antlr/v4/codegen/model/LL1PlusBlock.java | 6 -- .../codegen/model/LL1PlusBlockSingleAlt.java | 7 -- .../antlr/v4/codegen/model/LL1StarBlock.java | 3 - .../codegen/model/LL1StarBlockSingleAlt.java | 3 - .../org/antlr/v4/codegen/model/StarBlock.java | 1 - .../org/antlr/v4/test/TestParseErrors.java | 17 ++++- 12 files changed, 107 insertions(+), 96 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java b/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java index 5c5ee346e..f15335219 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java @@ -52,7 +52,12 @@ public interface ANTLRErrorStrategy { */ void recover(BaseRecognizer recognizer); - /** Implement Jim Idle's magic sync mechanism in closures and optional + /** Make sure that the current lookahead symbol is consistent with + * what were expecting at this point in the ATN. You can call this + * anytime but ANTLR only generates code to check before loops + * and each iteration. + * + * Implements Jim Idle's magic sync mechanism in closures and optional * subrules. E.g., * * a : sync ( stuff sync )* ; @@ -80,5 +85,5 @@ public interface ANTLRErrorStrategy { * when it matches a valid token (indicating no longer in recovery mode) * and from its own reset method. */ - void reset(); + void endErrorCondition(); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java b/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java index e189432f7..aa07726b5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java @@ -58,7 +58,7 @@ public abstract class BaseRecognizer extends Recognizer { * matched a token. Prevents generation of more than one error message * per error. */ - protected boolean errorRecovery = false; +// protected boolean errorRecovery = false; /** Did the recognizer encounter a syntax error? Track how many. */ protected int syntaxErrors = 0; @@ -70,8 +70,8 @@ public abstract class BaseRecognizer extends Recognizer { /** reset the parser's state */ public void reset() { if ( getInputStream()!=null ) getInputStream().seek(0); - _errHandler.reset(); - errorRecovery = false; + _errHandler.endErrorCondition(); +// errorRecovery = false; _ctx = null; } @@ -92,8 +92,8 @@ public abstract class BaseRecognizer extends Recognizer { Object matchedSymbol = getCurrentInputSymbol(); if ( getInputStream().LA(1)==ttype ) { getInputStream().consume(); - errorRecovery = false; - _errHandler.reset(); +// errorRecovery = false; + _errHandler.endErrorCondition(); if ( buildParseTrees ) _ctx.addChild((Token)matchedSymbol); return matchedSymbol; } @@ -102,8 +102,8 @@ public abstract class BaseRecognizer extends Recognizer { /** Match the wildcard: in a symbol */ public void matchAny() { - errorRecovery = false; - _errHandler.reset(); +// errorRecovery = false; + _errHandler.endErrorCondition(); getInputStream().consume(); } @@ -143,32 +143,6 @@ public abstract class BaseRecognizer extends Recognizer { return traceATNStates; } - /** Report a recognition problem. - * - * This method sets errorRecovery to indicate the parser is recovering - * not parsing. Once in recovery mode, no errors are generated. - * To get out of recovery mode, the parser must successfully match - * a token (after a resync). So it will go: - * - * 1. error occurs - * 2. enter recovery mode, report error - * 3. consume until token found in resynch set - * 4. try to resume parsing - * 5. next match() will reset errorRecovery mode - */ - public void reportError(RecognitionException e) { - // if we've already reported an error and have not matched a token - // yet successfully, don't report any errors. - if ( errorRecovery ) { - //System.err.print("[SPURIOUS] "); - return; - } - syntaxErrors++; // don't count spurious - errorRecovery = true; - - notifyListeners(e.offendingToken, e.getMessage(), e); - } - /** Get number of recognition errors (lexer, parser, tree parser). Each * recognizer tracks its own number. So parser and lexer each have * separate count. Does not count the spurious errors found between diff --git a/runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java b/runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java index d1f5bf65c..8a680de21 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java +++ b/runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java @@ -23,8 +23,12 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { protected IntervalSet lastErrorStates; + protected void beginErrorCondition() { + errorRecovery = true; + } + @Override - public void reset() { + public void endErrorCondition() { errorRecovery = false; lastErrorStates = null; lastErrorIndex = -1; @@ -35,6 +39,11 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { RecognitionException e) throws RecognitionException { + // if we've already reported an error and have not matched a token + // yet successfully, don't report any errors. + if ( errorRecovery ) return; // don't count spurious errors + recognizer.syntaxErrors++; + beginErrorCondition(); if ( e instanceof NoViableAltException ) { reportNoViableAlternative(recognizer, (NoViableAltException) e); } @@ -69,22 +78,36 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { lastErrorIndex = recognizer.getInputStream().index(); if ( lastErrorStates==null ) lastErrorStates = new IntervalSet(); lastErrorStates.add(recognizer._ctx.s); - IntervalSet followSet = computeErrorRecoverySet(recognizer); + IntervalSet followSet = getErrorRecoverySet(recognizer); consumeUntil(recognizer, followSet); } + /** Make sure that the current lookahead symbol is consistent with + * what were expecting at this point in the ATN. + */ @Override public void sync(BaseRecognizer recognizer) { + System.out.println("sync"); + // TODO: CACHE THESE RESULTS!! + IntervalSet expecting = getExpectedTokens(recognizer); + // TODO: subclass this class for treeparsers + TokenStream tokens = (TokenStream)recognizer.getInputStream(); + Token la = tokens.LT(1); + if ( expecting.contains(la.getType()) ) { + endErrorCondition(); + return; + } + recoverInline(recognizer); } public void reportNoViableAlternative(BaseRecognizer recognizer, NoViableAltException e) throws RecognitionException { - if ( recognizer.errorRecovery ) return; - trackError(recognizer); - - String msg = "no viable alternative at input "+getTokenErrorDisplay(e.offendingToken); + // TODO: subclass this class for treeparsers + TokenStream tokens = (TokenStream)recognizer.getInputStream(); + String input = tokens.toString(e.startToken, e.offendingToken); + String msg = "no viable alternative at input "+escapeWSAndQuote(input); recognizer.notifyListeners(e.offendingToken, msg, e); } @@ -92,9 +115,6 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { InputMismatchException e) throws RecognitionException { - if ( recognizer.errorRecovery ) return; - trackError(recognizer); - String msg = "mismatched input "+getTokenErrorDisplay(e.offendingToken)+ " expecting "+e.getExpectedTokens().toString(recognizer.getTokenNames()); recognizer.notifyListeners(e.offendingToken, msg, e); @@ -102,11 +122,8 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { public void reportFailedPredicate(BaseRecognizer recognizer, FailedPredicateException e) - throws RecognitionException + throws RecognitionException { - if ( recognizer.errorRecovery ) return; - trackError(recognizer); - String ruleName = recognizer.getRuleNames()[recognizer._ctx.getRuleIndex()]; String msg = "rule "+ruleName+" failed predicate: {"+ e.predicateText+"}?"; @@ -114,11 +131,11 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { } public void reportUnwantedToken(BaseRecognizer recognizer) { - if ( recognizer.errorRecovery ) return; - trackError(recognizer); + if ( errorRecovery ) return; + recognizer.syntaxErrors++; + beginErrorCondition(); Token t = (Token)recognizer.getCurrentInputSymbol(); - String tokenName = getTokenErrorDisplay(t); IntervalSet expecting = getExpectedTokens(recognizer); String msg = "extraneous input "+tokenName+" expecting "+ @@ -127,8 +144,9 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { } public void reportMissingToken(BaseRecognizer recognizer) { - if ( recognizer.errorRecovery ) return; - trackError(recognizer); + if ( errorRecovery ) return; + recognizer.syntaxErrors++; + beginErrorCondition(); Token t = (Token)recognizer.getCurrentInputSymbol(); IntervalSet expecting = getExpectedTokens(recognizer); @@ -268,6 +286,10 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { s = "<"+t.getType()+">"; } } + return escapeWSAndQuote(s); + } + + protected String escapeWSAndQuote(String s) { s = s.replaceAll("\n","\\\\n"); s = s.replaceAll("\r","\\\\r"); s = s.replaceAll("\t","\\\\t"); @@ -366,7 +388,7 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { * Like Grosch I implement context-sensitive FOLLOW sets that are combined * at run-time upon error to avoid overhead during parsing. */ - protected IntervalSet computeErrorRecoverySet(BaseRecognizer recognizer) { + protected IntervalSet getErrorRecoverySet(BaseRecognizer recognizer) { ATN atn = recognizer._interp.atn; RuleContext ctx = recognizer._ctx; IntervalSet recoverSet = new IntervalSet(); @@ -393,9 +415,9 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { } } - protected void trackError(BaseRecognizer recognizer) { - recognizer.syntaxErrors++; - recognizer.errorRecovery = true; - } +// protected void trackError(BaseRecognizer recognizer) { +// recognizer.syntaxErrors++; +// recognizer.errorRecovery = true; +// } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/TreeParser.java b/runtime/Java/src/org/antlr/v4/runtime/tree/TreeParser.java index 23cf6ff90..02f991b05 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/TreeParser.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/TreeParser.java @@ -106,8 +106,7 @@ public class TreeParser extends BaseRecognizer { * corresponding UP node. */ public void matchAny(IntStream ignore) { // ignore stream, copy of input - errorRecovery = false; -// failed = false; + _errHandler.endErrorCondition(); Object look = _input.LT(1); if ( _input.getTreeAdaptor().getChildCount(look)==0 ) { _input.consume(); // not subtree, consume 1 node and return diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index 120692c2c..142857c1d 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -218,6 +218,7 @@ CodeBlockForAlt(c, locals, preamble, ops) ::= << >> LL1AltBlock(choice, preamble, alts, error) ::= << +setState(); = _input.LT(1); switch ( _input.LA(1) ) { @@ -230,6 +231,7 @@ switch ( _input.LA(1) ) { >> LL1OptionalBlock(choice, alts, error) ::= << +setState(); switch ( _input.LA(1) ) { @@ -240,6 +242,7 @@ switch ( _input.LA(1) ) { >> LL1OptionalBlockSingleAlt(choice, expr, alts, preamble, error, followExpr) ::= << +setState(); if ( ) { @@ -247,7 +250,9 @@ if ( ) { ) ) !> >> -LL1StarBlock(choice, alts, sync) ::= << +LL1StarBlock(choice, alts) ::= << +setState(); +_errHandler.sync(this); : while (true) { switch ( _input.LA(1) ) { @@ -257,21 +262,26 @@ while (true) { break ; } - // + setState(); + _errHandler.sync(this); } >> -LL1StarBlockSingleAlt(choice, loopExpr, alts, preamble, iteration, sync) ::= << +LL1StarBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= << +setState(); +_errHandler.sync(this); while ( ) { + setState(); + _errHandler.sync(this); - // } >> -LL1PlusBlock(choice, alts, iteration, loopExpr, sync, error, iterationSync) ::= << -// +LL1PlusBlock(choice, alts, iteration, loopExpr, error) ::= << +setState(); +_errHandler.sync(this); do { switch ( _input.LA(1) ) { @@ -280,26 +290,28 @@ do { default : } + setState(); + _errHandler.sync(this); - // } while ( ); >> -LL1PlusBlockSingleAlt(choice, loopExpr, alts, preamble, iteration, - sync, iterationSync) ::= -<< -// +LL1PlusBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= << +setState(); +_errHandler.sync(this); do { + setState(); + _errHandler.sync(this); -// } while ( ); >> // LL(*) stuff AltBlock(choice, preamble, alts, error) ::= << +setState(); = _input.LT(1); switch ( _interp.adaptivePredict(_input,,_ctx) ) { @@ -311,6 +323,7 @@ case : >> OptionalBlock(choice, alts, error) ::= << +setState(); switch ( _interp.adaptivePredict(_input,,_ctx) ) { : @@ -322,6 +335,8 @@ case : // TODO: we we need uniqueID? a single _alt might work StarBlock(choice, alts, sync) ::= << +setState(); +_errHandler.sync(this); int _alt = _interp.adaptivePredict(_input,,_ctx); while ( _alt!= && _alt!=-1 ) { switch ( _alt ) { @@ -330,11 +345,15 @@ case : break;}; separator="\n"> } + setState(); + _errHandler.sync(this); _alt = _interp.adaptivePredict(_input,,_ctx); } >> PlusBlock(choice, alts, error) ::= << +setState(); +_errHandler.sync(this); int _alt = _interp.adaptivePredict(_input,,_ctx); do { switch ( _alt ) { @@ -345,6 +364,8 @@ case : default : } + setState(); + _errHandler.sync(this); _alt = _interp.adaptivePredict(_input,,_ctx); } while ( _alt!= && _alt!=-1 ); >> diff --git a/tool/src/org/antlr/v4/codegen/model/LL1Loop.java b/tool/src/org/antlr/v4/codegen/model/LL1Loop.java index 3d93b79e3..6159c30a8 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1Loop.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1Loop.java @@ -39,7 +39,6 @@ import java.util.*; public abstract class LL1Loop extends Choice { @ModelElement public OutputModelObject loopExpr; @ModelElement public List iteration; - @ModelElement public Sync sync; public LL1Loop(OutputModelFactory factory, GrammarAST blkAST, diff --git a/tool/src/org/antlr/v4/codegen/model/LL1PlusBlock.java b/tool/src/org/antlr/v4/codegen/model/LL1PlusBlock.java index fc973c66c..28f012199 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1PlusBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1PlusBlock.java @@ -41,7 +41,6 @@ public class LL1PlusBlock extends LL1Loop { /** Token names for each alt 0..n-1 */ public List altLook; - @ModelElement public Sync iterationSync; public String loopLabel; public String loopCounterVar; public String[] exitLook; @@ -73,10 +72,5 @@ public class LL1PlusBlock extends LL1Loop { IntervalSet exitLookSet = altLookSets[altLookSets.length-1]; this.exitLook = gen.target.getTokenTypesAsTargetLabels(g, exitLookSet.toArray()); - - //IntervalSet iterationExpected = (IntervalSet)loopBackLook.or(exitLookSet); -// this.sync = new Sync(factory, plusRoot, loopBackLook, decision, "enter"); -// this.iterationSync = new Sync(factory, plusRoot, iterationExpected, decision, "iter"); -// this.earlyExitError = new ThrowEarlyExitException(factory, plusRoot, null); } } diff --git a/tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java b/tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java index ac74832fe..f1157dbb4 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java @@ -38,21 +38,14 @@ import java.util.List; /** */ public class LL1PlusBlockSingleAlt extends LL1Loop { - @ModelElement public Sync iterationSync; - public LL1PlusBlockSingleAlt(OutputModelFactory factory, GrammarAST blkAST, List alts) { super(factory, blkAST, alts); PlusBlockStartState plus = (PlusBlockStartState)blkAST.atnState; this.decision = plus.loopBackState.decision; IntervalSet[] altLookSets = factory.getGrammar().decisionLOOK.get(decision); - IntervalSet exitLook = altLookSets[altLookSets.length-1]; IntervalSet loopBackLook = altLookSets[1]; loopExpr = addCodeForLoopLookaheadTempVar(loopBackLook); - - this.sync = new Sync(factory, blkAST, loopBackLook, decision, "enter"); - IntervalSet iterationExpected = (IntervalSet) loopBackLook.or(exitLook); - iterationSync = new Sync(factory, blkAST, iterationExpected, decision, "iter"); } } diff --git a/tool/src/org/antlr/v4/codegen/model/LL1StarBlock.java b/tool/src/org/antlr/v4/codegen/model/LL1StarBlock.java index f7d845e27..a7116006b 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1StarBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1StarBlock.java @@ -46,7 +46,6 @@ public class LL1StarBlock extends LL1Loop { public LL1StarBlock(OutputModelFactory factory, GrammarAST blkAST, List alts) { super(factory, blkAST, alts); -// StarBlockStartState blkStart = (StarBlockStartState)blkAST.atnState; StarLoopEntryState star = (StarLoopEntryState)blkAST.atnState; this.decision = star.decision; @@ -61,7 +60,5 @@ public class LL1StarBlock extends LL1Loop { this.exitLook = factory.getGenerator().target.getTokenTypesAsTargetLabels(factory.getGrammar(), lastLook.toArray()); - -// this.sync = new Sync(factory, blkAST, expecting, decision, "iter"); } } diff --git a/tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java b/tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java index 6198bc255..a1ad68743 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java @@ -48,8 +48,5 @@ public class LL1StarBlockSingleAlt extends LL1Loop { IntervalSet enterLook = altLookSets[1]; IntervalSet exitLook = altLookSets[2]; loopExpr = addCodeForLoopLookaheadTempVar(enterLook); - - IntervalSet enterExpecting = (IntervalSet)exitLook.or(enterLook); - this.sync = new Sync(factory, starRoot, enterExpecting, decision, "iter"); } } diff --git a/tool/src/org/antlr/v4/codegen/model/StarBlock.java b/tool/src/org/antlr/v4/codegen/model/StarBlock.java index 01577d47f..efaf077d9 100644 --- a/tool/src/org/antlr/v4/codegen/model/StarBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/StarBlock.java @@ -44,7 +44,6 @@ public class StarBlock extends Loop { { super(factory, blkOrEbnfRootAST, alts); loopLabel = factory.getGenerator().target.getLoopLabel(blkOrEbnfRootAST); -// BlockStartState star = (BlockStartState)blkOrEbnfRootAST.atnState; StarLoopEntryState star = (StarLoopEntryState)blkOrEbnfRootAST.atnState; decision = star.decision; exitAlt = alts.size()+1; diff --git a/tool/test/org/antlr/v4/test/TestParseErrors.java b/tool/test/org/antlr/v4/test/TestParseErrors.java index 5f4d3062f..6549de90b 100644 --- a/tool/test/org/antlr/v4/test/TestParseErrors.java +++ b/tool/test/org/antlr/v4/test/TestParseErrors.java @@ -108,7 +108,7 @@ public class TestParseErrors extends BaseTest { ";\n" + "q : 'e' ;\n"; String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "ae", false); - String expecting = "line 1:1 no viable alternative at input 'e'\n"; + String expecting = "line 1:1 no viable alternative at input 'ae'\n"; String result = stderrDuringParse; assertEquals(expecting, result); } @@ -122,7 +122,7 @@ public class TestParseErrors extends BaseTest { "q : 'e' ;\n"; System.out.println(grammar); String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "abe", false); - String expecting = "line 1:2 no viable alternative at input 'e'\n"; + String expecting = "line 1:2 no viable alternative at input 'abe'\n"; String result = stderrDuringParse; assertEquals(expecting, result); } @@ -135,11 +135,22 @@ public class TestParseErrors extends BaseTest { ";\n" + "q : 'e' ;\n"; String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "aaae", false); - String expecting = "line 1:3 no viable alternative at input 'e'\n"; + String expecting = "line 1:3 no viable alternative at input 'aaae'\n"; String result = stderrDuringParse; assertEquals(expecting, result); } + @Test public void testSingleTokenDeletionBeforeLoop() throws Exception { + String grammar = + "grammar T;\n" + + "a : 'a' 'b'*;"; + String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "aabc", false); + String expecting = "line 1:1 extraneous input 'a' expecting {, 'b'}\n"; + String result = stderrDuringParse; + assertEquals(expecting, result); + } + + @Test public void testLL1ErrorInfo() throws Exception { String grammar = "grammar T;\n" +