forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

got sync in 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9101]
This commit is contained in:
parrt 2011-10-02 20:31:26 -08:00
parent bf48ca5b73
commit 8508dd6da0
12 changed files with 107 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java
View File

@ -52,7 +52,12 @@ public interface ANTLRErrorStrategy {
*/
void recover(BaseRecognizer recognizer);
/** Implement Jim Idle's magic sync mechanism in closures and optional
/** Make sure that the current lookahead symbol is consistent with
* what were expecting at this point in the ATN. You can call this
* anytime but ANTLR only generates code to check before loops
* and each iteration.
*
* Implements Jim Idle's magic sync mechanism in closures and optional
* subrules. E.g.,
*
* a : sync ( stuff sync )* ;
@ -80,5 +85,5 @@ public interface ANTLRErrorStrategy {
* when it matches a valid token (indicating no longer in recovery mode)
* and from its own reset method.
*/
void reset();
void endErrorCondition();
}

40
runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java
View File

@ -58,7 +58,7 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
* matched a token. Prevents generation of more than one error message
* per error.
*/
protected boolean errorRecovery = false;
// protected boolean errorRecovery = false;
/** Did the recognizer encounter a syntax error? Track how many. */
protected int syntaxErrors = 0;
@ -70,8 +70,8 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
/** reset the parser's state */
public void reset() {
if ( getInputStream()!=null ) getInputStream().seek(0);
_errHandler.reset();
errorRecovery = false;
_errHandler.endErrorCondition();
// errorRecovery = false;
_ctx = null;
}
@ -92,8 +92,8 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
Object matchedSymbol = getCurrentInputSymbol();
if ( getInputStream().LA(1)==ttype ) {
getInputStream().consume();
errorRecovery = false;
_errHandler.reset();
// errorRecovery = false;
_errHandler.endErrorCondition();
if ( buildParseTrees ) _ctx.addChild((Token)matchedSymbol);
return matchedSymbol;
}
@ -102,8 +102,8 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
/** Match the wildcard: in a symbol */
public void matchAny() {
errorRecovery = false;
_errHandler.reset();
// errorRecovery = false;
_errHandler.endErrorCondition();
getInputStream().consume();
}
@ -143,32 +143,6 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
return traceATNStates;
}
/** Report a recognition problem.
*
* This method sets errorRecovery to indicate the parser is recovering
* not parsing. Once in recovery mode, no errors are generated.
* To get out of recovery mode, the parser must successfully match
* a token (after a resync). So it will go:
*
* 1. error occurs
* 2. enter recovery mode, report error
* 3. consume until token found in resynch set
* 4. try to resume parsing
* 5. next match() will reset errorRecovery mode
*/
public void reportError(RecognitionException e) {
// if we've already reported an error and have not matched a token
// yet successfully, don't report any errors.
if ( errorRecovery ) {
//System.err.print("[SPURIOUS] ");
return;
}
syntaxErrors++; // don't count spurious
errorRecovery = true;
notifyListeners(e.offendingToken, e.getMessage(), e);
}
/** Get number of recognition errors (lexer, parser, tree parser). Each
* recognizer tracks its own number. So parser and lexer each have
* separate count. Does not count the spurious errors found between

68
runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java
View File

@ -23,8 +23,12 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
protected IntervalSet lastErrorStates;
protected void beginErrorCondition() {
errorRecovery = true;
}
@Override
public void reset() {
public void endErrorCondition() {
errorRecovery = false;
lastErrorStates = null;
lastErrorIndex = -1;
@ -35,6 +39,11 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
RecognitionException e)
throws RecognitionException
{
// if we've already reported an error and have not matched a token
// yet successfully, don't report any errors.
if ( errorRecovery ) return; // don't count spurious errors
recognizer.syntaxErrors++;
beginErrorCondition();
if ( e instanceof NoViableAltException ) {
reportNoViableAlternative(recognizer, (NoViableAltException) e);
}
@ -69,22 +78,36 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
lastErrorIndex = recognizer.getInputStream().index();
if ( lastErrorStates==null ) lastErrorStates = new IntervalSet();
lastErrorStates.add(recognizer._ctx.s);
IntervalSet followSet = computeErrorRecoverySet(recognizer);
IntervalSet followSet = getErrorRecoverySet(recognizer);
consumeUntil(recognizer, followSet);
}
/** Make sure that the current lookahead symbol is consistent with
* what were expecting at this point in the ATN.
*/
@Override
public void sync(BaseRecognizer recognizer) {
System.out.println("sync");
// TODO: CACHE THESE RESULTS!!
IntervalSet expecting = getExpectedTokens(recognizer);
// TODO: subclass this class for treeparsers
TokenStream tokens = (TokenStream)recognizer.getInputStream();
Token la = tokens.LT(1);
if ( expecting.contains(la.getType()) ) {
endErrorCondition();
return;
}
recoverInline(recognizer);
}
public void reportNoViableAlternative(BaseRecognizer recognizer,
NoViableAltException e)
throws RecognitionException
{
if ( recognizer.errorRecovery ) return;
trackError(recognizer);
String msg = "no viable alternative at input "+getTokenErrorDisplay(e.offendingToken);
// TODO: subclass this class for treeparsers
TokenStream tokens = (TokenStream)recognizer.getInputStream();
String input = tokens.toString(e.startToken, e.offendingToken);
String msg = "no viable alternative at input "+escapeWSAndQuote(input);
recognizer.notifyListeners(e.offendingToken, msg, e);
}
@ -92,9 +115,6 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
InputMismatchException e)
throws RecognitionException
{
if ( recognizer.errorRecovery ) return;
trackError(recognizer);
String msg = "mismatched input "+getTokenErrorDisplay(e.offendingToken)+
" expecting "+e.getExpectedTokens().toString(recognizer.getTokenNames());
recognizer.notifyListeners(e.offendingToken, msg, e);
@ -102,11 +122,8 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
public void reportFailedPredicate(BaseRecognizer recognizer,
FailedPredicateException e)
throws RecognitionException
throws RecognitionException
{
if ( recognizer.errorRecovery ) return;
trackError(recognizer);
String ruleName = recognizer.getRuleNames()[recognizer._ctx.getRuleIndex()];
String msg = "rule "+ruleName+" failed predicate: {"+
e.predicateText+"}?";
@ -114,11 +131,11 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
}
public void reportUnwantedToken(BaseRecognizer recognizer) {
if ( recognizer.errorRecovery ) return;
trackError(recognizer);
if ( errorRecovery ) return;
recognizer.syntaxErrors++;
beginErrorCondition();
Token t = (Token)recognizer.getCurrentInputSymbol();
String tokenName = getTokenErrorDisplay(t);
IntervalSet expecting = getExpectedTokens(recognizer);
String msg = "extraneous input "+tokenName+" expecting "+
@ -127,8 +144,9 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
}
public void reportMissingToken(BaseRecognizer recognizer) {
if ( recognizer.errorRecovery ) return;
trackError(recognizer);
if ( errorRecovery ) return;
recognizer.syntaxErrors++;
beginErrorCondition();
Token t = (Token)recognizer.getCurrentInputSymbol();
IntervalSet expecting = getExpectedTokens(recognizer);
@ -268,6 +286,10 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
s = "<"+t.getType()+">";
}
}
return escapeWSAndQuote(s);
}
protected String escapeWSAndQuote(String s) {
s = s.replaceAll("\n","\\\\n");
s = s.replaceAll("\r","\\\\r");
s = s.replaceAll("\t","\\\\t");
@ -366,7 +388,7 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
* Like Grosch I implement context-sensitive FOLLOW sets that are combined
* at run-time upon error to avoid overhead during parsing.
*/
protected IntervalSet computeErrorRecoverySet(BaseRecognizer recognizer) {
protected IntervalSet getErrorRecoverySet(BaseRecognizer recognizer) {
ATN atn = recognizer._interp.atn;
RuleContext ctx = recognizer._ctx;
IntervalSet recoverSet = new IntervalSet();
@ -393,9 +415,9 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
}
}
protected void trackError(BaseRecognizer recognizer) {
recognizer.syntaxErrors++;
recognizer.errorRecovery = true;
}
// protected void trackError(BaseRecognizer recognizer) {
// recognizer.syntaxErrors++;
// recognizer.errorRecovery = true;
// }
}

3
runtime/Java/src/org/antlr/v4/runtime/tree/TreeParser.java
View File

@ -106,8 +106,7 @@ public class TreeParser extends BaseRecognizer {
* corresponding UP node.
*/
public void matchAny(IntStream ignore) { // ignore stream, copy of input
errorRecovery = false;
// failed = false;
_errHandler.endErrorCondition();
Object look = _input.LT(1);
if ( _input.getTreeAdaptor().getChildCount(look)==0 ) {
_input.consume(); // not subtree, consume 1 node and return

45
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -218,6 +218,7 @@ CodeBlockForAlt(c, locals, preamble, ops) ::= <<
>>
LL1AltBlock(choice, preamble, alts, error) ::= <<
setState(<choice.stateNumber>);
<if(choice.label)><labelref(choice.label)> = _input.LT(1);<endif>
<preamble; separator="\n">
switch ( _input.LA(1) ) {
@ -230,6 +231,7 @@ switch ( _input.LA(1) ) {
>>
LL1OptionalBlock(choice, alts, error) ::= <<
setState(<choice.stateNumber>);
switch ( _input.LA(1) ) {
<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
<alt>
@ -240,6 +242,7 @@ switch ( _input.LA(1) ) {
>>
LL1OptionalBlockSingleAlt(choice, expr, alts, preamble, error, followExpr) ::= <<
setState(<choice.stateNumber>);
<preamble; separator="\n">
if ( <expr> ) {
<alts; separator="\n">
@ -247,7 +250,9 @@ if ( <expr> ) {
<!else if ( !(<followExpr>) ) <error>!>
>>
LL1StarBlock(choice, alts, sync) ::= <<
LL1StarBlock(choice, alts) ::= <<
setState(<choice.stateNumber>);
_errHandler.sync(this);
<choice.loopLabel>:
while (true) {
switch ( _input.LA(1) ) {
@ -257,21 +262,26 @@ while (true) {
<cases(choice.exitLook)>
break <choice.loopLabel>;
}
//<sync>
setState(<choice.stateNumber>);
_errHandler.sync(this);
}
>>
LL1StarBlockSingleAlt(choice, loopExpr, alts, preamble, iteration, sync) ::= <<
LL1StarBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= <<
setState(<choice.stateNumber>);
_errHandler.sync(this);
<preamble; separator="\n">
while ( <loopExpr> ) {
<alts; separator="\n">
setState(<choice.stateNumber>);
_errHandler.sync(this);
<iteration>
//<sync>
}
>>
LL1PlusBlock(choice, alts, iteration, loopExpr, sync, error, iterationSync) ::= <<
//<sync>
LL1PlusBlock(choice, alts, iteration, loopExpr, error) ::= <<
setState(<choice.stateNumber>);
_errHandler.sync(this);
do {
switch ( _input.LA(1) ) {
<choice.altLook,alts:{look,alt| <cases(look)>
@ -280,26 +290,28 @@ do {
default :
<error>
}
setState(<choice.stateNumber>);
_errHandler.sync(this);
<iteration>
//<iterationSync>
} while ( <loopExpr> );
>>
LL1PlusBlockSingleAlt(choice, loopExpr, alts, preamble, iteration,
sync, iterationSync) ::=
<<
//<sync>
LL1PlusBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= <<
setState(<choice.stateNumber>);
_errHandler.sync(this);
<preamble; separator="\n">
do {
<alts; separator="\n">
setState(<choice.stateNumber>);
_errHandler.sync(this);
<iteration>
// <iterationSync>
} while ( <loopExpr> );
>>
// LL(*) stuff
AltBlock(choice, preamble, alts, error) ::= <<
setState(<choice.stateNumber>);
<if(choice.label)><labelref(choice.label)> = _input.LT(1);<endif>
<preamble; separator="\n">
switch ( _interp.adaptivePredict(_input,<choice.decision>,_ctx) ) {
@ -311,6 +323,7 @@ case <i>:
>>
OptionalBlock(choice, alts, error) ::= <<
setState(<choice.stateNumber>);
switch ( _interp.adaptivePredict(_input,<choice.decision>,_ctx) ) {
<alts:{alt |
case <i>:
@ -322,6 +335,8 @@ case <i>:
// TODO: we we need uniqueID? a single _alt might work
StarBlock(choice, alts, sync) ::= <<
setState(<choice.stateNumber>);
_errHandler.sync(this);
int _alt<choice.uniqueID> = _interp.adaptivePredict(_input,<choice.decision>,_ctx);
while ( _alt<choice.uniqueID>!=<choice.exitAlt> && _alt<choice.uniqueID>!=-1 ) {
switch ( _alt<choice.uniqueID> ) {
@ -330,11 +345,15 @@ case <i>:
<alt>
break;}; separator="\n">
}
setState(<choice.stateNumber>);
_errHandler.sync(this);
_alt<choice.uniqueID> = _interp.adaptivePredict(_input,<choice.decision>,_ctx);
}
>>
PlusBlock(choice, alts, error) ::= <<
setState(<choice.stateNumber>);
_errHandler.sync(this);
int _alt<choice.uniqueID> = _interp.adaptivePredict(_input,<choice.decision>,_ctx);
do {
switch ( _alt<choice.uniqueID> ) {
@ -345,6 +364,8 @@ case <i>:
default :
<error>
}
setState(<choice.stateNumber>);
_errHandler.sync(this);
_alt<choice.uniqueID> = _interp.adaptivePredict(_input,<choice.decision>,_ctx);
} while ( _alt<choice.uniqueID>!=<choice.exitAlt> && _alt<choice.uniqueID>!=-1 );
>>

1
tool/src/org/antlr/v4/codegen/model/LL1Loop.java
View File

@ -39,7 +39,6 @@ import java.util.*;
public abstract class LL1Loop extends Choice {
@ModelElement public OutputModelObject loopExpr;
@ModelElement public List<SrcOp> iteration;
@ModelElement public Sync sync;
public LL1Loop(OutputModelFactory factory,
GrammarAST blkAST,

6
tool/src/org/antlr/v4/codegen/model/LL1PlusBlock.java
View File

@ -41,7 +41,6 @@ public class LL1PlusBlock extends LL1Loop {
/** Token names for each alt 0..n-1 */
public List<String[]> altLook;
@ModelElement public Sync iterationSync;
public String loopLabel;
public String loopCounterVar;
public String[] exitLook;
@ -73,10 +72,5 @@ public class LL1PlusBlock extends LL1Loop {
IntervalSet exitLookSet = altLookSets[altLookSets.length-1];
this.exitLook = gen.target.getTokenTypesAsTargetLabels(g,
exitLookSet.toArray());
//IntervalSet iterationExpected = (IntervalSet)loopBackLook.or(exitLookSet);
// this.sync = new Sync(factory, plusRoot, loopBackLook, decision, "enter");
// this.iterationSync = new Sync(factory, plusRoot, iterationExpected, decision, "iter");
// this.earlyExitError = new ThrowEarlyExitException(factory, plusRoot, null);
}
}

7
tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java
View File

@ -38,21 +38,14 @@ import java.util.List;
/** */
public class LL1PlusBlockSingleAlt extends LL1Loop {
@ModelElement public Sync iterationSync;
public LL1PlusBlockSingleAlt(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlockForAlt> alts) {
super(factory, blkAST, alts);
PlusBlockStartState plus = (PlusBlockStartState)blkAST.atnState;
this.decision = plus.loopBackState.decision;
IntervalSet[] altLookSets = factory.getGrammar().decisionLOOK.get(decision);
IntervalSet exitLook = altLookSets[altLookSets.length-1];
IntervalSet loopBackLook = altLookSets[1];
loopExpr = addCodeForLoopLookaheadTempVar(loopBackLook);
this.sync = new Sync(factory, blkAST, loopBackLook, decision, "enter");
IntervalSet iterationExpected = (IntervalSet) loopBackLook.or(exitLook);
iterationSync = new Sync(factory, blkAST, iterationExpected, decision, "iter");
}
}

3
tool/src/org/antlr/v4/codegen/model/LL1StarBlock.java
View File

@ -46,7 +46,6 @@ public class LL1StarBlock extends LL1Loop {
public LL1StarBlock(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlockForAlt> alts) {
super(factory, blkAST, alts);
// StarBlockStartState blkStart = (StarBlockStartState)blkAST.atnState;
StarLoopEntryState star = (StarLoopEntryState)blkAST.atnState;
this.decision = star.decision;
@ -61,7 +60,5 @@ public class LL1StarBlock extends LL1Loop {
this.exitLook =
factory.getGenerator().target.getTokenTypesAsTargetLabels(factory.getGrammar(), lastLook.toArray());
// this.sync = new Sync(factory, blkAST, expecting, decision, "iter");
}
}

3
tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java
View File

@ -48,8 +48,5 @@ public class LL1StarBlockSingleAlt extends LL1Loop {
IntervalSet enterLook = altLookSets[1];
IntervalSet exitLook = altLookSets[2];
loopExpr = addCodeForLoopLookaheadTempVar(enterLook);
IntervalSet enterExpecting = (IntervalSet)exitLook.or(enterLook);
this.sync = new Sync(factory, starRoot, enterExpecting, decision, "iter");
}
}

1
tool/src/org/antlr/v4/codegen/model/StarBlock.java
View File

@ -44,7 +44,6 @@ public class StarBlock extends Loop {
{
super(factory, blkOrEbnfRootAST, alts);
loopLabel = factory.getGenerator().target.getLoopLabel(blkOrEbnfRootAST);
// BlockStartState star = (BlockStartState)blkOrEbnfRootAST.atnState;
StarLoopEntryState star = (StarLoopEntryState)blkOrEbnfRootAST.atnState;
decision = star.decision;
exitAlt = alts.size()+1;

17
tool/test/org/antlr/v4/test/TestParseErrors.java
View File

@ -108,7 +108,7 @@ public class TestParseErrors extends BaseTest {
";\n" +
"q : 'e' ;\n";
String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "ae", false);
String expecting = "line 1:1 no viable alternative at input 'e'\n";
String expecting = "line 1:1 no viable alternative at input 'ae'\n";
String result = stderrDuringParse;
assertEquals(expecting, result);
}
@ -122,7 +122,7 @@ public class TestParseErrors extends BaseTest {
"q : 'e' ;\n";
System.out.println(grammar);
String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "abe", false);
String expecting = "line 1:2 no viable alternative at input 'e'\n";
String expecting = "line 1:2 no viable alternative at input 'abe'\n";
String result = stderrDuringParse;
assertEquals(expecting, result);
}
@ -135,11 +135,22 @@ public class TestParseErrors extends BaseTest {
";\n" +
"q : 'e' ;\n";
String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "aaae", false);
String expecting = "line 1:3 no viable alternative at input 'e'\n";
String expecting = "line 1:3 no viable alternative at input 'aaae'\n";
String result = stderrDuringParse;
assertEquals(expecting, result);
}
@Test public void testSingleTokenDeletionBeforeLoop() throws Exception {
String grammar =
"grammar T;\n" +
"a : 'a' 'b'*;";
String found = execParser("T.g", grammar, "TParser", "TLexer", "a", "aabc", false);
String expecting = "line 1:1 extraneous input 'a' expecting {<EOF>, 'b'}\n";
String result = stderrDuringParse;
assertEquals(expecting, result);
}
@Test public void testLL1ErrorInfo() throws Exception {
String grammar =
"grammar T;\n" +