From eaaa98bde92b8ec9a0964621b05bd6440f5ef998 Mon Sep 17 00:00:00 2001 From: parrt Date: Sat, 22 May 2010 12:23:36 -0800 Subject: [PATCH] got sync in [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 6883] --- .../org/antlr/v4/runtime/BaseRecognizer.java | 11 ++++++++-- .../v4/tool/templates/codegen/Java/Java.stg | 20 +++++++++++-------- .../antlr/v4/analysis/AnalysisPipeline.java | 2 +- .../antlr/v4/analysis/LinearApproximator.java | 16 ++++++++++----- tool/src/org/antlr/v4/automata/NFA.java | 2 ++ tool/src/org/antlr/v4/codegen/src/Choice.java | 7 +++++-- .../org/antlr/v4/codegen/src/InvokeRule.java | 4 ++-- .../src/org/antlr/v4/codegen/src/LL1Loop.java | 6 ++---- .../v4/codegen/src/LL1PlusBlockSingleAlt.java | 2 +- .../v4/codegen/src/LL1StarBlockSingleAlt.java | 2 +- .../org/antlr/v4/codegen/src/MatchToken.java | 3 ++- tool/src/org/antlr/v4/codegen/src/Sync.java | 15 ++++++++++++++ .../test/TestLinearApproximateLookahead.java | 2 +- 13 files changed, 64 insertions(+), 28 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java b/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java index ca706ad91..5b339fd59 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java @@ -83,7 +83,6 @@ public abstract class BaseRecognizer { } } - /** Match current input symbol against ttype. Attempt * single token insertion or deletion error recovery. If * that fails, throw MismatchedTokenException. @@ -115,6 +114,15 @@ public abstract class BaseRecognizer { return matchedSymbol; } + // like matchSet but w/o consume; error checking routine. + public void sync(LABitSet expecting) { + if ( expecting.member(state.input.LA(1)) ) return; + LABitSet followSet = computeErrorRecoverySet(); + followSet.orInPlace(expecting); + NoViableAltException e = new NoViableAltException(this, followSet); + recoverFromMismatchedSet(e, followSet); + } + /** Match the wildcard: in a symbol */ public void matchAny() { state.errorRecovery = false; @@ -601,7 +609,6 @@ public abstract class BaseRecognizer { throw e; } - /** Not currently used */ public Object recoverFromMismatchedSet(RecognitionException e, LABitSet follow) throws RecognitionException diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index e400d0236..6800a1524 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -121,7 +121,7 @@ else { } >> -LL1StarBlock(choice, alts, error) ::= << +LL1StarBlock(choice, alts, error, sync) ::= << : while (true) { switch ( state.input.LA(1) ) { @@ -130,21 +130,22 @@ while (true) { break;}; separator="\n"> break ; - default : - } + } >> -LL1StarBlockSingleAlt(choice, expr, alts, preamble, iteration, error) ::= << +LL1StarBlockSingleAlt(choice, expr, alts, preamble, iteration, error, sync) ::= << while ( ) { + } >> -LL1PlusBlock(choice, alts, error, earlyExitError) ::= << +LL1PlusBlock(choice, alts, error, earlyExitError, sync) ::= << + int = 0; : while (true) { @@ -155,22 +156,25 @@ while (true) { if ( >= 1 ) break ; else - default : - } + } >> LL1PlusBlockSingleAlt(choice, expr, alts, preamble, iteration, - error, earlyExitError) ::= + error, earlyExitError, sync) ::= << + do { + } while ( ); >> +Sync(s) ::= "sync();" + ThrowNoViableAlt(t) ::= "throw new NoViableAlt(this, );" ThrowEarlyExitException(t) ::= "throw new ThrowEarlyExitException(this, );" diff --git a/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java b/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java index 9e23b7742..029a4a5c2 100644 --- a/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java +++ b/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java @@ -51,7 +51,7 @@ public class AnalysisPipeline { System.out.println("\nDECISION "+s.decision); // TRY LINEAR APPROX FIXED LOOKAHEAD FIRST - LinearApproximator lin = new LinearApproximator(g); + LinearApproximator lin = new LinearApproximator(g, s.decision); DFA dfa = lin.createDFA(s); // IF NOT LINEAR APPROX, TRY NFA TO DFA CONVERSION diff --git a/tool/src/org/antlr/v4/analysis/LinearApproximator.java b/tool/src/org/antlr/v4/analysis/LinearApproximator.java index 8619bd03f..1c42b2be6 100644 --- a/tool/src/org/antlr/v4/analysis/LinearApproximator.java +++ b/tool/src/org/antlr/v4/analysis/LinearApproximator.java @@ -1,5 +1,6 @@ package org.antlr.v4.analysis; +import org.antlr.runtime.Token; import org.antlr.v4.automata.*; import org.antlr.v4.misc.IntervalSet; import org.antlr.v4.misc.OrderedHashSet; @@ -65,13 +66,13 @@ public class LinearApproximator { */ OrderedHashSet[] configs; - public LinearApproximator(Grammar g) { + public LinearApproximator(Grammar g, int decision) { this.g = g; this.decision = decision; } - public LinearApproximator(Grammar g, int k) { - this(g); + public LinearApproximator(Grammar g, int decision, int k) { + this(g, decision); max_k = k; } @@ -212,6 +213,10 @@ public class LinearApproximator { } } + /** Compute FOLLOW of element but don't leave rule to compute global + * context-free FOLLOW. Used for rule invocation, match token, and + * error sync'ing. + */ public IntervalSet LOOK(NFAState s) { System.out.println("LOOK("+s.stateNumber+")"); lookBusy.clear(); @@ -226,8 +231,9 @@ public class LinearApproximator { if ( lookBusy.contains(ac) ) return; lookBusy.add(ac); - if ( s instanceof RuleStopState && !context.isEmpty() ) { - _LOOK(context.returnState, context.parent, fset); + if ( s instanceof RuleStopState ) { + if ( !context.isEmpty() ) _LOOK(context.returnState, context.parent, fset); + else fset.add(Token.EOR_TOKEN_TYPE); // hit end of rule return; } diff --git a/tool/src/org/antlr/v4/automata/NFA.java b/tool/src/org/antlr/v4/automata/NFA.java index 54b875bac..03a483dd8 100644 --- a/tool/src/org/antlr/v4/automata/NFA.java +++ b/tool/src/org/antlr/v4/automata/NFA.java @@ -11,6 +11,7 @@ import java.util.Map; /** */ public class NFA { public static final int INVALID_ALT_NUMBER = -1; + public static final int INVALID_DECISION_NUMBER = -1; public Grammar g; public List states = new ArrayList(); @@ -38,6 +39,7 @@ public class NFA { public int defineDecisionState(DecisionState s) { decisionToNFAState.add(s); s.decision = decisionToNFAState.size()-1; + System.out.println("dec state "+s.stateNumber+" gets dec # "+s.decision); return s.decision; } } diff --git a/tool/src/org/antlr/v4/codegen/src/Choice.java b/tool/src/org/antlr/v4/codegen/src/Choice.java index 5cd8b97c7..26d6169b8 100644 --- a/tool/src/org/antlr/v4/codegen/src/Choice.java +++ b/tool/src/org/antlr/v4/codegen/src/Choice.java @@ -17,18 +17,21 @@ public abstract class Choice extends SrcOp { public List preamble; public IntervalSet expecting; public ThrowNoViableAlt error; + public Sync sync; public Choice(OutputModelFactory factory, GrammarAST blkOrEbnfRootAST, List alts) { super(factory, blkOrEbnfRootAST); this.alts = alts; this.decision = ((BlockStartState)blkOrEbnfRootAST.nfaState).decision; - LinearApproximator approx = new LinearApproximator(factory.g); + // TODO: use existing lookahead! don't compute + LinearApproximator approx = new LinearApproximator(factory.g, decision); NFAState decisionState = ast.nfaState; expecting = approx.LOOK(decisionState); - System.out.println("expecting="+expecting); + System.out.println(blkOrEbnfRootAST.toStringTree()+" loop expecting="+expecting); this.error = new ThrowNoViableAlt(factory, blkOrEbnfRootAST, expecting); + this.sync = new Sync(factory, blkOrEbnfRootAST, expecting); } public void addPreambleOp(SrcOp op) { diff --git a/tool/src/org/antlr/v4/codegen/src/InvokeRule.java b/tool/src/org/antlr/v4/codegen/src/InvokeRule.java index 4c0aaf6b4..f0a20f1cf 100644 --- a/tool/src/org/antlr/v4/codegen/src/InvokeRule.java +++ b/tool/src/org/antlr/v4/codegen/src/InvokeRule.java @@ -1,6 +1,7 @@ package org.antlr.v4.codegen.src; import org.antlr.v4.analysis.LinearApproximator; +import org.antlr.v4.automata.NFA; import org.antlr.v4.codegen.OutputModelFactory; import org.antlr.v4.codegen.RuleContextDecl; import org.antlr.v4.misc.IntervalSet; @@ -47,8 +48,7 @@ public class InvokeRule extends SrcOp implements LabeledOp { factory.currentRule.peek().addDecl(d); } - // compute follow - LinearApproximator approx = new LinearApproximator(factory.g); + LinearApproximator approx = new LinearApproximator(factory.g, NFA.INVALID_DECISION_NUMBER); IntervalSet fset = approx.LOOK(ast.nfaState.transition(0).target); System.out.println("follow="+follow); follow = factory.createFollowBitSet(ast, fset); diff --git a/tool/src/org/antlr/v4/codegen/src/LL1Loop.java b/tool/src/org/antlr/v4/codegen/src/LL1Loop.java index f9f44a4ef..58e0eb1ac 100644 --- a/tool/src/org/antlr/v4/codegen/src/LL1Loop.java +++ b/tool/src/org/antlr/v4/codegen/src/LL1Loop.java @@ -9,7 +9,7 @@ import java.util.List; /** */ public abstract class LL1Loop extends LL1Choice { - public Object expr; + public OutputModelObject expr; public List iteration; public LL1Loop(OutputModelFactory factory, GrammarAST blkAST, List alts) { @@ -21,7 +21,7 @@ public abstract class LL1Loop extends LL1Choice { iteration.add(op); } - public void addLookaheadTempVar(IntervalSet look) { + public void addCodeForLookaheadTempVar(IntervalSet look) { expr = factory.getLL1Test(look, ast); if ( expr instanceof TestSetInline ) { TestSetInline e = (TestSetInline) expr; @@ -30,8 +30,6 @@ public abstract class LL1Loop extends LL1Choice { CaptureNextToken nextToken = new CaptureNextToken(e.varName); addPreambleOp(nextToken); addIterationOp(nextToken); - iteration = new ArrayList(); - iteration.add(nextToken); } } diff --git a/tool/src/org/antlr/v4/codegen/src/LL1PlusBlockSingleAlt.java b/tool/src/org/antlr/v4/codegen/src/LL1PlusBlockSingleAlt.java index bd871f4f1..7140b9ac9 100644 --- a/tool/src/org/antlr/v4/codegen/src/LL1PlusBlockSingleAlt.java +++ b/tool/src/org/antlr/v4/codegen/src/LL1PlusBlockSingleAlt.java @@ -12,7 +12,7 @@ public class LL1PlusBlockSingleAlt extends LL1Loop { public LL1PlusBlockSingleAlt(OutputModelFactory factory, GrammarAST blkAST, List alts) { super(factory, blkAST, alts); IntervalSet loopBackLook = altLookSets[2]; // loop exit is alt 1 - addLookaheadTempVar(loopBackLook); + addCodeForLookaheadTempVar(loopBackLook); } // @Override // public List getChildren() { diff --git a/tool/src/org/antlr/v4/codegen/src/LL1StarBlockSingleAlt.java b/tool/src/org/antlr/v4/codegen/src/LL1StarBlockSingleAlt.java index f4dc954a2..6c5ffb918 100644 --- a/tool/src/org/antlr/v4/codegen/src/LL1StarBlockSingleAlt.java +++ b/tool/src/org/antlr/v4/codegen/src/LL1StarBlockSingleAlt.java @@ -11,6 +11,6 @@ public class LL1StarBlockSingleAlt extends LL1Loop { public LL1StarBlockSingleAlt(OutputModelFactory factory, GrammarAST blkAST, List alts) { super(factory, blkAST, alts); IntervalSet look = altLookSets[1]; - addLookaheadTempVar(look); + addCodeForLookaheadTempVar(look); } } diff --git a/tool/src/org/antlr/v4/codegen/src/MatchToken.java b/tool/src/org/antlr/v4/codegen/src/MatchToken.java index 30db45ae0..44ac06de3 100644 --- a/tool/src/org/antlr/v4/codegen/src/MatchToken.java +++ b/tool/src/org/antlr/v4/codegen/src/MatchToken.java @@ -1,6 +1,7 @@ package org.antlr.v4.codegen.src; import org.antlr.v4.analysis.LinearApproximator; +import org.antlr.v4.automata.NFA; import org.antlr.v4.codegen.OutputModelFactory; import org.antlr.v4.misc.IntervalSet; import org.antlr.v4.parse.ANTLRParser; @@ -39,7 +40,7 @@ public class MatchToken extends SrcOp implements LabeledOp { factory.currentRule.peek().addDecl(d); } - LinearApproximator approx = new LinearApproximator(factory.g); + LinearApproximator approx = new LinearApproximator(factory.g, NFA.INVALID_DECISION_NUMBER); IntervalSet fset = approx.LOOK(ast.nfaState.transition(0).target); System.out.println("follow="+fset); follow = factory.createFollowBitSet(ast, fset); diff --git a/tool/src/org/antlr/v4/codegen/src/Sync.java b/tool/src/org/antlr/v4/codegen/src/Sync.java index 764c90ecc..11e97ff6c 100644 --- a/tool/src/org/antlr/v4/codegen/src/Sync.java +++ b/tool/src/org/antlr/v4/codegen/src/Sync.java @@ -1,5 +1,20 @@ package org.antlr.v4.codegen.src; +import org.antlr.v4.automata.BlockStartState; +import org.antlr.v4.codegen.OutputModelFactory; +import org.antlr.v4.misc.IntervalSet; +import org.antlr.v4.tool.GrammarAST; + /** */ public class Sync extends SrcOp { + public int decision; + public BitSetDecl expecting; + public Sync(OutputModelFactory factory, GrammarAST blkOrEbnfRootAST, + IntervalSet expecting) + { + super(factory, blkOrEbnfRootAST); + this.decision = ((BlockStartState)blkOrEbnfRootAST.nfaState).decision; + this.expecting = factory.createExpectingBitSet(ast, decision, expecting); + factory.defineBitSet(this.expecting); + } } diff --git a/tool/test/org/antlr/v4/test/TestLinearApproximateLookahead.java b/tool/test/org/antlr/v4/test/TestLinearApproximateLookahead.java index f37cd6a69..a4f9b9ec0 100644 --- a/tool/test/org/antlr/v4/test/TestLinearApproximateLookahead.java +++ b/tool/test/org/antlr/v4/test/TestLinearApproximateLookahead.java @@ -102,7 +102,7 @@ public class TestLinearApproximateLookahead extends BaseTest { return; } DecisionState blk = (DecisionState)s.transition(0).target; - LinearApproximator lin = new LinearApproximator(g); + LinearApproximator lin = new LinearApproximator(g,blk.decision); DFA dfa = lin.createDFA(blk); String result = null; if ( dfa!=null ) result = dfa.toString();