From 6b4e9905fb066f55ff309926e40962f9874d5ede Mon Sep 17 00:00:00 2001 From: parrt Date: Fri, 30 Sep 2011 20:40:14 -0800 Subject: [PATCH] rename member, tweak output. new chk for inf loop from error sync works [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9088] --- .../org/antlr/v4/runtime/BaseRecognizer.java | 14 ++++------- .../v4/runtime/DefaultANTLRErrorStrategy.java | 25 +++++++++++-------- .../src/org/antlr/v4/runtime/Recognizer.java | 6 ++--- .../v4/runtime/atn/LexerATNSimulator.java | 2 +- .../v4/runtime/atn/ParserATNSimulator.java | 2 +- .../src/org/antlr/v4/runtime/misc/IntSet.java | 2 +- .../antlr/v4/runtime/misc/IntervalSet.java | 2 +- .../runtime/tree/BufferedASTNodeStream.java | 16 +++++------- tool/playground/U.g | 4 +-- .../v4/tool/templates/codegen/Java/Java.stg | 2 +- .../v4/automata/TreeParserATNFactory.java | 18 ++++++------- .../org/antlr/v4/codegen/model/MatchTree.java | 13 ++++------ tool/src/org/antlr/v4/tool/GrammarAST.java | 2 +- .../org/antlr/v4/test/TestIntervalSet.java | 14 +++++------ 14 files changed, 55 insertions(+), 67 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java b/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java index 2689452af..c755ef545 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java @@ -28,14 +28,10 @@ */ package org.antlr.v4.runtime; -import org.antlr.v4.runtime.atn.ATNConfig; -import org.antlr.v4.runtime.atn.ParserATNSimulator; -import org.antlr.v4.runtime.misc.IntervalSet; -import org.antlr.v4.runtime.misc.OrderedHashSet; +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.misc.*; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; +import java.util.*; /** A generic recognizer that can handle recognizers generated from * parser and tree grammars. This is all the parsing @@ -111,7 +107,7 @@ public abstract class BaseRecognizer extends Recognizer { // like matchSet but w/o consume; error checking routine. public void sync(IntervalSet expecting) { - if ( expecting.member(getInputStream().LA(1)) ) return; + if ( expecting.contains(getInputStream().LA(1)) ) return; // System.out.println("failed sync to "+expecting); IntervalSet followSet = computeErrorRecoverySet(); followSet.addAll(expecting); @@ -582,7 +578,7 @@ public abstract class BaseRecognizer extends Recognizer { public void consumeUntil(IntervalSet set) { //System.out.println("consumeUntil("+set.toString(getTokenNames())+")"); int ttype = getInputStream().LA(1); - while (ttype != Token.EOF && !set.member(ttype) ) { + while (ttype != Token.EOF && !set.contains(ttype) ) { //System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]); getInputStream().consume(); ttype = getInputStream().LA(1); diff --git a/runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java b/runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java index 1bdce00e5..dbf78841a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java +++ b/runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java @@ -1,8 +1,6 @@ package org.antlr.v4.runtime; -import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.runtime.atn.ATNState; -import org.antlr.v4.runtime.atn.RuleTransition; +import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.misc.IntervalSet; /** This is the default error handling mechanism for ANTLR parsers @@ -23,9 +21,12 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { */ protected int lastErrorIndex = -1; + protected IntervalSet lastErrorStates; + @Override public void reset() { errorRecovery = false; + lastErrorStates = null; lastErrorIndex = -1; } @@ -56,14 +57,18 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { */ @Override public void recover(BaseRecognizer recognizer) { - if ( lastErrorIndex==recognizer.getInputStream().index() ) { - // uh oh, another error at same token index; must be a case - // where LT(1) is in the recovery token set so nothing is - // consumed; consume a single token at least to prevent - // an infinite loop; this is a failsafe. + if ( lastErrorIndex==recognizer.getInputStream().index() && + lastErrorStates.contains(recognizer._ctx.s) ) + { + // uh oh, another error at same token index and previously-visited + // state in ATN; must be a case where LT(1) is in the recovery + // token set so nothing got consumed. Consume a single token + // at least to prevent an infinite loop; this is a failsafe. recognizer.getInputStream().consume(); } lastErrorIndex = recognizer.getInputStream().index(); + if ( lastErrorStates==null ) lastErrorStates = new IntervalSet(); + lastErrorStates.add(recognizer._ctx.s); IntervalSet followSet = computeErrorRecoverySet(recognizer); consumeUntil(recognizer, followSet); } @@ -168,7 +173,7 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { RecognitionException e = null; // if next token is what we are looking for then "delete" this token int nextTokenType = recognizer.getInputStream().LA(2); - if ( expecting.member(nextTokenType) ) { + if ( expecting.contains(nextTokenType) ) { reportUnwantedToken(recognizer); System.err.println("recoverFromMismatchedToken deleting "+ ((TokenStream)recognizer.getInputStream()).LT(1)+ @@ -433,7 +438,7 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy { public void consumeUntil(BaseRecognizer recognizer, IntervalSet set) { //System.out.println("consumeUntil("+set.toString(getTokenNames())+")"); int ttype = recognizer.getInputStream().LA(1); - while (ttype != Token.EOF && !set.member(ttype) ) { + while (ttype != Token.EOF && !set.contains(ttype) ) { //System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]); recognizer.getInputStream().consume(); ttype = recognizer.getInputStream().LA(1); diff --git a/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java b/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java index ed8d3ead4..cccc00906 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java @@ -91,7 +91,7 @@ public class Recognizer { if ( e instanceof UnwantedTokenException ) { UnwantedTokenException ute = (UnwantedTokenException)e; String tokenName=""; - if ( ute.expecting.member(Token.EOF) ) { + if ( ute.expecting.contains(Token.EOF) ) { tokenName = "EOF"; } else { @@ -103,7 +103,7 @@ public class Recognizer { else if ( e instanceof MissingTokenException ) { MissingTokenException mte = (MissingTokenException)e; String tokenName=""; - if ( mte.expecting.member(Token.EOF) ) { + if ( mte.expecting.contains(Token.EOF) ) { tokenName = "EOF"; } else { @@ -126,7 +126,7 @@ public class Recognizer { else if ( e instanceof MismatchedASTNodeException) { MismatchedASTNodeException mtne = (MismatchedASTNodeException)e; String tokenName=""; - if ( mtne.expecting.member(Token.EOF) ) { + if ( mtne.expecting.contains(Token.EOF) ) { tokenName = "EOF"; } else { diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index c1ea3ecc8..6f6048ac9 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -291,7 +291,7 @@ public class LexerATNSimulator extends ATNSimulator { else if ( trans instanceof SetTransition ) { SetTransition st = (SetTransition)trans; boolean not = trans instanceof NotSetTransition; - if ( !not && st.set.member(t) || not && !st.set.member(t) ) { + if ( !not && st.set.contains(t) || not && !st.set.contains(t) ) { // if ( st.set.toString().equals("0") ) { // System.out.println("eh?"); // } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index e4f1c61d1..5eaf7f29e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -485,7 +485,7 @@ public class ParserATNSimulator extends ATNSimulator { else if ( trans instanceof SetTransition ) { SetTransition st = (SetTransition)trans; boolean not = trans instanceof NotSetTransition; - if ( !not && st.set.member(ttype) || not && !st.set.member(ttype) ) { + if ( !not && st.set.contains(ttype) || not && !st.set.contains(ttype) ) { return st.target; } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/IntSet.java b/runtime/Java/src/org/antlr/v4/runtime/misc/IntSet.java index b23a03af9..4dfc0e3a4 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/IntSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/IntSet.java @@ -70,7 +70,7 @@ public interface IntSet { int getSingleElement(); - boolean member(int el); + boolean contains(int el); /** remove this element from this set */ void remove(int el); diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/IntervalSet.java b/runtime/Java/src/org/antlr/v4/runtime/misc/IntervalSet.java index 7c7d93b9d..62153c9c0 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/IntervalSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/IntervalSet.java @@ -318,7 +318,7 @@ public class IntervalSet implements IntSet { } /** Is el in any range of this set? */ - public boolean member(int el) { + public boolean contains(int el) { int n = intervals.size(); for (int i = 0; i < n; i++) { Interval I = (Interval) intervals.get(i); diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/BufferedASTNodeStream.java b/runtime/Java/src/org/antlr/v4/runtime/tree/BufferedASTNodeStream.java index 7a95bbadf..22a70ebe4 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/BufferedASTNodeStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/BufferedASTNodeStream.java @@ -27,13 +27,9 @@ */ package org.antlr.v4.runtime.tree; -import org.antlr.runtime.misc.IntArray; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.*; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; +import java.util.*; /** A buffered stream of tree nodes. Nodes can be from a tree of ANY kind. * @@ -120,7 +116,7 @@ public class BufferedASTNodeStream implements ASTNodeStream { protected int lastMarker; /** Stack of indexes used for push/pop calls */ - protected IntArray calls; + protected List calls; public BufferedASTNodeStream(Object tree) { this(new CommonASTAdaptor(), tree); @@ -350,9 +346,9 @@ public class BufferedASTNodeStream implements ASTNodeStream { */ public void push(int index) { if ( calls==null ) { - calls = new IntArray(); + calls = new ArrayList(); } - calls.push(p); // save current index + calls.add(p); // save current index seek(index); } @@ -360,7 +356,7 @@ public class BufferedASTNodeStream implements ASTNodeStream { * Return top of stack (return index). */ public int pop() { - int ret = calls.pop(); + int ret = calls.remove(calls.size() - 1); seek(ret); return ret; } diff --git a/tool/playground/U.g b/tool/playground/U.g index a45153f9f..514029e80 100644 --- a/tool/playground/U.g +++ b/tool/playground/U.g @@ -1,7 +1,5 @@ grammar U; -options {output=AST;} -tokens {DECL;} -a : ID '=' INT -> ^(DECL ID INT) ; +a : ({true}?ID|{false}?ID{;})* ID ; INT : '0'..'9'+ ; ID : 'a'..'z'+ ; diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index 95abb80bf..18464fe3a 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -409,7 +409,7 @@ CommonSetStuff(m, expr, capture, invert) ::= << setState(); = }>_input.LT(1); -if ( !() ) throw new InputMismatchedException(this); +if ( !() ) throw new InputMismatchException(this); _input.consume(); >> diff --git a/tool/src/org/antlr/v4/automata/TreeParserATNFactory.java b/tool/src/org/antlr/v4/automata/TreeParserATNFactory.java index 8b6273f3e..3a3637ad2 100644 --- a/tool/src/org/antlr/v4/automata/TreeParserATNFactory.java +++ b/tool/src/org/antlr/v4/automata/TreeParserATNFactory.java @@ -29,16 +29,12 @@ package org.antlr.v4.automata; -import org.antlr.v4.runtime.RuleContext; -import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.*; import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.misc.IntervalSet; -import org.antlr.v4.tool.Grammar; -import org.antlr.v4.tool.GrammarAST; -import org.antlr.v4.tool.TreePatternAST; +import org.antlr.v4.tool.*; -import java.util.ArrayList; -import java.util.List; +import java.util.*; /** Build ATNs for tree grammars */ public class TreeParserATNFactory extends ParserATNFactory { @@ -60,9 +56,9 @@ public class TreeParserATNFactory extends ParserATNFactory { LL1Analyzer analyzer = new LL1Analyzer(atn); IntervalSet look = analyzer.LOOK(firstChild, RuleContext.EMPTY); TreePatternAST root = treePatternRootNodes.get(i); - System.out.println(root.toStringTree()+"==nullable? "+look.member(Token.UP)); + System.out.println(root.toStringTree()+"==nullable? "+look.contains(Token.UP)); - if ( look.member(Token.UP) ) { + if ( look.contains(Token.UP) ) { // nullable child list if we can see the UP as the next token. // convert r DN kids UP to r (DN kids UP)?; leave AST alone-- // that drives code gen. This just affects analysis @@ -85,7 +81,7 @@ public class TreeParserATNFactory extends ParserATNFactory { // find DOWN node then first child for (Handle elh : els) { Transition trans = elh.left.transition(0); - if ( !trans.isEpsilon() && trans.label().member(Token.DOWN) ) { + if ( !trans.isEpsilon() && trans.label().contains(Token.DOWN) ) { ATNState downState = elh.left; downStates.add(downState); root.downState = downState; @@ -96,7 +92,7 @@ public class TreeParserATNFactory extends ParserATNFactory { // find UP node for (Handle elh : els) { Transition trans = elh.left.transition(0); - if ( trans instanceof AtomTransition && trans.label().member(Token.UP) ) { + if ( trans instanceof AtomTransition && trans.label().contains(Token.UP) ) { ATNState upTargetState = elh.right; root.upState = elh.left; upTargetStates.add(upTargetState); diff --git a/tool/src/org/antlr/v4/codegen/model/MatchTree.java b/tool/src/org/antlr/v4/codegen/model/MatchTree.java index 5d4119385..8865b1006 100644 --- a/tool/src/org/antlr/v4/codegen/model/MatchTree.java +++ b/tool/src/org/antlr/v4/codegen/model/MatchTree.java @@ -31,13 +31,10 @@ package org.antlr.v4.codegen.model; import org.antlr.v4.codegen.OutputModelFactory; import org.antlr.v4.misc.Utils; -import org.antlr.v4.runtime.RuleContext; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.atn.ATNState; -import org.antlr.v4.runtime.atn.LL1Analyzer; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.misc.IntervalSet; -import org.antlr.v4.tool.GrammarAST; -import org.antlr.v4.tool.TreePatternAST; +import org.antlr.v4.tool.*; import java.util.List; @@ -81,8 +78,8 @@ public class MatchTree extends RuleElement { ATNState firstChildState = rootNode.downState.transition(0).target; LL1Analyzer analyzer = new LL1Analyzer(firstChildState.atn); IntervalSet look = analyzer.LOOK(firstChildState, RuleContext.EMPTY); - System.out.println(rootNode.toStringTree()+"==nullable? "+look.member(Token.UP)); - return look.member(Token.UP); + System.out.println(rootNode.toStringTree()+"==nullable? "+look.contains(Token.UP)); + return look.contains(Token.UP); } } diff --git a/tool/src/org/antlr/v4/tool/GrammarAST.java b/tool/src/org/antlr/v4/tool/GrammarAST.java index 5340cd273..20738988c 100644 --- a/tool/src/org/antlr/v4/tool/GrammarAST.java +++ b/tool/src/org/antlr/v4/tool/GrammarAST.java @@ -88,7 +88,7 @@ public class GrammarAST extends CommonTree { GrammarAST t = null; while ( work.size()>0 ) { t = work.remove(0); - if ( types.member(t.getType()) ) nodes.add(t); + if ( types.contains(t.getType()) ) nodes.add(t); if ( t.children!=null ) work.addAll(t.children); } return nodes; diff --git a/tool/test/org/antlr/v4/test/TestIntervalSet.java b/tool/test/org/antlr/v4/test/TestIntervalSet.java index 6d7f077d7..20d2108d7 100644 --- a/tool/test/org/antlr/v4/test/TestIntervalSet.java +++ b/tool/test/org/antlr/v4/test/TestIntervalSet.java @@ -259,13 +259,13 @@ public class TestIntervalSet extends BaseTest { @Test public void testMembership() throws Exception { IntervalSet s = IntervalSet.of(15,15); s.add(50,60); - assertTrue(!s.member(0)); - assertTrue(!s.member(20)); - assertTrue(!s.member(100)); - assertTrue(s.member(15)); - assertTrue(s.member(55)); - assertTrue(s.member(50)); - assertTrue(s.member(60)); + assertTrue(!s.contains(0)); + assertTrue(!s.contains(20)); + assertTrue(!s.contains(100)); + assertTrue(s.contains(15)); + assertTrue(s.contains(55)); + assertTrue(s.contains(50)); + assertTrue(s.contains(60)); } // {2,15,18} & 10..20