forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

rename member, tweak output. new chk for inf loop from error sync works 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9088]
This commit is contained in:
parrt 2011-09-30 20:40:14 -08:00
parent 60c9fe76c0
commit 6b4e9905fb
14 changed files with 55 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java
View File

@ -28,14 +28,10 @@
*/ */
package org.antlr.v4.runtime; package org.antlr.v4.runtime;
import org.antlr.v4.runtime.atn.ATNConfig; import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.ParserATNSimulator; import org.antlr.v4.runtime.misc.*;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.OrderedHashSet;
import java.util.ArrayList; import java.util.*;
import java.util.List;
import java.util.Set;
/** A generic recognizer that can handle recognizers generated from /** A generic recognizer that can handle recognizers generated from
* parser and tree grammars. This is all the parsing * parser and tree grammars. This is all the parsing
@ -111,7 +107,7 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
// like matchSet but w/o consume; error checking routine. // like matchSet but w/o consume; error checking routine.
public void sync(IntervalSet expecting) { public void sync(IntervalSet expecting) {
if ( expecting.member(getInputStream().LA(1)) ) return; if ( expecting.contains(getInputStream().LA(1)) ) return;
// System.out.println("failed sync to "+expecting); // System.out.println("failed sync to "+expecting);
IntervalSet followSet = computeErrorRecoverySet(); IntervalSet followSet = computeErrorRecoverySet();
followSet.addAll(expecting); followSet.addAll(expecting);
@ -582,7 +578,7 @@ public abstract class BaseRecognizer extends Recognizer<ParserATNSimulator> {
public void consumeUntil(IntervalSet set) { public void consumeUntil(IntervalSet set) {
//System.out.println("consumeUntil("+set.toString(getTokenNames())+")"); //System.out.println("consumeUntil("+set.toString(getTokenNames())+")");
int ttype = getInputStream().LA(1); int ttype = getInputStream().LA(1);
while (ttype != Token.EOF && !set.member(ttype) ) { while (ttype != Token.EOF && !set.contains(ttype) ) {
//System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]); //System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]);
getInputStream().consume(); getInputStream().consume();
ttype = getInputStream().LA(1); ttype = getInputStream().LA(1);

25
runtime/Java/src/org/antlr/v4/runtime/DefaultANTLRErrorStrategy.java
View File

@ -1,8 +1,6 @@
package org.antlr.v4.runtime; package org.antlr.v4.runtime;
import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.RuleTransition;
import org.antlr.v4.runtime.misc.IntervalSet; import org.antlr.v4.runtime.misc.IntervalSet;
/** This is the default error handling mechanism for ANTLR parsers /** This is the default error handling mechanism for ANTLR parsers
@ -23,9 +21,12 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
*/ */
protected int lastErrorIndex = -1; protected int lastErrorIndex = -1;
protected IntervalSet lastErrorStates;
@Override @Override
public void reset() { public void reset() {
errorRecovery = false; errorRecovery = false;
lastErrorStates = null;
lastErrorIndex = -1; lastErrorIndex = -1;
} }
@ -56,14 +57,18 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
*/ */
@Override @Override
public void recover(BaseRecognizer recognizer) { public void recover(BaseRecognizer recognizer) {
if ( lastErrorIndex==recognizer.getInputStream().index() ) { if ( lastErrorIndex==recognizer.getInputStream().index() &&
// uh oh, another error at same token index; must be a case lastErrorStates.contains(recognizer._ctx.s) )
// where LT(1) is in the recovery token set so nothing is {
// consumed; consume a single token at least to prevent // uh oh, another error at same token index and previously-visited
// an infinite loop; this is a failsafe. // state in ATN; must be a case where LT(1) is in the recovery
// token set so nothing got consumed. Consume a single token
// at least to prevent an infinite loop; this is a failsafe.
recognizer.getInputStream().consume(); recognizer.getInputStream().consume();
} }
lastErrorIndex = recognizer.getInputStream().index(); lastErrorIndex = recognizer.getInputStream().index();
if ( lastErrorStates==null ) lastErrorStates = new IntervalSet();
lastErrorStates.add(recognizer._ctx.s);
IntervalSet followSet = computeErrorRecoverySet(recognizer); IntervalSet followSet = computeErrorRecoverySet(recognizer);
consumeUntil(recognizer, followSet); consumeUntil(recognizer, followSet);
} }
@ -168,7 +173,7 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
RecognitionException e = null; RecognitionException e = null;
// if next token is what we are looking for then "delete" this token // if next token is what we are looking for then "delete" this token
int nextTokenType = recognizer.getInputStream().LA(2); int nextTokenType = recognizer.getInputStream().LA(2);
if ( expecting.member(nextTokenType) ) { if ( expecting.contains(nextTokenType) ) {
reportUnwantedToken(recognizer); reportUnwantedToken(recognizer);
System.err.println("recoverFromMismatchedToken deleting "+ System.err.println("recoverFromMismatchedToken deleting "+
((TokenStream)recognizer.getInputStream()).LT(1)+ ((TokenStream)recognizer.getInputStream()).LT(1)+
@ -433,7 +438,7 @@ public class DefaultANTLRErrorStrategy implements ANTLRErrorStrategy {
public void consumeUntil(BaseRecognizer recognizer, IntervalSet set) { public void consumeUntil(BaseRecognizer recognizer, IntervalSet set) {
//System.out.println("consumeUntil("+set.toString(getTokenNames())+")"); //System.out.println("consumeUntil("+set.toString(getTokenNames())+")");
int ttype = recognizer.getInputStream().LA(1); int ttype = recognizer.getInputStream().LA(1);
while (ttype != Token.EOF && !set.member(ttype) ) { while (ttype != Token.EOF && !set.contains(ttype) ) {
//System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]); //System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]);
recognizer.getInputStream().consume(); recognizer.getInputStream().consume();
ttype = recognizer.getInputStream().LA(1); ttype = recognizer.getInputStream().LA(1);

6
runtime/Java/src/org/antlr/v4/runtime/Recognizer.java
View File

@ -91,7 +91,7 @@ public class Recognizer<ATNInterpreter> {
if ( e instanceof UnwantedTokenException ) { if ( e instanceof UnwantedTokenException ) {
UnwantedTokenException ute = (UnwantedTokenException)e; UnwantedTokenException ute = (UnwantedTokenException)e;
String tokenName="<unknown>"; String tokenName="<unknown>";
if ( ute.expecting.member(Token.EOF) ) { if ( ute.expecting.contains(Token.EOF) ) {
tokenName = "EOF"; tokenName = "EOF";
} }
else { else {
@ -103,7 +103,7 @@ public class Recognizer<ATNInterpreter> {
else if ( e instanceof MissingTokenException ) { else if ( e instanceof MissingTokenException ) {
MissingTokenException mte = (MissingTokenException)e; MissingTokenException mte = (MissingTokenException)e;
String tokenName="<unknown>"; String tokenName="<unknown>";
if ( mte.expecting.member(Token.EOF) ) { if ( mte.expecting.contains(Token.EOF) ) {
tokenName = "EOF"; tokenName = "EOF";
} }
else { else {
@ -126,7 +126,7 @@ public class Recognizer<ATNInterpreter> {
else if ( e instanceof MismatchedASTNodeException) { else if ( e instanceof MismatchedASTNodeException) {
MismatchedASTNodeException mtne = (MismatchedASTNodeException)e; MismatchedASTNodeException mtne = (MismatchedASTNodeException)e;
String tokenName="<unknown>"; String tokenName="<unknown>";
if ( mtne.expecting.member(Token.EOF) ) { if ( mtne.expecting.contains(Token.EOF) ) {
tokenName = "EOF"; tokenName = "EOF";
} }
else { else {

2
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -291,7 +291,7 @@ public class LexerATNSimulator extends ATNSimulator {
else if ( trans instanceof SetTransition ) { else if ( trans instanceof SetTransition ) {
SetTransition st = (SetTransition)trans; SetTransition st = (SetTransition)trans;
boolean not = trans instanceof NotSetTransition; boolean not = trans instanceof NotSetTransition;
if ( !not && st.set.member(t) || not && !st.set.member(t) ) { if ( !not && st.set.contains(t) || not && !st.set.contains(t) ) {
// if ( st.set.toString().equals("0") ) { // if ( st.set.toString().equals("0") ) {
// System.out.println("eh?"); // System.out.println("eh?");
// } // }

2
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -485,7 +485,7 @@ public class ParserATNSimulator extends ATNSimulator {
else if ( trans instanceof SetTransition ) { else if ( trans instanceof SetTransition ) {
SetTransition st = (SetTransition)trans; SetTransition st = (SetTransition)trans;
boolean not = trans instanceof NotSetTransition; boolean not = trans instanceof NotSetTransition;
if ( !not && st.set.member(ttype) || not && !st.set.member(ttype) ) { if ( !not && st.set.contains(ttype) || not && !st.set.contains(ttype) ) {
return st.target; return st.target;
} }
} }

2
runtime/Java/src/org/antlr/v4/runtime/misc/IntSet.java
View File

@ -70,7 +70,7 @@ public interface IntSet {
int getSingleElement(); int getSingleElement();
boolean member(int el); boolean contains(int el);
/** remove this element from this set */ /** remove this element from this set */
void remove(int el); void remove(int el);

2
runtime/Java/src/org/antlr/v4/runtime/misc/IntervalSet.java
View File

@ -318,7 +318,7 @@ public class IntervalSet implements IntSet {
} }
/** Is el in any range of this set? */ /** Is el in any range of this set? */
public boolean member(int el) { public boolean contains(int el) {
int n = intervals.size(); int n = intervals.size();
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
Interval I = (Interval) intervals.get(i); Interval I = (Interval) intervals.get(i);

16
runtime/Java/src/org/antlr/v4/runtime/tree/BufferedASTNodeStream.java
View File

@ -27,13 +27,9 @@
*/ */
package org.antlr.v4.runtime.tree; package org.antlr.v4.runtime.tree;
import org.antlr.runtime.misc.IntArray; import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream;
import java.util.ArrayList; import java.util.*;
import java.util.Iterator;
import java.util.List;
/** A buffered stream of tree nodes. Nodes can be from a tree of ANY kind. /** A buffered stream of tree nodes. Nodes can be from a tree of ANY kind.
* *
@ -120,7 +116,7 @@ public class BufferedASTNodeStream implements ASTNodeStream {
protected int lastMarker; protected int lastMarker;
/** Stack of indexes used for push/pop calls */ /** Stack of indexes used for push/pop calls */
protected IntArray calls; protected List<Integer> calls;
public BufferedASTNodeStream(Object tree) { public BufferedASTNodeStream(Object tree) {
this(new CommonASTAdaptor(), tree); this(new CommonASTAdaptor(), tree);
@ -350,9 +346,9 @@ public class BufferedASTNodeStream implements ASTNodeStream {
*/ */
public void push(int index) { public void push(int index) {
if ( calls==null ) { if ( calls==null ) {
calls = new IntArray(); calls = new ArrayList<Integer>();
} }
calls.push(p); // save current index calls.add(p); // save current index
seek(index); seek(index);
} }
@ -360,7 +356,7 @@ public class BufferedASTNodeStream implements ASTNodeStream {
* Return top of stack (return index). * Return top of stack (return index).
*/ */
public int pop() { public int pop() {
int ret = calls.pop(); int ret = calls.remove(calls.size() - 1);
seek(ret); seek(ret);
return ret; return ret;
} }

4
tool/playground/U.g
View File

@ -1,7 +1,5 @@
grammar U; grammar U;
options {output=AST;} a : ({true}?ID|{false}?ID{;})* ID ;
tokens {DECL;}
a : ID '=' INT -> ^(DECL ID INT) ;
INT : '0'..'9'+ ; INT : '0'..'9'+ ;
ID : 'a'..'z'+ ; ID : 'a'..'z'+ ;

2
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -409,7 +409,7 @@ CommonSetStuff(m, expr, capture, invert) ::= <<
setState(<m.stateNumber>); setState(<m.stateNumber>);
<if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>_input.LT(1); <if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>_input.LT(1);
<capture> <capture>
if ( <if(!invert)>!<endif>(<expr>) ) throw new InputMismatchedException(this); if ( <if(!invert)>!<endif>(<expr>) ) throw new InputMismatchException(this);
_input.consume(); _input.consume();
>> >>

18
tool/src/org/antlr/v4/automata/TreeParserATNFactory.java
View File

@ -29,16 +29,12 @@
package org.antlr.v4.automata; package org.antlr.v4.automata;
import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.misc.IntervalSet; import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.*;
import org.antlr.v4.tool.GrammarAST;
import org.antlr.v4.tool.TreePatternAST;
import java.util.ArrayList; import java.util.*;
import java.util.List;
/** Build ATNs for tree grammars */ /** Build ATNs for tree grammars */
public class TreeParserATNFactory extends ParserATNFactory { public class TreeParserATNFactory extends ParserATNFactory {
@ -60,9 +56,9 @@ public class TreeParserATNFactory extends ParserATNFactory {
LL1Analyzer analyzer = new LL1Analyzer(atn); LL1Analyzer analyzer = new LL1Analyzer(atn);
IntervalSet look = analyzer.LOOK(firstChild, RuleContext.EMPTY); IntervalSet look = analyzer.LOOK(firstChild, RuleContext.EMPTY);
TreePatternAST root = treePatternRootNodes.get(i); TreePatternAST root = treePatternRootNodes.get(i);
System.out.println(root.toStringTree()+"==nullable? "+look.member(Token.UP)); System.out.println(root.toStringTree()+"==nullable? "+look.contains(Token.UP));
if ( look.member(Token.UP) ) { if ( look.contains(Token.UP) ) {
// nullable child list if we can see the UP as the next token. // nullable child list if we can see the UP as the next token.
// convert r DN kids UP to r (DN kids UP)?; leave AST alone-- // convert r DN kids UP to r (DN kids UP)?; leave AST alone--
// that drives code gen. This just affects analysis // that drives code gen. This just affects analysis
@ -85,7 +81,7 @@ public class TreeParserATNFactory extends ParserATNFactory {
// find DOWN node then first child // find DOWN node then first child
for (Handle elh : els) { for (Handle elh : els) {
Transition trans = elh.left.transition(0); Transition trans = elh.left.transition(0);
if ( !trans.isEpsilon() && trans.label().member(Token.DOWN) ) { if ( !trans.isEpsilon() && trans.label().contains(Token.DOWN) ) {
ATNState downState = elh.left; ATNState downState = elh.left;
downStates.add(downState); downStates.add(downState);
root.downState = downState; root.downState = downState;
@ -96,7 +92,7 @@ public class TreeParserATNFactory extends ParserATNFactory {
// find UP node // find UP node
for (Handle elh : els) { for (Handle elh : els) {
Transition trans = elh.left.transition(0); Transition trans = elh.left.transition(0);
if ( trans instanceof AtomTransition && trans.label().member(Token.UP) ) { if ( trans instanceof AtomTransition && trans.label().contains(Token.UP) ) {
ATNState upTargetState = elh.right; ATNState upTargetState = elh.right;
root.upState = elh.left; root.upState = elh.left;
upTargetStates.add(upTargetState); upTargetStates.add(upTargetState);

13
tool/src/org/antlr/v4/codegen/model/MatchTree.java
View File

@ -31,13 +31,10 @@ package org.antlr.v4.codegen.model;
import org.antlr.v4.codegen.OutputModelFactory; import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.misc.Utils; import org.antlr.v4.misc.Utils;
import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.LL1Analyzer;
import org.antlr.v4.runtime.misc.IntervalSet; import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.tool.GrammarAST; import org.antlr.v4.tool.*;
import org.antlr.v4.tool.TreePatternAST;
import java.util.List; import java.util.List;
@ -81,8 +78,8 @@ public class MatchTree extends RuleElement {
ATNState firstChildState = rootNode.downState.transition(0).target; ATNState firstChildState = rootNode.downState.transition(0).target;
LL1Analyzer analyzer = new LL1Analyzer(firstChildState.atn); LL1Analyzer analyzer = new LL1Analyzer(firstChildState.atn);
IntervalSet look = analyzer.LOOK(firstChildState, RuleContext.EMPTY); IntervalSet look = analyzer.LOOK(firstChildState, RuleContext.EMPTY);
System.out.println(rootNode.toStringTree()+"==nullable? "+look.member(Token.UP)); System.out.println(rootNode.toStringTree()+"==nullable? "+look.contains(Token.UP));
return look.member(Token.UP); return look.contains(Token.UP);
} }
} }

2
tool/src/org/antlr/v4/tool/GrammarAST.java
View File

@ -88,7 +88,7 @@ public class GrammarAST extends CommonTree {
GrammarAST t = null; GrammarAST t = null;
while ( work.size()>0 ) { while ( work.size()>0 ) {
t = work.remove(0); t = work.remove(0);
if ( types.member(t.getType()) ) nodes.add(t); if ( types.contains(t.getType()) ) nodes.add(t);
if ( t.children!=null ) work.addAll(t.children); if ( t.children!=null ) work.addAll(t.children);
} }
return nodes; return nodes;

14
tool/test/org/antlr/v4/test/TestIntervalSet.java
View File

@ -259,13 +259,13 @@ public class TestIntervalSet extends BaseTest {
@Test public void testMembership() throws Exception { @Test public void testMembership() throws Exception {
IntervalSet s = IntervalSet.of(15,15); IntervalSet s = IntervalSet.of(15,15);
s.add(50,60); s.add(50,60);
assertTrue(!s.member(0)); assertTrue(!s.contains(0));
assertTrue(!s.member(20)); assertTrue(!s.contains(20));
assertTrue(!s.member(100)); assertTrue(!s.contains(100));
assertTrue(s.member(15)); assertTrue(s.contains(15));
assertTrue(s.member(55)); assertTrue(s.contains(55));
assertTrue(s.member(50)); assertTrue(s.contains(50));
assertTrue(s.member(60)); assertTrue(s.contains(60));
} }
// {2,15,18} & 10..20 // {2,15,18} & 10..20