forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

snapshot; removed some dependencies to tool in runtime; made sure java grammar / parser still works 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8777]
This commit is contained in:
parrt 2011-06-27 17:38:15 -08:00
parent de0881a88b
commit ad98d17191
21 changed files with 85 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java
View File

@ -2,7 +2,7 @@ package org.antlr.v4.runtime.atn;
import org.antlr.v4.misc.IntervalSet;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.Rule;
import java.util.*;
@ -12,7 +12,6 @@ public class ATN {
public static final int INVALID_ALT_NUMBER = -1;
public static final int INVALID_DECISION_NUMBER = -1;
public Grammar g;
public List<ATNState> states = new ArrayList<ATNState>();
public List<ATNState> rules = new ArrayList<ATNState>(); // rule index to start state
@ -41,17 +40,6 @@ public class ATN {
// TODO: for runtime all we need is states, decisionToATNState I think
public ATN(Grammar g) {
this.g = g;
if ( g.isLexer() ) {
for (Rule r : g.rules.values()) {
ruleToTokenType.add(g.getTokenType(r.name));
if ( r.actionIndex>0 ) ruleToActionIndex.add(r.actionIndex);
else ruleToActionIndex.add(0);
}
}
}
/** Used for runtime deserialization of ATNs from strings */
public ATN() { }

4
runtime/Java/src/org/antlr/v4/runtime/atn/ATNState.java
View File

@ -1,6 +1,6 @@
package org.antlr.v4.runtime.atn;
import org.antlr.v4.tool.*;
import org.antlr.v4.tool.Rule;
import java.util.*;
@ -57,7 +57,7 @@ public class ATNState {
public ATN atn = null;
/** ATN state is associated with which node in AST? */
public GrammarAST ast;
// public GrammarAST ast;
public Transition transition;
/** For o-A->o type ATN tranitions, record the label that leads to this
* state. Useful for creating rich error messages when we find

4
runtime/Java/src/org/antlr/v4/runtime/atn/LexerInterpreter.java
View File

@ -225,7 +225,7 @@ public class LexerInterpreter extends ATNInterpreter {
else if ( trans.getClass() == RangeTransition.class ) {
RangeTransition rt = (RangeTransition)trans;
if ( t>=rt.from && t<=rt.to ) {
if ( debug ) System.out.println("match range "+rt.toString(atn.g));
if ( debug ) System.out.println("match range "+rt.toString());
return rt.target;
}
}
@ -233,7 +233,7 @@ public class LexerInterpreter extends ATNInterpreter {
SetTransition st = (SetTransition)trans;
boolean not = trans instanceof NotSetTransition;
if ( !not && st.label.member(t) || not && !st.label.member(t) ) {
if ( debug ) System.out.println("match set "+st.label.toString(atn.g));
if ( debug ) System.out.println("match set "+st.label.toString());
return st.target;
}
}

3
runtime/Java/src/org/antlr/v4/runtime/atn/ParserInterpreter.java
View File

@ -601,8 +601,7 @@ public class ParserInterpreter extends ATNInterpreter {
public String getTokenName(int t) {
if ( t==-1 ) return "EOF";
if ( atn.g!=null ) return atn.g.getTokenDisplayName(t);
if ( parser !=null && parser.getTokenNames()!=null ) return parser.getTokenNames()[t]+"<"+t+">";
if ( parser!=null && parser.getTokenNames()!=null ) return parser.getTokenNames()[t]+"<"+t+">";
return String.valueOf(t);
}

4
runtime/Java/src/org/antlr/v4/runtime/misc/QStack.java
View File

@ -39,8 +39,10 @@ public class QStack<T> {
public T pop() {
if ( sp<0 ) throw new EmptyStackException();
T o = elements[sp];
elements[sp] = null; // let gc reclaim that element
return elements[sp--];
sp--;
return o;
}
public void clear() { sp = -1; }

9
tool/playground/L.g
View File

@ -1,7 +1,8 @@
lexer grammar L;
WS : ' '+ {skip();} ;
STRING_START : '"' {pushMode(STRING_MODE); more();} ;
WS : ' '|'\n' {skip();} ;
StringLiteral
: '"' ( ~('\\'|'"') )* '"'
;
mode STRING_MODE;
STRING : '"' {popMode();} ;
ANY : . {more();} ;

10
tool/playground/TestL.java
View File

@ -1,8 +1,10 @@
import org.antlr.v4.runtime.*;
public class TestL {
public static void main(String[] args) throws Exception {
// L lexer = new L(new ANTLRFileStream(args[0]));
// CommonTokenStream tokens = new CommonTokenStream(lexer);
// tokens.fill();
// System.out.println(tokens.getTokens());
L lexer = new L(new ANTLRFileStream(args[0]));
CommonTokenStream tokens = new CommonTokenStream(lexer);
tokens.fill();
System.out.println(tokens.getTokens());
}
}

2
tool/src/org/antlr/v4/analysis/AnalysisPipeline.java
View File

@ -15,7 +15,7 @@ public class AnalysisPipeline {
public void process() {
// LEFT-RECURSION CHECK
LeftRecursionDetector lr = new LeftRecursionDetector(g.atn);
LeftRecursionDetector lr = new LeftRecursionDetector(g, g.atn);
lr.check();
if ( lr.listOfRecursiveCycles.size()>0 ) return; // bail out

10
tool/src/org/antlr/v4/analysis/LeftRecursionDetector.java
View File

@ -2,11 +2,12 @@ package org.antlr.v4.analysis;
import org.antlr.v4.misc.OrderedHashSet;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.*;
import java.util.*;
public class LeftRecursionDetector {
Grammar g;
public ATN atn;
/** Holds a list of cycles (sets of rule names). */
@ -17,7 +18,10 @@ public class LeftRecursionDetector {
*/
Set<RuleStartState> rulesVisitedPerRuleCheck = new HashSet<RuleStartState>();
public LeftRecursionDetector(ATN atn) { this.atn = atn; }
public LeftRecursionDetector(Grammar g, ATN atn) {
this.g = g;
this.atn = atn;
}
public void check() {
for (RuleStartState start : atn.ruleToStartState.values()) {
@ -31,7 +35,7 @@ public class LeftRecursionDetector {
}
//System.out.println("cycles="+listOfRecursiveCycles);
if ( listOfRecursiveCycles.size()>0 ) {
atn.g.tool.errMgr.leftRecursionCycles(atn.g.fileName, listOfRecursiveCycles);
g.tool.errMgr.leftRecursionCycles(g.fileName, listOfRecursiveCycles);
}
}

36
tool/src/org/antlr/v4/automata/ATNSerializer.java
View File

@ -2,15 +2,19 @@ package org.antlr.v4.automata;
import org.antlr.v4.misc.*;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.*;
import java.util.*;
public class ATNSerializer {
public Grammar g;
public ATN atn;
public List<IntervalSet> sets = new ArrayList<IntervalSet>();
public ATNSerializer(ATN atn) { this.atn = atn; }
public ATNSerializer(Grammar g, ATN atn) {
this.g = g;
this.atn = atn;
}
/** Serialize state descriptors, edge descriptors, and decision->state map
* into list of ints:
@ -35,8 +39,8 @@ public class ATNSerializer {
*/
public List<Integer> serialize() {
List<Integer> data = new ArrayList<Integer>();
data.add(atn.g.getType());
data.add(atn.g.getMaxTokenType());
data.add(g.getType());
data.add(g.getMaxTokenType());
data.add(atn.states.size());
int nedges = 0;
// dump states, count edges and collect sets while doing so
@ -59,10 +63,10 @@ public class ATNSerializer {
for (int r=0; r<nrules; r++) {
ATNState ruleStartState = atn.rules.get(r);
data.add(ruleStartState.stateNumber);
if ( atn.g.isLexer() ) {
if ( g.isLexer() ) {
data.add(atn.ruleToTokenType.get(r));
String ruleName = atn.g.rules.getKey(r);
Rule rule = atn.g.getRule(ruleName);
String ruleName = g.rules.getKey(r);
Rule rule = g.getRule(ruleName);
data.add(rule.actionIndex);
}
else {
@ -212,24 +216,24 @@ public class ATNSerializer {
public String getTokenName(int t) {
if ( t==-1 ) return "EOF";
if ( atn.g!=null ) return atn.g.getTokenDisplayName(t);
if ( g!=null ) return g.getTokenDisplayName(t);
return String.valueOf(t);
}
/** Used by Java target to encode short/int array as chars in string. */
public static String getSerializedAsString(ATN atn) {
return new String(Utils.toCharArray(getSerialized(atn)));
public static String getSerializedAsString(Grammar g, ATN atn) {
return new String(Utils.toCharArray(getSerialized(g, atn)));
}
public static List<Integer> getSerialized(ATN atn) {
return new ATNSerializer(atn).serialize();
public static List<Integer> getSerialized(Grammar g, ATN atn) {
return new ATNSerializer(g, atn).serialize();
}
public static char[] getSerializedAsChars(ATN atn) {
return Utils.toCharArray(new ATNSerializer(atn).serialize());
public static char[] getSerializedAsChars(Grammar g, ATN atn) {
return Utils.toCharArray(new ATNSerializer(g, atn).serialize());
}
public static String getDecoded(ATN atn) {
return new ATNSerializer(atn).decode(Utils.toCharArray(getSerialized(atn)));
public static String getDecoded(Grammar g, ATN atn) {
return new ATNSerializer(g, atn).decode(Utils.toCharArray(getSerialized(g, atn)));
}
}

7
tool/src/org/antlr/v4/automata/LexerATNFactory.java
View File

@ -20,6 +20,13 @@ public class LexerATNFactory extends ParserATNFactory {
atn.defineDecisionState(startState);
}
// INIT ACTION, RULE->TOKEN_TYPE MAP
for (Rule r : g.rules.values()) {
atn.ruleToTokenType.add(g.getTokenType(r.name));
if ( r.actionIndex>0 ) atn.ruleToActionIndex.add(r.actionIndex);
else atn.ruleToActionIndex.add(0);
}
// CREATE ATN FOR EACH RULE
_createATN(g.rules.values());

4
tool/src/org/antlr/v4/automata/ParserATNFactory.java
View File

@ -20,7 +20,7 @@ public class ParserATNFactory implements ATNFactory {
public Rule currentRule;
ATN atn;
public ParserATNFactory(Grammar g) { this.g = g; atn = new ATN(g); }
public ParserATNFactory(Grammar g) { this.g = g; atn = new ATN(); }
public ATN createATN() {
_createATN(g.rules.values());
@ -458,7 +458,6 @@ public class ParserATNFactory implements ATNFactory {
try {
Constructor ctor = nodeType.getConstructor();
ATNState s = (ATNState)ctor.newInstance();
s.ast = node;
s.setRule(currentRule);
atn.addState(s);
return s;
@ -472,7 +471,6 @@ public class ParserATNFactory implements ATNFactory {
public ATNState newState(GrammarAST node) {
ATNState n = new ATNState();
n.setRule(currentRule);
n.ast = node;
atn.addState(n);
return n;
}

5
tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java
View File

@ -84,7 +84,10 @@ public abstract class DefaultOutputModelFactory implements OutputModelFactory {
public void setRoot(OutputModelObject root) { this.root = root; }
public RuleFunction getCurrentRule() { return currentRule.peek(); }
public RuleFunction getCurrentRule() {
if ( currentRule.size()>0 ) return currentRule.peek();
return null;
}
public void pushCurrentRule(RuleFunction r) { currentRule.push(r); }

7
tool/src/org/antlr/v4/codegen/LexerFactory.java
View File

@ -7,8 +7,9 @@ public class LexerFactory extends DefaultOutputModelFactory {
public LexerFactory(CodeGenerator gen) { super(gen); }
public OutputModelObject buildOutputModel(OutputModelController controller) {
LexerFile lexer = new LexerFile(this, getGenerator().getRecognizerFileName());
setRoot(lexer);
return lexer;
LexerFile file = new LexerFile(this, getGenerator().getRecognizerFileName());
setRoot(file);
file.lexer = new Lexer(this, file);
return file;
}
}

2
tool/src/org/antlr/v4/codegen/model/Choice.java
View File

@ -51,7 +51,7 @@ public abstract class Choice extends RuleElement {
if ( expr instanceof TestSetInline) {
TestSetInline e = (TestSetInline)expr;
Decl d = new TokenTypeDecl(factory, e.varName);
factory.getCurrentRule().addContextDecl(d);
factory.getCurrentRule().addLocalDecl(d);
CaptureNextTokenType nextType = new CaptureNextTokenType(e.varName);
addPreambleOp(nextType);
}

1
tool/src/org/antlr/v4/codegen/model/LexerFile.java
View File

@ -20,6 +20,5 @@ public class LexerFile extends OutputModelObject {
GrammarAST ast = g.namedActions.get(name);
namedActions.put(name, new Action(factory, ast));
}
lexer = new Lexer(factory, this);
}
}

19
tool/src/org/antlr/v4/codegen/model/RuleFunction.java
View File

@ -78,25 +78,6 @@ public class RuleFunction extends OutputModelObject {
}
startState = factory.getGrammar().atn.ruleToStartState.get(r);
// // TRIGGER factory functions for rule elements
// factory.currentRule.push(this);
// GrammarASTAdaptor adaptor = new GrammarASTAdaptor(r.ast.token.getInputStream());
// GrammarAST blk = (GrammarAST)r.ast.getFirstChildWithType(ANTLRParser.BLOCK);
// CommonTreeNodeStream nodes = new CommonTreeNodeStream(adaptor,blk);
// SourceGenTriggers genTriggers = new SourceGenTriggers(nodes, factory);
// try {
// code = genTriggers.block(null,null); // GEN Instr OBJECTS
// }
// catch (Exception e){
// e.printStackTrace(System.err);
// }
// ctxType = factory.gen.target.getRuleFunctionContextStructName(r);
// ruleCtx.name = ctxType;
//
// if ( ruleCtx.isEmpty() ) ruleCtx = null;
// factory.currentRule.pop();
}
/** Add local var decl */

2
tool/src/org/antlr/v4/codegen/model/SerializedATN.java
View File

@ -11,7 +11,7 @@ public class SerializedATN extends OutputModelObject {
public List<String> serialized;
public SerializedATN(OutputModelFactory factory, ATN atn) {
super(factory);
List<Integer> data = ATNSerializer.getSerialized(atn);
List<Integer> data = ATNSerializer.getSerialized(factory.getGrammar(), atn);
serialized = new ArrayList<String>(data.size());
for (int c : data) {
String encoded = factory.getGenerator().target.encodeIntAsCharEscape(c);

2
tool/src/org/antlr/v4/semantics/CollectSymbols.g
View File

@ -270,7 +270,7 @@ LabelElementPair lp = new LabelElementPair(g, $id, $e, $start.getType());
//currentRule.labelDefs.map($id.text, lp);
currentRule.alt[currentAlt].labelDefs.map($id.text, lp);
}
: {inContext("RULE ...")}?
: {inContext("RULE ...") && !inContext("OPTIONS ...")}?
( ^(ASSIGN id=ID e=.)
| ^(PLUS_ASSIGN id=ID e=.)
)

7
tool/test/org/antlr/v4/test/TestATNDeserialization.java
View File

@ -115,11 +115,10 @@ public class TestATNDeserialization extends BaseTest {
protected void checkDeserializationIsStable(Grammar g) {
ATN atn = createATN(g);
char[] data = Utils.toCharArray(ATNSerializer.getSerialized(atn));
String atnData = ATNSerializer.getDecoded(atn);
char[] data = Utils.toCharArray(ATNSerializer.getSerialized(g, atn));
String atnData = ATNSerializer.getDecoded(g, atn);
ATN atn2 = ParserInterpreter.deserialize(data);
atn2.g = g;
String atn2Data = ATNSerializer.getDecoded(atn2);
String atn2Data = ATNSerializer.getDecoded(g, atn2);
assertEquals(atnData, atn2Data);
}

30
tool/test/org/antlr/v4/test/TestATNSerialization.java
View File

@ -27,7 +27,7 @@ public class TestATNSerialization extends BaseTest {
"4->5 ATOM 4,0\n" +
"5->1 EPSILON 0,0\n";
ATN atn = createATN(g);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result);
}
@ -49,7 +49,7 @@ public class TestATNSerialization extends BaseTest {
"2->3 NOT_ATOM 3,0\n" +
"3->1 EPSILON 0,0\n";
ATN atn = createATN(g);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result);
}
@ -71,7 +71,7 @@ public class TestATNSerialization extends BaseTest {
"2->3 WILDCARD 0,0\n" +
"3->1 EPSILON 0,0\n";
ATN atn = createATN(g);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result);
}
@ -106,7 +106,7 @@ public class TestATNSerialization extends BaseTest {
"9->1 EPSILON 0,0\n" +
"0:8\n";
ATN atn = createATN(g);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result);
}
@ -154,7 +154,7 @@ public class TestATNSerialization extends BaseTest {
"15->1 EPSILON 0,0\n" +
"0:14\n";
ATN atn = createATN(g);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result);
}
@ -191,7 +191,7 @@ public class TestATNSerialization extends BaseTest {
"1:4\n" +
"2:6\n";
ATN atn = createATN(g);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result);
}
@ -222,7 +222,7 @@ public class TestATNSerialization extends BaseTest {
"6->7 ATOM 3,0\n" +
"7->3 EPSILON 0,0\n";
ATN atn = createATN(g);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result);
}
@ -255,7 +255,7 @@ public class TestATNSerialization extends BaseTest {
"8->4 EPSILON 0,0\n" +
"0:0\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
}
@ -278,7 +278,7 @@ public class TestATNSerialization extends BaseTest {
"4->2 EPSILON 0,0\n" +
"0:0\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
}
@ -313,7 +313,7 @@ public class TestATNSerialization extends BaseTest {
"2:5\n" +
"3:7\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
}
@ -360,7 +360,7 @@ public class TestATNSerialization extends BaseTest {
"14->6 EPSILON 0,0\n" +
"0:0\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
}
@ -384,7 +384,7 @@ public class TestATNSerialization extends BaseTest {
"4->2 EPSILON 0,0\n" +
"0:0\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
}
@ -408,7 +408,7 @@ public class TestATNSerialization extends BaseTest {
"4->2 EPSILON 0,0\n" +
"0:0\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
}
@ -437,7 +437,7 @@ public class TestATNSerialization extends BaseTest {
"6->2 EPSILON 0,0\n" +
"0:0\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
}
@ -488,7 +488,7 @@ public class TestATNSerialization extends BaseTest {
"1:1\n" +
"2:2\n";
ATN atn = createATN(lg);
String result = ATNSerializer.getDecoded(atn);
String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result);
}