forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

cleaned up; mv'd stuff to parser. reorg to have lexer in controller 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8794]
This commit is contained in:
parrt 2011-06-28 17:51:39 -08:00
parent 2d5c4fd686
commit 68cef48483
17 changed files with 147 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java
View File

@ -46,10 +46,29 @@ public abstract class BaseTreeAdaptor implements TreeAdaptor {
return null;
}
public Object createWithChildren(Object root, List kids) {
/** if oldRoot is null then:
* create root for rootToken using kids.
* return as new root
* if oldRoot not null then:
* add kids to oldRoot
* clear kids
* create rootToken using kids
* return as new root
*/
public Object becomeRoot(Object newRoot, Token rootToken, List kids) {
if ( newRoot==null ) {
newRoot = create(rootToken);
addChildren(newRoot, kids);
kids.clear();
return newRoot;
}
return null;
}
public void addChildren(Object root, List kids) {
if ( root!=null ) ((Tree)root).addChildren(kids);
}
// END v4 stuff
public Object nil() {

5
runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java
View File

@ -44,6 +44,11 @@ import java.util.List;
public interface Tree {
public static final Tree INVALID_NODE = new CommonTree(Token.INVALID_TOKEN);
// BEGIN v4
void addChildren(List t);
// END v4
Tree getChild(int i);
int getChildCount();

15
runtime/Java/src/org/antlr/v4/runtime/tree/TreeAdaptor.java
View File

@ -43,10 +43,21 @@ import java.util.List;
public interface TreeAdaptor {
// BEGIN new v4 stuff
// x^
public Object becomeRoot(Object newRoot, Object oldRoot, List kids);
public Object becomeRoot(Object oldRoot, Object newRoot, List kids);
/** if oldRoot is null then:
* create root for rootToken using kids.
* return as new root
* if oldRoot not null then:
* add kids to oldRoot
* clear kids
* create rootToken using kids
* return as new root
*/
public Object becomeRoot(Object oldRoot, Token rootToken, List kids);
// end of outer rule block, set return value
public Object createWithChildren(Object root, List kids);
public void addChildren(Object root, List kids);
// END new v4 stuff

9
tool/playground/T.g
View File

@ -1,8 +1,13 @@
parser grammar T;
grammar T;
options {output=AST;}
a : x+=A^ y+=b B! b!;
a : x+=A^ y+=b B! B b!;
b : B ;
A : 'a';
B : 'b';
WS : ' '|'\t'|'\n' {skip();} ;
/*
r[int a] returns [int b]
scope {int qq;}

9
tool/playground/Test.java → tool/playground/TestT.java
View File

@ -7,15 +7,10 @@ import org.antlr.v4.tool.*;
import java.util.List;
public class Test {
public class TestT {
public static void main(String[] args) throws Exception {
// T t = new T(new ANTLRFileStream(args[0]));
// TLexer t = new TLexer(new ANTLRFileStream(args[0]));
// CommonTokenStream tokens = new CommonTokenStream(t);
// tokens.fill();
// for (Object tok : tokens.getTokens()) {
// System.out.println(tok);
// }
}
public static void dump() throws Exception {

8
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -363,9 +363,13 @@ RootDecl(d) ::= "Object <d.name> = null;"
RootName(level) ::= "_root<level>"
KidsListName(level) ::= "_kids<level>"
AddLeaf(a) ::= "_kids0.add(<labelref(a.label)>);"
AddTokenLeaf(a) ::= "_kids0.add(_adaptor.create(<labelref(a.label)>));"
AddRuleLeaf(a) ::= "_kids0.add(<labelref(a.label)>.tree);"
BecomeRoot(r) ::= "_root0 = _root0==null ? <labelref(r.label)> : _adaptor.becomeRoot(_root0, <labelref(r.label)>, _kids0);"
AssignTreeResult(a) ::= "_localctx.tree = _adaptor.createWithChildren(_root0, _kids0);"
AssignTreeResult(a) ::= <<
_adaptor.addChildren(_root0, _kids0);
_localctx.tree = _root0;
>>
/*
BitSetDecl(b) ::= <<

3
tool/src/org/antlr/v4/Tool.java
View File

@ -226,8 +226,9 @@ public class Tool {
}
// BUILD ATN FROM AST
ATNFactory factory = new ParserATNFactory(g);
ATNFactory factory;
if ( g.isLexer() ) factory = new LexerATNFactory((LexerGrammar)g);
else factory = new ParserATNFactory(g);
g.atn = factory.createATN();
if ( generate_ATN_dot ) generateATNs(g);

29
tool/src/org/antlr/v4/codegen/CodeGenPipeline.java
View File

@ -11,36 +11,7 @@ public class CodeGenPipeline {
public void process() {
CodeGenerator gen = new CodeGenerator(g);
// for (Rule r : g.rules.values()) {
// for (int i=1; i<=r.numberOfAlts; i++) {
// Alternative alt = r.alt[i];
// for (String ref : alt.tokenRefs.keySet()) {
// if ( alt.tokenRefsInActions.get(ref)!=null ) {
// String label = gen.target.getImplicitTokenLabel(ast);
// alt.implicitTokenLabels.put(, label);
// }
// }
// }
// }
ST outputFileST = gen.generate();
gen.write(outputFileST);
// if ( g.isLexer() ) processLexer();
// else if ( g.isParser() ) processParser();
}
void processParser() {
CodeGenerator gen = new CodeGenerator(g);
ST outputFileST = gen.generate();
gen.write(outputFileST);
}
void processLexer() {
CodeGenerator gen = new CodeGenerator(g);
ST outputFileST = gen.generate();
gen.write(outputFileST);
}
}

15
tool/src/org/antlr/v4/codegen/CodeGenerator.java
View File

@ -75,13 +75,10 @@ public class CodeGenerator {
}
public ST generate() {
OutputModelFactory factory = null;
if ( g.isParser() || g.isCombined() || g.isTreeGrammar() ) {
factory = new ParserFactory(this);
}
else {
factory = new LexerFactory(this);
}
OutputModelFactory factory;
if ( g.isLexer() ) factory = new LexerFactory(this);
else factory = new ParserFactory(this);
// TODO: let someone add their own factory?
// CREATE OUTPUT MODEL FROM GRAMMAR OBJ AND AST WITHIN RULES
@ -90,7 +87,9 @@ public class CodeGenerator {
controller.addExtension( new ParserASTExtension(factory) );
}
OutputModelObject outputModel = controller.buildOutputModel();
OutputModelObject outputModel;
if ( g.isLexer() ) outputModel = controller.buildLexerOutputModel();
else outputModel = controller.buildParserOutputModel();
// CREATE TEMPLATES BY WALKING MODEL
OutputModelWalker walker = new OutputModelWalker(g.tool, templates);

8
tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java
View File

@ -17,6 +17,10 @@ public class CodeGeneratorExtension {
public Parser parser(Parser p) { return p; }
public LexerFile lexerFile(LexerFile f) { return f; }
public Lexer lexer(Lexer l) { return l; }
public RuleFunction rule(RuleFunction rf) { return rf; }
public List<SrcOp> rulePostamble(List<SrcOp> ops) { return ops; }
@ -45,6 +49,10 @@ public class CodeGeneratorExtension {
public List<SrcOp> rootRule(List<SrcOp> ops) { return ops; }
public List<SrcOp> leafToken(List<SrcOp> ops) { return ops; }
public List<SrcOp> leafRule(List<SrcOp> ops) { return ops; }
// BLOCKS
public List<SrcOp> getChoiceBlock(List<SrcOp> ops) { return ops; }

20
tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java
View File

@ -30,21 +30,17 @@ public abstract class DefaultOutputModelFactory implements OutputModelFactory {
public CodeGenerator getGenerator() { return gen; }
public ParserFile parserFile(String fileName) {
return new ParserFile(this, fileName);
}
public ParserFile parserFile(String fileName) { return null; }
public Parser parser(ParserFile file) {
return new Parser(this, file);
}
public Parser parser(ParserFile file) { return null; }
public RuleFunction rule(Rule r) {
return new RuleFunction(this, r);
}
public RuleFunction rule(Rule r) { return null; }
public List<SrcOp> rulePostamble(RuleFunction function, Rule r) {
return null;
}
public List<SrcOp> rulePostamble(RuleFunction function, Rule r) { return null; }
public LexerFile lexerFile(String fileName) { return null; }
public Lexer lexer(LexerFile file) { return null; }
// ALTERNATIVES / ELEMENTS

9
tool/src/org/antlr/v4/codegen/LexerFactory.java
View File

@ -1,15 +1,6 @@
package org.antlr.v4.codegen;
import org.antlr.v4.codegen.model.*;
/** */
public class LexerFactory extends DefaultOutputModelFactory {
public LexerFactory(CodeGenerator gen) { super(gen); }
public OutputModelObject buildOutputModel(OutputModelController controller) {
LexerFile file = new LexerFile(this, getGenerator().getRecognizerFileName());
setRoot(file);
file.lexer = new Lexer(this, file);
return file;
}
}

40
tool/src/org/antlr/v4/codegen/OutputModelController.java
View File

@ -1,6 +1,6 @@
package org.antlr.v4.codegen;
import org.antlr.runtime.tree.CommonTreeNodeStream;
import org.antlr.runtime.tree.*;
import org.antlr.v4.codegen.model.*;
import org.antlr.v4.parse.*;
import org.antlr.v4.runtime.misc.IntervalSet;
@ -28,7 +28,7 @@ public class OutputModelController implements OutputModelFactory {
* controller as factory in SourceGenTriggers so it triggers codegen
* extensions too, not just the factory functions in this factory.
*/
public OutputModelObject buildOutputModel() {
public OutputModelObject buildParserOutputModel() {
Grammar g = delegate.getGrammar();
CodeGenerator gen = delegate.getGenerator();
ParserFile file = parserFile(gen.getRecognizerFileName());
@ -65,6 +65,14 @@ public class OutputModelController implements OutputModelFactory {
return file;
}
public OutputModelObject buildLexerOutputModel() {
CodeGenerator gen = delegate.getGenerator();
LexerFile file = lexerFile(gen.getRecognizerFileName());
setRoot(file);
file.lexer = lexer(file);
return file;
}
public ParserFile parserFile(String fileName) {
ParserFile f = delegate.parserFile(fileName);
for (CodeGeneratorExtension ext : extensions) f = ext.parserFile(f);
@ -77,6 +85,14 @@ public class OutputModelController implements OutputModelFactory {
return p;
}
public LexerFile lexerFile(String fileName) {
return new LexerFile(this, getGenerator().getRecognizerFileName());
}
public Lexer lexer(LexerFile file) {
return new Lexer(this, file);
}
public RuleFunction rule(Rule r) {
RuleFunction rf = delegate.rule(r);
for (CodeGeneratorExtension ext : extensions) rf = ext.rule(rf);
@ -101,13 +117,29 @@ public class OutputModelController implements OutputModelFactory {
public List<SrcOp> ruleRef(GrammarAST ID, GrammarAST label, GrammarAST args) {
List<SrcOp> ops = delegate.ruleRef(ID, label, args);
for (CodeGeneratorExtension ext : extensions) ops = ext.ruleRef(ops);
for (CodeGeneratorExtension ext : extensions) {
ops = ext.ruleRef(ops);
Tree parent = ID.getParent();
if ( parent!=null && parent.getType()!=ANTLRParser.BANG &&
parent.getType()!=ANTLRParser.ROOT )
{
ops = ext.leafRule(ops);
}
}
return ops;
}
public List<SrcOp> tokenRef(GrammarAST ID, GrammarAST label, GrammarAST args) {
List<SrcOp> ops = delegate.tokenRef(ID, label, args);
for (CodeGeneratorExtension ext : extensions) ops = ext.tokenRef(ops);
for (CodeGeneratorExtension ext : extensions) {
ops = ext.tokenRef(ops);
Tree parent = ID.getParent();
if ( parent!=null && parent.getType()!=ANTLRParser.BANG &&
parent.getType()!=ANTLRParser.ROOT )
{
ops = ext.leafToken(ops);
}
}
return ops;
}

14
tool/src/org/antlr/v4/codegen/ParserASTExtension.java
View File

@ -1,10 +1,8 @@
package org.antlr.v4.codegen;
import org.antlr.runtime.tree.Tree;
import org.antlr.v4.codegen.model.*;
import org.antlr.v4.codegen.model.ast.*;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.parse.ANTLRParser;
import java.util.List;
@ -34,20 +32,16 @@ public class ParserASTExtension extends CodeGeneratorExtension {
}
@Override
public List<SrcOp> ruleRef(List<SrcOp> ops) {
public List<SrcOp> leafRule(List<SrcOp> ops) {
InvokeRule invokeOp = (InvokeRule)Utils.find(ops, InvokeRule.class);
Tree parent = invokeOp.ast.getParent();
if ( parent!=null && parent.getType()==ANTLRParser.BANG ) return ops;
SrcOp treeOp = new AddLeaf(factory, invokeOp.ast, invokeOp);
SrcOp treeOp = new AddRuleLeaf(factory, invokeOp.ast, invokeOp);
return DefaultOutputModelFactory.list(ops, treeOp);
}
@Override
public List<SrcOp> tokenRef(List<SrcOp> ops) {
public List<SrcOp> leafToken(List<SrcOp> ops) {
MatchToken matchOp = (MatchToken)Utils.find(ops, MatchToken.class);
Tree parent = matchOp.ast.getParent();
if ( parent!=null && parent.getType()==ANTLRParser.BANG ) return ops;
SrcOp treeOp = new AddLeaf(factory, matchOp.ast, matchOp);
SrcOp treeOp = new AddTokenLeaf(factory, matchOp.ast, matchOp);
return DefaultOutputModelFactory.list(ops, treeOp);
}
}

36
tool/src/org/antlr/v4/codegen/ParserFactory.java
View File

@ -14,41 +14,17 @@ import java.util.List;
public class ParserFactory extends DefaultOutputModelFactory {
public ParserFactory(CodeGenerator gen) { super(gen); }
/** Build a file with a parser containing rule functions. Use the
* controller as factory in SourceGenTriggers so it triggers codegen
* extensions too, not just the factory functions in this factory.
public OutputModelObject buildOutputModel(OutputModelController controller) {
ParserFile file = new ParserFile(this, gen.getRecognizerFileName());
setRoot(file);
Parser parser = new Parser(this, file);
file.parser = parser;
for (Rule r : g.rules.values()) {
RuleFunction function = new RuleFunction(this, r);
parser.funcs.add(function);
// TRIGGER factory functions for rule alts, elements
currentRule.push(function);
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(r.ast.token.getInputStream());
GrammarAST blk = (GrammarAST)r.ast.getFirstChildWithType(ANTLRParser.BLOCK);
CommonTreeNodeStream nodes = new CommonTreeNodeStream(adaptor,blk);
SourceGenTriggers genTriggers = new SourceGenTriggers(nodes, controller);
try {
function.code = genTriggers.block(null,null); // walk AST of rule alts/elements
}
catch (Exception e){
e.printStackTrace(System.err);
public ParserFile parserFile(String fileName) {
return new ParserFile(this, fileName);
}
function.ctxType = gen.target.getRuleFunctionContextStructName(r);
function.ruleCtx.name = function.ctxType;
if ( function.ruleCtx.isEmpty() ) function.ruleCtx = null;
currentRule.pop();
public Parser parser(ParserFile file) {
return new Parser(this, file);
}
return file;
public RuleFunction rule(Rule r) {
return new RuleFunction(this, r);
}
**/
public List<SrcOp> epsilon() { return list(new CodeBlock(this)); }

17
tool/src/org/antlr/v4/codegen/model/ast/AddRuleLeaf.java Normal file
View File

@ -0,0 +1,17 @@
package org.antlr.v4.codegen.model.ast;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.model.*;
import org.antlr.v4.codegen.model.decl.Decl;
import org.antlr.v4.tool.GrammarAST;
public class AddRuleLeaf extends SrcOp {
public LabeledOp opWithResultToAdd;
public Decl label;
public AddRuleLeaf(OutputModelFactory factory, GrammarAST ast, LabeledOp opWithResultToAdd) {
super(factory, ast);
this.opWithResultToAdd = opWithResultToAdd;
label = opWithResultToAdd.getLabels().get(0);
}
}

4
tool/src/org/antlr/v4/codegen/model/ast/AddLeaf.java → tool/src/org/antlr/v4/codegen/model/ast/AddTokenLeaf.java
View File

@ -6,11 +6,11 @@ import org.antlr.v4.codegen.model.decl.Decl;
import org.antlr.v4.tool.GrammarAST;
/** */
public class AddLeaf extends SrcOp {
public class AddTokenLeaf extends SrcOp {
public LabeledOp opWithResultToAdd;
public Decl label;
public AddLeaf(OutputModelFactory factory, GrammarAST ast, LabeledOp opWithResultToAdd) {
public AddTokenLeaf(OutputModelFactory factory, GrammarAST ast, LabeledOp opWithResultToAdd) {
super(factory, ast);
this.opWithResultToAdd = opWithResultToAdd;
label = opWithResultToAdd.getLabels().get(0);