From 68cef48483c7545e3ef5a52f69159c33b044761f Mon Sep 17 00:00:00 2001 From: parrt Date: Tue, 28 Jun 2011 17:51:39 -0800 Subject: [PATCH] cleaned up; mv'd stuff to parser. reorg to have lexer in controller [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8794] --- .../v4/runtime/tree/BaseTreeAdaptor.java | 21 ++++++++- .../src/org/antlr/v4/runtime/tree/Tree.java | 5 +++ .../antlr/v4/runtime/tree/TreeAdaptor.java | 15 ++++++- tool/playground/T.g | 9 +++- tool/playground/{Test.java => TestT.java} | 9 +--- .../v4/tool/templates/codegen/Java/Java.stg | 8 +++- tool/src/org/antlr/v4/Tool.java | 3 +- .../org/antlr/v4/codegen/CodeGenPipeline.java | 29 ------------ .../org/antlr/v4/codegen/CodeGenerator.java | 15 +++---- .../v4/codegen/CodeGeneratorExtension.java | 8 ++++ .../v4/codegen/DefaultOutputModelFactory.java | 20 ++++----- .../org/antlr/v4/codegen/LexerFactory.java | 9 ---- .../v4/codegen/OutputModelController.java | 40 +++++++++++++++-- .../antlr/v4/codegen/ParserASTExtension.java | 14 ++---- .../org/antlr/v4/codegen/ParserFactory.java | 44 +++++-------------- .../v4/codegen/model/ast/AddRuleLeaf.java | 17 +++++++ .../ast/{AddLeaf.java => AddTokenLeaf.java} | 4 +- 17 files changed, 147 insertions(+), 123 deletions(-) rename tool/playground/{Test.java => TestT.java} (93%) create mode 100644 tool/src/org/antlr/v4/codegen/model/ast/AddRuleLeaf.java rename tool/src/org/antlr/v4/codegen/model/ast/{AddLeaf.java => AddTokenLeaf.java} (74%) diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java b/runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java index fac7254dc..bcb49eae7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java @@ -46,10 +46,29 @@ public abstract class BaseTreeAdaptor implements TreeAdaptor { return null; } - public Object createWithChildren(Object root, List kids) { + /** if oldRoot is null then: + * create root for rootToken using kids. + * return as new root + * if oldRoot not null then: + * add kids to oldRoot + * clear kids + * create rootToken using kids + * return as new root + */ + public Object becomeRoot(Object newRoot, Token rootToken, List kids) { + if ( newRoot==null ) { + newRoot = create(rootToken); + addChildren(newRoot, kids); + kids.clear(); + return newRoot; + } return null; } + public void addChildren(Object root, List kids) { + if ( root!=null ) ((Tree)root).addChildren(kids); + } + // END v4 stuff public Object nil() { diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java b/runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java index 71e2e85d4..fc3083cb9 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java @@ -44,6 +44,11 @@ import java.util.List; public interface Tree { public static final Tree INVALID_NODE = new CommonTree(Token.INVALID_TOKEN); + // BEGIN v4 + void addChildren(List t); + + // END v4 + Tree getChild(int i); int getChildCount(); diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/TreeAdaptor.java b/runtime/Java/src/org/antlr/v4/runtime/tree/TreeAdaptor.java index ef81c5810..7d40323e7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/TreeAdaptor.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/TreeAdaptor.java @@ -43,10 +43,21 @@ import java.util.List; public interface TreeAdaptor { // BEGIN new v4 stuff // x^ - public Object becomeRoot(Object newRoot, Object oldRoot, List kids); + public Object becomeRoot(Object oldRoot, Object newRoot, List kids); + + /** if oldRoot is null then: + * create root for rootToken using kids. + * return as new root + * if oldRoot not null then: + * add kids to oldRoot + * clear kids + * create rootToken using kids + * return as new root + */ + public Object becomeRoot(Object oldRoot, Token rootToken, List kids); // end of outer rule block, set return value - public Object createWithChildren(Object root, List kids); + public void addChildren(Object root, List kids); // END new v4 stuff diff --git a/tool/playground/T.g b/tool/playground/T.g index 5f23b4e9d..29dcd843f 100644 --- a/tool/playground/T.g +++ b/tool/playground/T.g @@ -1,8 +1,13 @@ -parser grammar T; +grammar T; options {output=AST;} -a : x+=A^ y+=b B! b!; +a : x+=A^ y+=b B! B b!; b : B ; + +A : 'a'; +B : 'b'; +WS : ' '|'\t'|'\n' {skip();} ; + /* r[int a] returns [int b] scope {int qq;} diff --git a/tool/playground/Test.java b/tool/playground/TestT.java similarity index 93% rename from tool/playground/Test.java rename to tool/playground/TestT.java index 6500d97db..7c896f789 100644 --- a/tool/playground/Test.java +++ b/tool/playground/TestT.java @@ -7,15 +7,10 @@ import org.antlr.v4.tool.*; import java.util.List; -public class Test { +public class TestT { public static void main(String[] args) throws Exception { -// T t = new T(new ANTLRFileStream(args[0])); +// TLexer t = new TLexer(new ANTLRFileStream(args[0])); // CommonTokenStream tokens = new CommonTokenStream(t); -// tokens.fill(); -// for (Object tok : tokens.getTokens()) { -// System.out.println(tok); -// } - } public static void dump() throws Exception { diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index 6da5b9dbb..c621ae0dc 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -363,9 +363,13 @@ RootDecl(d) ::= "Object = null;" RootName(level) ::= "_root" KidsListName(level) ::= "_kids" -AddLeaf(a) ::= "_kids0.add();" +AddTokenLeaf(a) ::= "_kids0.add(_adaptor.create());" +AddRuleLeaf(a) ::= "_kids0.add(.tree);" BecomeRoot(r) ::= "_root0 = _root0==null ? : _adaptor.becomeRoot(_root0, , _kids0);" -AssignTreeResult(a) ::= "_localctx.tree = _adaptor.createWithChildren(_root0, _kids0);" +AssignTreeResult(a) ::= << +_adaptor.addChildren(_root0, _kids0); +_localctx.tree = _root0; +>> /* BitSetDecl(b) ::= << diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java index 24ba46435..c0431ea88 100644 --- a/tool/src/org/antlr/v4/Tool.java +++ b/tool/src/org/antlr/v4/Tool.java @@ -226,8 +226,9 @@ public class Tool { } // BUILD ATN FROM AST - ATNFactory factory = new ParserATNFactory(g); + ATNFactory factory; if ( g.isLexer() ) factory = new LexerATNFactory((LexerGrammar)g); + else factory = new ParserATNFactory(g); g.atn = factory.createATN(); if ( generate_ATN_dot ) generateATNs(g); diff --git a/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java b/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java index eeb414602..57d78e0fc 100644 --- a/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java +++ b/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java @@ -11,36 +11,7 @@ public class CodeGenPipeline { public void process() { CodeGenerator gen = new CodeGenerator(g); -// for (Rule r : g.rules.values()) { -// for (int i=1; i<=r.numberOfAlts; i++) { -// Alternative alt = r.alt[i]; -// for (String ref : alt.tokenRefs.keySet()) { -// if ( alt.tokenRefsInActions.get(ref)!=null ) { -// String label = gen.target.getImplicitTokenLabel(ast); -// alt.implicitTokenLabels.put(, label); -// } -// } -// } -// } - ST outputFileST = gen.generate(); gen.write(outputFileST); - -// if ( g.isLexer() ) processLexer(); -// else if ( g.isParser() ) processParser(); - } - - void processParser() { - CodeGenerator gen = new CodeGenerator(g); - ST outputFileST = gen.generate(); - gen.write(outputFileST); - - } - - void processLexer() { - CodeGenerator gen = new CodeGenerator(g); - ST outputFileST = gen.generate(); - gen.write(outputFileST); - } } diff --git a/tool/src/org/antlr/v4/codegen/CodeGenerator.java b/tool/src/org/antlr/v4/codegen/CodeGenerator.java index da0160300..43365722e 100644 --- a/tool/src/org/antlr/v4/codegen/CodeGenerator.java +++ b/tool/src/org/antlr/v4/codegen/CodeGenerator.java @@ -75,13 +75,10 @@ public class CodeGenerator { } public ST generate() { - OutputModelFactory factory = null; - if ( g.isParser() || g.isCombined() || g.isTreeGrammar() ) { - factory = new ParserFactory(this); - } - else { - factory = new LexerFactory(this); - } + OutputModelFactory factory; + if ( g.isLexer() ) factory = new LexerFactory(this); + else factory = new ParserFactory(this); + // TODO: let someone add their own factory? // CREATE OUTPUT MODEL FROM GRAMMAR OBJ AND AST WITHIN RULES @@ -90,7 +87,9 @@ public class CodeGenerator { controller.addExtension( new ParserASTExtension(factory) ); } - OutputModelObject outputModel = controller.buildOutputModel(); + OutputModelObject outputModel; + if ( g.isLexer() ) outputModel = controller.buildLexerOutputModel(); + else outputModel = controller.buildParserOutputModel(); // CREATE TEMPLATES BY WALKING MODEL OutputModelWalker walker = new OutputModelWalker(g.tool, templates); diff --git a/tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java b/tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java index 782bf0df7..a71d9cc08 100644 --- a/tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java +++ b/tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java @@ -17,6 +17,10 @@ public class CodeGeneratorExtension { public Parser parser(Parser p) { return p; } + public LexerFile lexerFile(LexerFile f) { return f; } + + public Lexer lexer(Lexer l) { return l; } + public RuleFunction rule(RuleFunction rf) { return rf; } public List rulePostamble(List ops) { return ops; } @@ -45,6 +49,10 @@ public class CodeGeneratorExtension { public List rootRule(List ops) { return ops; } + public List leafToken(List ops) { return ops; } + + public List leafRule(List ops) { return ops; } + // BLOCKS public List getChoiceBlock(List ops) { return ops; } diff --git a/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java b/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java index 1cf306c40..9f66af2e4 100644 --- a/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java +++ b/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java @@ -30,21 +30,17 @@ public abstract class DefaultOutputModelFactory implements OutputModelFactory { public CodeGenerator getGenerator() { return gen; } - public ParserFile parserFile(String fileName) { - return new ParserFile(this, fileName); - } + public ParserFile parserFile(String fileName) { return null; } - public Parser parser(ParserFile file) { - return new Parser(this, file); - } + public Parser parser(ParserFile file) { return null; } - public RuleFunction rule(Rule r) { - return new RuleFunction(this, r); - } + public RuleFunction rule(Rule r) { return null; } - public List rulePostamble(RuleFunction function, Rule r) { - return null; - } + public List rulePostamble(RuleFunction function, Rule r) { return null; } + + public LexerFile lexerFile(String fileName) { return null; } + + public Lexer lexer(LexerFile file) { return null; } // ALTERNATIVES / ELEMENTS diff --git a/tool/src/org/antlr/v4/codegen/LexerFactory.java b/tool/src/org/antlr/v4/codegen/LexerFactory.java index 341488e81..78cc08a83 100644 --- a/tool/src/org/antlr/v4/codegen/LexerFactory.java +++ b/tool/src/org/antlr/v4/codegen/LexerFactory.java @@ -1,15 +1,6 @@ package org.antlr.v4.codegen; -import org.antlr.v4.codegen.model.*; - /** */ public class LexerFactory extends DefaultOutputModelFactory { public LexerFactory(CodeGenerator gen) { super(gen); } - - public OutputModelObject buildOutputModel(OutputModelController controller) { - LexerFile file = new LexerFile(this, getGenerator().getRecognizerFileName()); - setRoot(file); - file.lexer = new Lexer(this, file); - return file; - } } diff --git a/tool/src/org/antlr/v4/codegen/OutputModelController.java b/tool/src/org/antlr/v4/codegen/OutputModelController.java index eadf0f284..ef2dbc25d 100644 --- a/tool/src/org/antlr/v4/codegen/OutputModelController.java +++ b/tool/src/org/antlr/v4/codegen/OutputModelController.java @@ -1,6 +1,6 @@ package org.antlr.v4.codegen; -import org.antlr.runtime.tree.CommonTreeNodeStream; +import org.antlr.runtime.tree.*; import org.antlr.v4.codegen.model.*; import org.antlr.v4.parse.*; import org.antlr.v4.runtime.misc.IntervalSet; @@ -28,7 +28,7 @@ public class OutputModelController implements OutputModelFactory { * controller as factory in SourceGenTriggers so it triggers codegen * extensions too, not just the factory functions in this factory. */ - public OutputModelObject buildOutputModel() { + public OutputModelObject buildParserOutputModel() { Grammar g = delegate.getGrammar(); CodeGenerator gen = delegate.getGenerator(); ParserFile file = parserFile(gen.getRecognizerFileName()); @@ -65,6 +65,14 @@ public class OutputModelController implements OutputModelFactory { return file; } + public OutputModelObject buildLexerOutputModel() { + CodeGenerator gen = delegate.getGenerator(); + LexerFile file = lexerFile(gen.getRecognizerFileName()); + setRoot(file); + file.lexer = lexer(file); + return file; + } + public ParserFile parserFile(String fileName) { ParserFile f = delegate.parserFile(fileName); for (CodeGeneratorExtension ext : extensions) f = ext.parserFile(f); @@ -77,6 +85,14 @@ public class OutputModelController implements OutputModelFactory { return p; } + public LexerFile lexerFile(String fileName) { + return new LexerFile(this, getGenerator().getRecognizerFileName()); + } + + public Lexer lexer(LexerFile file) { + return new Lexer(this, file); + } + public RuleFunction rule(Rule r) { RuleFunction rf = delegate.rule(r); for (CodeGeneratorExtension ext : extensions) rf = ext.rule(rf); @@ -101,13 +117,29 @@ public class OutputModelController implements OutputModelFactory { public List ruleRef(GrammarAST ID, GrammarAST label, GrammarAST args) { List ops = delegate.ruleRef(ID, label, args); - for (CodeGeneratorExtension ext : extensions) ops = ext.ruleRef(ops); + for (CodeGeneratorExtension ext : extensions) { + ops = ext.ruleRef(ops); + Tree parent = ID.getParent(); + if ( parent!=null && parent.getType()!=ANTLRParser.BANG && + parent.getType()!=ANTLRParser.ROOT ) + { + ops = ext.leafRule(ops); + } + } return ops; } public List tokenRef(GrammarAST ID, GrammarAST label, GrammarAST args) { List ops = delegate.tokenRef(ID, label, args); - for (CodeGeneratorExtension ext : extensions) ops = ext.tokenRef(ops); + for (CodeGeneratorExtension ext : extensions) { + ops = ext.tokenRef(ops); + Tree parent = ID.getParent(); + if ( parent!=null && parent.getType()!=ANTLRParser.BANG && + parent.getType()!=ANTLRParser.ROOT ) + { + ops = ext.leafToken(ops); + } + } return ops; } diff --git a/tool/src/org/antlr/v4/codegen/ParserASTExtension.java b/tool/src/org/antlr/v4/codegen/ParserASTExtension.java index 21b06a434..29b00becc 100644 --- a/tool/src/org/antlr/v4/codegen/ParserASTExtension.java +++ b/tool/src/org/antlr/v4/codegen/ParserASTExtension.java @@ -1,10 +1,8 @@ package org.antlr.v4.codegen; -import org.antlr.runtime.tree.Tree; import org.antlr.v4.codegen.model.*; import org.antlr.v4.codegen.model.ast.*; import org.antlr.v4.misc.Utils; -import org.antlr.v4.parse.ANTLRParser; import java.util.List; @@ -34,20 +32,16 @@ public class ParserASTExtension extends CodeGeneratorExtension { } @Override - public List ruleRef(List ops) { + public List leafRule(List ops) { InvokeRule invokeOp = (InvokeRule)Utils.find(ops, InvokeRule.class); - Tree parent = invokeOp.ast.getParent(); - if ( parent!=null && parent.getType()==ANTLRParser.BANG ) return ops; - SrcOp treeOp = new AddLeaf(factory, invokeOp.ast, invokeOp); + SrcOp treeOp = new AddRuleLeaf(factory, invokeOp.ast, invokeOp); return DefaultOutputModelFactory.list(ops, treeOp); } @Override - public List tokenRef(List ops) { + public List leafToken(List ops) { MatchToken matchOp = (MatchToken)Utils.find(ops, MatchToken.class); - Tree parent = matchOp.ast.getParent(); - if ( parent!=null && parent.getType()==ANTLRParser.BANG ) return ops; - SrcOp treeOp = new AddLeaf(factory, matchOp.ast, matchOp); + SrcOp treeOp = new AddTokenLeaf(factory, matchOp.ast, matchOp); return DefaultOutputModelFactory.list(ops, treeOp); } } diff --git a/tool/src/org/antlr/v4/codegen/ParserFactory.java b/tool/src/org/antlr/v4/codegen/ParserFactory.java index d2421ceae..cd0b9eff3 100644 --- a/tool/src/org/antlr/v4/codegen/ParserFactory.java +++ b/tool/src/org/antlr/v4/codegen/ParserFactory.java @@ -14,41 +14,17 @@ import java.util.List; public class ParserFactory extends DefaultOutputModelFactory { public ParserFactory(CodeGenerator gen) { super(gen); } - /** Build a file with a parser containing rule functions. Use the - * controller as factory in SourceGenTriggers so it triggers codegen - * extensions too, not just the factory functions in this factory. - public OutputModelObject buildOutputModel(OutputModelController controller) { - ParserFile file = new ParserFile(this, gen.getRecognizerFileName()); - setRoot(file); - Parser parser = new Parser(this, file); - file.parser = parser; - for (Rule r : g.rules.values()) { - RuleFunction function = new RuleFunction(this, r); - parser.funcs.add(function); - - // TRIGGER factory functions for rule alts, elements - currentRule.push(function); - GrammarASTAdaptor adaptor = new GrammarASTAdaptor(r.ast.token.getInputStream()); - GrammarAST blk = (GrammarAST)r.ast.getFirstChildWithType(ANTLRParser.BLOCK); - CommonTreeNodeStream nodes = new CommonTreeNodeStream(adaptor,blk); - SourceGenTriggers genTriggers = new SourceGenTriggers(nodes, controller); - try { - function.code = genTriggers.block(null,null); // walk AST of rule alts/elements - } - catch (Exception e){ - e.printStackTrace(System.err); - } - - function.ctxType = gen.target.getRuleFunctionContextStructName(r); - function.ruleCtx.name = function.ctxType; - - if ( function.ruleCtx.isEmpty() ) function.ruleCtx = null; - currentRule.pop(); - } - - return file; + public ParserFile parserFile(String fileName) { + return new ParserFile(this, fileName); + } + + public Parser parser(ParserFile file) { + return new Parser(this, file); + } + + public RuleFunction rule(Rule r) { + return new RuleFunction(this, r); } - **/ public List epsilon() { return list(new CodeBlock(this)); } diff --git a/tool/src/org/antlr/v4/codegen/model/ast/AddRuleLeaf.java b/tool/src/org/antlr/v4/codegen/model/ast/AddRuleLeaf.java new file mode 100644 index 000000000..99218c574 --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/model/ast/AddRuleLeaf.java @@ -0,0 +1,17 @@ +package org.antlr.v4.codegen.model.ast; + +import org.antlr.v4.codegen.OutputModelFactory; +import org.antlr.v4.codegen.model.*; +import org.antlr.v4.codegen.model.decl.Decl; +import org.antlr.v4.tool.GrammarAST; + +public class AddRuleLeaf extends SrcOp { + public LabeledOp opWithResultToAdd; + public Decl label; + + public AddRuleLeaf(OutputModelFactory factory, GrammarAST ast, LabeledOp opWithResultToAdd) { + super(factory, ast); + this.opWithResultToAdd = opWithResultToAdd; + label = opWithResultToAdd.getLabels().get(0); + } +} \ No newline at end of file diff --git a/tool/src/org/antlr/v4/codegen/model/ast/AddLeaf.java b/tool/src/org/antlr/v4/codegen/model/ast/AddTokenLeaf.java similarity index 74% rename from tool/src/org/antlr/v4/codegen/model/ast/AddLeaf.java rename to tool/src/org/antlr/v4/codegen/model/ast/AddTokenLeaf.java index 26d21f976..7a6d37dc0 100644 --- a/tool/src/org/antlr/v4/codegen/model/ast/AddLeaf.java +++ b/tool/src/org/antlr/v4/codegen/model/ast/AddTokenLeaf.java @@ -6,11 +6,11 @@ import org.antlr.v4.codegen.model.decl.Decl; import org.antlr.v4.tool.GrammarAST; /** */ -public class AddLeaf extends SrcOp { +public class AddTokenLeaf extends SrcOp { public LabeledOp opWithResultToAdd; public Decl label; - public AddLeaf(OutputModelFactory factory, GrammarAST ast, LabeledOp opWithResultToAdd) { + public AddTokenLeaf(OutputModelFactory factory, GrammarAST ast, LabeledOp opWithResultToAdd) { super(factory, ast); this.opWithResultToAdd = opWithResultToAdd; label = opWithResultToAdd.getLabels().get(0);