cleaned up; mv'd stuff to parser. reorg to have lexer in controller

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8794]

runtime/Java/src/org/antlr/v4/runtime/tree/BaseTreeAdaptor.java

@@ -46,10 +46,29 @@ public abstract class BaseTreeAdaptor implements TreeAdaptor {
 		return null;
 	}
 
-	public Object createWithChildren(Object root, List kids) {
+	/** if oldRoot is null then:
+	 * 		create root for rootToken using kids.
+	 *  	return as new root
+	 *  if oldRoot not null then:
+	 *  	add kids to oldRoot
+	 *  	clear kids
+	 *  	create rootToken using kids
+	 *  	return as new root
+	 */
+	public Object becomeRoot(Object newRoot, Token rootToken, List kids) {
+		if ( newRoot==null ) {
+			newRoot = create(rootToken);
+			addChildren(newRoot, kids);
+			kids.clear();
+			return newRoot;
+		}
 		return null;
 	}
 
+	public void addChildren(Object root, List kids) {
+		if ( root!=null ) ((Tree)root).addChildren(kids);
+	}
+
 	// END v4 stuff
 
 	public Object nil() {

runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java

@@ -44,6 +44,11 @@ import java.util.List;
 public interface Tree {
 	public static final Tree INVALID_NODE = new CommonTree(Token.INVALID_TOKEN);
 
+	// BEGIN v4
+	void addChildren(List t);
+
+	// END v4
+
 	Tree getChild(int i);
 
 	int getChildCount();

runtime/Java/src/org/antlr/v4/runtime/tree/TreeAdaptor.java

@@ -43,10 +43,21 @@ import java.util.List;
 public interface TreeAdaptor {
 	// BEGIN new v4 stuff
 	// x^
-	public Object becomeRoot(Object newRoot, Object oldRoot, List kids);
+	public Object becomeRoot(Object oldRoot, Object newRoot, List kids);
+
+	/** if oldRoot is null then:
+	 * 		create root for rootToken using kids.
+	 *  	return as new root
+	 *  if oldRoot not null then:
+	 *  	add kids to oldRoot
+	 *  	clear kids
+	 *  	create rootToken using kids
+	 *  	return as new root
+	 */
+	public Object becomeRoot(Object oldRoot, Token rootToken, List kids);
 
 	// end of outer rule block, set return value
-	public Object createWithChildren(Object root, List kids);
+	public void addChildren(Object root, List kids);
 
 	// END new v4 stuff
 

tool/playground/T.g

@@ -1,8 +1,13 @@
-parser grammar T;
+grammar T;
 options {output=AST;}
-a : x+=A^ y+=b B! b!;
+a : x+=A^ y+=b B! B b!;
 
 b : B ;
+
+A : 'a';
+B : 'b';
+WS : ' '|'\t'|'\n' {skip();} ;
+
 /*
 r[int a] returns [int b]
 scope {int qq;}

tool/playground/TestT.java

@@ -7,15 +7,10 @@ import org.antlr.v4.tool.*;
 
 import java.util.List;
 
-public class Test {
+public class TestT {
 	public static void main(String[] args) throws Exception {
-//		T t = new T(new ANTLRFileStream(args[0]));
+//		TLexer t = new TLexer(new ANTLRFileStream(args[0]));
 //		CommonTokenStream tokens = new CommonTokenStream(t);
-//		tokens.fill();
-//		for (Object tok : tokens.getTokens()) {
-//			System.out.println(tok);
-//		}
-
 	}
 
 	public static void dump() throws Exception {

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -363,9 +363,13 @@ RootDecl(d)     ::= "Object <d.name> = null;"
 RootName(level)     ::= "_root<level>"
 KidsListName(level) ::= "_kids<level>"
 
-AddLeaf(a) ::= "_kids0.add(<labelref(a.label)>);"
+AddTokenLeaf(a) ::= "_kids0.add(_adaptor.create(<labelref(a.label)>));"
+AddRuleLeaf(a) ::= "_kids0.add(<labelref(a.label)>.tree);"
 BecomeRoot(r) ::= "_root0 = _root0==null ? <labelref(r.label)> : _adaptor.becomeRoot(_root0, <labelref(r.label)>, _kids0);"
-AssignTreeResult(a) ::= "_localctx.tree = _adaptor.createWithChildren(_root0, _kids0);"
+AssignTreeResult(a) ::= <<
+_adaptor.addChildren(_root0, _kids0);
+_localctx.tree = _root0;
+>>
 
 /*
 BitSetDecl(b) ::= <<

tool/src/org/antlr/v4/Tool.java

@@ -226,8 +226,9 @@ public class Tool {
 		}
 
 		// BUILD ATN FROM AST
-		ATNFactory factory = new ParserATNFactory(g);
+		ATNFactory factory;
 		if ( g.isLexer() ) factory = new LexerATNFactory((LexerGrammar)g);
+		else factory = new ParserATNFactory(g);
 		g.atn = factory.createATN();
 
 		if ( generate_ATN_dot ) generateATNs(g);

tool/src/org/antlr/v4/codegen/CodeGenPipeline.java

@@ -11,36 +11,7 @@ public class CodeGenPipeline {
 	public void process() {
 		CodeGenerator gen = new CodeGenerator(g);
 
-//		for (Rule r : g.rules.values()) {
-//			for (int i=1; i<=r.numberOfAlts; i++) {
-//				Alternative alt = r.alt[i];
-//				for (String ref : alt.tokenRefs.keySet()) {
-//					if ( alt.tokenRefsInActions.get(ref)!=null ) {
-//						String label = gen.target.getImplicitTokenLabel(ast);
-//						alt.implicitTokenLabels.put(, label);
-//					}
-//				}
-//			}
-//		}
-
 		ST outputFileST = gen.generate();
 		gen.write(outputFileST);
-
-//		if ( g.isLexer() ) processLexer();
-//		else if ( g.isParser() ) processParser();
-	}
-
-	void processParser() {
-		CodeGenerator gen = new CodeGenerator(g);
-		ST outputFileST = gen.generate();
-		gen.write(outputFileST);
-
-	}
-
-	void processLexer() {
-		CodeGenerator gen = new CodeGenerator(g);
-		ST outputFileST = gen.generate();
-		gen.write(outputFileST);
-
 	}
 }

tool/src/org/antlr/v4/codegen/CodeGenerator.java

@@ -75,13 +75,10 @@ public class CodeGenerator {
 	}
 
 	public ST generate() {
-		OutputModelFactory factory = null;
+		OutputModelFactory factory;
-		if ( g.isParser() || g.isCombined() || g.isTreeGrammar() ) {
+		if ( g.isLexer() ) factory = new LexerFactory(this);
-			factory = new ParserFactory(this);
+		else factory = new ParserFactory(this);
-		}
+
-		else {
-			factory = new LexerFactory(this);
-		}
 		// TODO: let someone add their own factory?
 
 		// CREATE OUTPUT MODEL FROM GRAMMAR OBJ AND AST WITHIN RULES
@@ -90,7 +87,9 @@ public class CodeGenerator {
 			controller.addExtension( new ParserASTExtension(factory) );
 		}
 
-		OutputModelObject outputModel = controller.buildOutputModel();
+		OutputModelObject outputModel;
+		if ( g.isLexer() ) outputModel = controller.buildLexerOutputModel();
+		else outputModel = controller.buildParserOutputModel();
 
 		// CREATE TEMPLATES BY WALKING MODEL
 		OutputModelWalker walker = new OutputModelWalker(g.tool, templates);

tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java

@@ -17,6 +17,10 @@ public class CodeGeneratorExtension {
 
 	public Parser parser(Parser p) { return p; }
 
+	public LexerFile lexerFile(LexerFile f) { return f; }
+
+	public Lexer lexer(Lexer l) { return l; }
+
 	public RuleFunction rule(RuleFunction rf) { return rf; }
 
 	public List<SrcOp> rulePostamble(List<SrcOp> ops) { return ops; }
@@ -45,6 +49,10 @@ public class CodeGeneratorExtension {
 
 	public List<SrcOp> rootRule(List<SrcOp> ops) { return ops; }
 
+	public List<SrcOp> leafToken(List<SrcOp> ops) { return ops; }
+
+	public List<SrcOp> leafRule(List<SrcOp> ops) { return ops; }
+
 	// BLOCKS
 
 	public List<SrcOp> getChoiceBlock(List<SrcOp> ops) { return ops; }

tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java

@@ -30,21 +30,17 @@ public abstract class DefaultOutputModelFactory implements OutputModelFactory {
 
 	public CodeGenerator getGenerator() { return gen; }
 
-	public ParserFile parserFile(String fileName) {
+	public ParserFile parserFile(String fileName) { return null; }
-		return new ParserFile(this, fileName);
-	}
 
-	public Parser parser(ParserFile file) {
+	public Parser parser(ParserFile file) { return null; }
-		return new Parser(this, file);
-	}
 
-	public RuleFunction rule(Rule r) {
+	public RuleFunction rule(Rule r) { return null; }
-		return new RuleFunction(this, r);
-	}
 
-	public List<SrcOp> rulePostamble(RuleFunction function, Rule r) {
+	public List<SrcOp> rulePostamble(RuleFunction function, Rule r) { return null; }
-		return null;
+
-	}
+	public LexerFile lexerFile(String fileName) { return null; }
+
+	public Lexer lexer(LexerFile file) { return null; }
 
 	// ALTERNATIVES / ELEMENTS
 

tool/src/org/antlr/v4/codegen/LexerFactory.java

@@ -1,15 +1,6 @@
 package org.antlr.v4.codegen;
 
-import org.antlr.v4.codegen.model.*;
-
 /** */
 public class LexerFactory extends DefaultOutputModelFactory {
 	public LexerFactory(CodeGenerator gen) { super(gen); }
-
-	public OutputModelObject buildOutputModel(OutputModelController controller) {
-		LexerFile file = new LexerFile(this, getGenerator().getRecognizerFileName());
-		setRoot(file);
-		file.lexer = new Lexer(this, file);
-		return file;
-	}
 }

tool/src/org/antlr/v4/codegen/OutputModelController.java

@@ -1,6 +1,6 @@
 package org.antlr.v4.codegen;
 
-import org.antlr.runtime.tree.CommonTreeNodeStream;
+import org.antlr.runtime.tree.*;
 import org.antlr.v4.codegen.model.*;
 import org.antlr.v4.parse.*;
 import org.antlr.v4.runtime.misc.IntervalSet;
@@ -28,7 +28,7 @@ public class OutputModelController implements OutputModelFactory {
 	 *  controller as factory in SourceGenTriggers so it triggers codegen
 	 *  extensions too, not just the factory functions in this factory.
 	 */
-	public OutputModelObject buildOutputModel() {
+	public OutputModelObject buildParserOutputModel() {
 		Grammar g = delegate.getGrammar();
 		CodeGenerator gen = delegate.getGenerator();
 		ParserFile file = parserFile(gen.getRecognizerFileName());
@@ -65,6 +65,14 @@ public class OutputModelController implements OutputModelFactory {
 		return file;
 	}
 
+	public OutputModelObject buildLexerOutputModel() {
+		CodeGenerator gen = delegate.getGenerator();
+		LexerFile file = lexerFile(gen.getRecognizerFileName());
+		setRoot(file);
+		file.lexer = lexer(file);
+		return file;
+	}
+
 	public ParserFile parserFile(String fileName) {
 		ParserFile f = delegate.parserFile(fileName);
 		for (CodeGeneratorExtension ext : extensions) f = ext.parserFile(f);
@@ -77,6 +85,14 @@ public class OutputModelController implements OutputModelFactory {
 		return p;
 	}
 
+	public LexerFile lexerFile(String fileName) {
+		return new LexerFile(this, getGenerator().getRecognizerFileName());
+	}
+
+	public Lexer lexer(LexerFile file) {
+		return new Lexer(this, file);
+	}
+
 	public RuleFunction rule(Rule r) {
 		RuleFunction rf = delegate.rule(r);
 		for (CodeGeneratorExtension ext : extensions) rf = ext.rule(rf);
@@ -101,13 +117,29 @@ public class OutputModelController implements OutputModelFactory {
 
 	public List<SrcOp> ruleRef(GrammarAST ID, GrammarAST label, GrammarAST args) {
 		List<SrcOp> ops = delegate.ruleRef(ID, label, args);
-		for (CodeGeneratorExtension ext : extensions) ops = ext.ruleRef(ops);
+		for (CodeGeneratorExtension ext : extensions) {
+			ops = ext.ruleRef(ops);
+			Tree parent = ID.getParent();
+			if ( parent!=null && parent.getType()!=ANTLRParser.BANG &&
+				 parent.getType()!=ANTLRParser.ROOT )
+			{
+				ops = ext.leafRule(ops);
+			}
+		}
 		return ops;
 	}
 
 	public List<SrcOp> tokenRef(GrammarAST ID, GrammarAST label, GrammarAST args) {
 		List<SrcOp> ops = delegate.tokenRef(ID, label, args);
-		for (CodeGeneratorExtension ext : extensions) ops = ext.tokenRef(ops);
+		for (CodeGeneratorExtension ext : extensions) {
+			ops = ext.tokenRef(ops);
+			Tree parent = ID.getParent();
+			if ( parent!=null && parent.getType()!=ANTLRParser.BANG &&
+				 parent.getType()!=ANTLRParser.ROOT )
+			{
+				ops = ext.leafToken(ops);
+			}
+		}
 		return ops;
 	}
 

tool/src/org/antlr/v4/codegen/ParserASTExtension.java

@@ -1,10 +1,8 @@
 package org.antlr.v4.codegen;
 
-import org.antlr.runtime.tree.Tree;
 import org.antlr.v4.codegen.model.*;
 import org.antlr.v4.codegen.model.ast.*;
 import org.antlr.v4.misc.Utils;
-import org.antlr.v4.parse.ANTLRParser;
 
 import java.util.List;
 
@@ -34,20 +32,16 @@ public class ParserASTExtension extends CodeGeneratorExtension {
 	}
 
 	@Override
-	public List<SrcOp> ruleRef(List<SrcOp> ops) {
+	public List<SrcOp> leafRule(List<SrcOp> ops) {
 		InvokeRule invokeOp = (InvokeRule)Utils.find(ops, InvokeRule.class);
-		Tree parent = invokeOp.ast.getParent();
+		SrcOp treeOp = new AddRuleLeaf(factory, invokeOp.ast, invokeOp);
-		if ( parent!=null && parent.getType()==ANTLRParser.BANG ) return ops;
-		SrcOp treeOp = new AddLeaf(factory, invokeOp.ast, invokeOp);
 		return DefaultOutputModelFactory.list(ops, treeOp);
 	}
 
 	@Override
-	public List<SrcOp> tokenRef(List<SrcOp> ops) {
+	public List<SrcOp> leafToken(List<SrcOp> ops) {
 		MatchToken matchOp = (MatchToken)Utils.find(ops, MatchToken.class);
-		Tree parent = matchOp.ast.getParent();
+		SrcOp treeOp = new AddTokenLeaf(factory, matchOp.ast, matchOp);
-		if ( parent!=null && parent.getType()==ANTLRParser.BANG ) return ops;
-		SrcOp treeOp = new AddLeaf(factory, matchOp.ast, matchOp);
 		return DefaultOutputModelFactory.list(ops, treeOp);
 	}
 }

tool/src/org/antlr/v4/codegen/ParserFactory.java

@@ -14,41 +14,17 @@ import java.util.List;
 public class ParserFactory extends DefaultOutputModelFactory {
 	public ParserFactory(CodeGenerator gen) { super(gen); }
 
-	/** Build a file with a parser containing rule functions. Use the
+	public ParserFile parserFile(String fileName) {
-	 *  controller as factory in SourceGenTriggers so it triggers codegen
+		return new ParserFile(this, fileName);
-	 *  extensions too, not just the factory functions in this factory.
+	}
-	public OutputModelObject buildOutputModel(OutputModelController controller) {
+
-		ParserFile file = new ParserFile(this, gen.getRecognizerFileName());
+	public Parser parser(ParserFile file) {
-		setRoot(file);
+		return new Parser(this, file);
-		Parser parser = new Parser(this, file);
+	}
-		file.parser = parser;
+
-		for (Rule r : g.rules.values()) {
+	public RuleFunction rule(Rule r) {
-			RuleFunction function = new RuleFunction(this, r);
+		return new RuleFunction(this, r);
-			parser.funcs.add(function);
-
-			// TRIGGER factory functions for rule alts, elements
-			currentRule.push(function);
-			GrammarASTAdaptor adaptor = new GrammarASTAdaptor(r.ast.token.getInputStream());
-			GrammarAST blk = (GrammarAST)r.ast.getFirstChildWithType(ANTLRParser.BLOCK);
-			CommonTreeNodeStream nodes = new CommonTreeNodeStream(adaptor,blk);
-			SourceGenTriggers genTriggers = new SourceGenTriggers(nodes, controller);
-			try {
-				function.code = genTriggers.block(null,null); // walk AST of rule alts/elements
-			}
-			catch (Exception e){
-				e.printStackTrace(System.err);
-			}
-
-			function.ctxType = gen.target.getRuleFunctionContextStructName(r);
-			function.ruleCtx.name = function.ctxType;
-
-			if ( function.ruleCtx.isEmpty() ) function.ruleCtx = null;
-			currentRule.pop();
-		}
-
-		return file;
 	}
-	 **/
 
 	public List<SrcOp> epsilon() { return list(new CodeBlock(this)); }
 

tool/src/org/antlr/v4/codegen/model/ast/AddRuleLeaf.java

@@ -0,0 +1,17 @@
+package org.antlr.v4.codegen.model.ast;
+
+import org.antlr.v4.codegen.OutputModelFactory;
+import org.antlr.v4.codegen.model.*;
+import org.antlr.v4.codegen.model.decl.Decl;
+import org.antlr.v4.tool.GrammarAST;
+
+public class AddRuleLeaf extends SrcOp {
+	public LabeledOp opWithResultToAdd;
+	public Decl label;
+
+	public AddRuleLeaf(OutputModelFactory factory, GrammarAST ast, LabeledOp opWithResultToAdd) {
+		super(factory, ast);
+		this.opWithResultToAdd = opWithResultToAdd;
+		label = opWithResultToAdd.getLabels().get(0);
+	}
+}

tool/src/org/antlr/v4/codegen/model/ast/AddTokenLeaf.java

@@ -6,11 +6,11 @@ import org.antlr.v4.codegen.model.decl.Decl;
 import org.antlr.v4.tool.GrammarAST;
 
 /** */
-public class AddLeaf extends SrcOp {
+public class AddTokenLeaf extends SrcOp {
 	public LabeledOp opWithResultToAdd;
 	public Decl label;
 
-	public AddLeaf(OutputModelFactory factory, GrammarAST ast, LabeledOp opWithResultToAdd) {
+	public AddTokenLeaf(OutputModelFactory factory, GrammarAST ast, LabeledOp opWithResultToAdd) {
 		super(factory, ast);
 		this.opWithResultToAdd = opWithResultToAdd;
 		label = opWithResultToAdd.getLabels().get(0);