gen listener file; reorg file generation

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9053]

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -35,6 +35,15 @@ import java.util.ArrayList;
 <parser>
 >>
 
+ListenerFile(listener) ::= <<
+@SuppressWarnings({"all", "warnings", "unchecked", "unused"})
+public interface <listener.grammarName> {
+    <listener.rules:{r |
+void enterRule(<r.name>_ctx ctx);
+void exitRule(<r.name>_ctx ctx);}; separator="\n">
+}
+>>
+
 Parser(parser, scopes, funcs, atn, sempredFuncs) ::= <<
 @SuppressWarnings({"all", "warnings", "unchecked", "unused"})
 public class <parser.name> extends Parser {
@@ -415,10 +424,8 @@ public static class <s.name> extends ParserRuleContext {
     	super(parent, state);
         <s.ctorAttrs:{a | this.<a.name> = <a.name>;}; separator="\n">
     }
-    /*
 	public void enterRule(TListener listener) { listener.enterRule(this); }
 	public void exitRule(TListener listener) { listener.exitRule(this); }
-	*/
 }
 >>
 

tool/src/org/antlr/v4/codegen/CodeGenPipeline.java

@@ -30,17 +30,25 @@
 package org.antlr.v4.codegen;
 
 import org.antlr.v4.tool.Grammar;
-import org.stringtemplate.v4.ST;
 
 public class CodeGenPipeline {
 	Grammar g;
+
 	public CodeGenPipeline(Grammar g) {
 		this.g = g;
 	}
+
 	public void process() {
 		CodeGenerator gen = new CodeGenerator(g);
 
-		ST outputFileST = gen.generate();
-		gen.write(outputFileST);
+		if ( g.isLexer() ) {
+			gen.writeRecognizer(gen.generateLexer());
+		}
+		else {
+			gen.writeRecognizer(gen.generateParser());
+			gen.writeListener(gen.generateListener());
+			gen.writeHeaderFile();
+		}
+		gen.writeVocabFile();
 	}
 }

tool/src/org/antlr/v4/codegen/CodeGenerator.java

@@ -32,12 +32,16 @@ package org.antlr.v4.codegen;
 import org.antlr.v4.Tool;
 import org.antlr.v4.codegen.model.OutputModelObject;
 import org.antlr.v4.runtime.Token;
-import org.antlr.v4.tool.*;
+import org.antlr.v4.tool.ErrorType;
+import org.antlr.v4.tool.Grammar;
 import org.stringtemplate.v4.*;
 
-import java.io.*;
-import java.lang.reflect.*;
-import java.util.*;
+import java.io.IOException;
+import java.io.Writer;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashMap;
+import java.util.Map;
 
 /** General controller for code gen.  Can instantiate sub generator(s).
  */
@@ -48,6 +52,8 @@ public class CodeGenerator {
 		"<tokens.keys:{t | <t>=<tokens.(t)>\n}>" +
 		"<literals.keys:{t | <t>=<literals.(t)>\n}>";
 
+	public OutputModelObject outputModel;
+
 	public Grammar g;
 	public Tool tool;
 	public Target target;
@@ -108,10 +114,47 @@ public class CodeGenerator {
 		}
 	}
 
-	public ST generate() {
-		OutputModelFactory factory;
-		if ( g.isLexer() ) factory = new LexerFactory(this);
-		else factory = new ParserFactory(this);
+//	public void buildOutputModel() {
+//		OutputModelFactory factory;
+//		if ( g.isLexer() ) factory = new LexerFactory(this);
+//		else factory = new ParserFactory(this);
+//
+//		// TODO: let someone add their own factory?
+//
+//		// CREATE OUTPUT MODEL FROM GRAMMAR OBJ AND AST WITHIN RULES
+//		OutputModelController controller = new OutputModelController(factory);
+//		if ( g.hasASTOption() ) {
+//			controller.addExtension( new ParserASTExtension(factory) );
+//		}
+//		factory.setController(controller);
+//
+//		if ( g.isLexer() ) outputModel = controller.buildLexerOutputModel();
+//		else outputModel = controller.buildParserOutputModel();
+//	}
+
+	// CREATE TEMPLATES BY WALKING MODEL
+	public ST generateLexer() {
+		OutputModelFactory factory = new LexerFactory(this);
+
+		// CREATE OUTPUT MODEL FROM GRAMMAR OBJ AND AST WITHIN RULES
+		OutputModelController controller = new OutputModelController(factory);
+		factory.setController(controller);
+
+		outputModel = controller.buildLexerOutputModel();
+
+		OutputModelWalker walker = new OutputModelWalker(tool, templates);
+		ST st = walker.walk(outputModel);
+
+		if ( tool.launch_ST_inspector ) {
+			st.inspect();
+			//if ( templates.isDefined("headerFile") ) headerFileST.inspect();
+		}
+
+		return st;
+	}
+
+	public ST generateParser() {
+		OutputModelFactory factory = new ParserFactory(this);
 
 		// TODO: let someone add their own factory?
 
@@ -122,11 +165,33 @@ public class CodeGenerator {
 		}
 		factory.setController(controller);
 
-		OutputModelObject outputModel;
-		if ( g.isLexer() ) outputModel = controller.buildLexerOutputModel();
-		else outputModel = controller.buildParserOutputModel();
+		outputModel = controller.buildParserOutputModel();
+
+		OutputModelWalker walker = new OutputModelWalker(tool, templates);
+		ST st = walker.walk(outputModel);
+
+		if ( tool.launch_ST_inspector ) {
+			st.inspect();
+			//if ( templates.isDefined("headerFile") ) headerFileST.inspect();
+		}
+
+		return st;
+	}
+
+	public ST generateListener() {
+		OutputModelFactory factory = new ParserFactory(this);
+
+		// TODO: let someone add their own factory?
+
+		// CREATE OUTPUT MODEL FROM GRAMMAR OBJ AND AST WITHIN RULES
+		OutputModelController controller = new OutputModelController(factory);
+		if ( g.hasASTOption() ) {
+			controller.addExtension( new ParserASTExtension(factory) );
+		}
+		factory.setController(controller);
+
+		outputModel = controller.buildListenerOutputModel();
 
-		// CREATE TEMPLATES BY WALKING MODEL
 		OutputModelWalker walker = new OutputModelWalker(tool, templates);
 		ST st = walker.walk(outputModel);
 
@@ -170,42 +235,73 @@ public class CodeGenerator {
 		return vocabFileST;
 	}
 
-	public void write(ST outputFileST) {
-		// WRITE FILES
-		String fileName = "unknown";
-		try {
-			fileName = getRecognizerFileName();
-			target.genRecognizerFile(g,outputFileST);
-			if ( templates.isDefined("headerFile") ) {
-				fileName = getHeaderFileName();
-				ST extST = templates.getInstanceOf("headerFileExtension");
-				ST headerFileST = null;
-				target.genRecognizerHeaderFile(g,headerFileST,extST.render(lineWidth));
-			}
-			// write out the vocab interchange file; used by antlr,
-			// does not change per target
-			ST tokenVocabSerialization = getTokenVocabOutput();
-			fileName = getVocabFileName();
-			if ( fileName!=null ) {
-				write(tokenVocabSerialization, fileName);
-			}
-		}
-		catch (IOException ioe) {
-			tool.errMgr.toolError(ErrorType.CANNOT_WRITE_FILE,
-									ioe,
-									fileName);
+	public void writeRecognizer(ST outputFileST) {
+		target.genFile(g, outputFileST, getRecognizerFileName());
+	}
+
+	public void writeListener(ST outputFileST) {
+		target.genFile(g,outputFileST,getListenerFileName());
+	}
+
+	public void writeHeaderFile() {
+		String fileName = getHeaderFileName();
+		if ( fileName==null ) return;
+		if ( templates.isDefined("headerFile") ) {
+			ST extST = templates.getInstanceOf("headerFileExtension");
+			ST headerFileST = null;
+			// TODO:  don't hide this header file generation here!
+			target.genRecognizerHeaderFile(g,headerFileST,extST.render(lineWidth));
 		}
 	}
 
-	public void write(ST code, String fileName) throws IOException {
-		long start = System.currentTimeMillis();
-		Writer w = tool.getOutputFileWriter(g, fileName);
-		STWriter wr = new AutoIndentWriter(w);
-		wr.setLineWidth(lineWidth);
-		code.write(wr);
-		w.close();
-		long stop = System.currentTimeMillis();
-		System.out.println("render time for "+fileName+": "+(int)(stop-start)+"ms");
+	public void writeVocabFile() {
+		// write out the vocab interchange file; used by antlr,
+		// does not change per target
+		ST tokenVocabSerialization = getTokenVocabOutput();
+		String fileName = getVocabFileName();
+		if ( fileName!=null ) {
+			write(tokenVocabSerialization, fileName);
+		}
+	}
+
+//	public void write(ST outputFileST) {
+//		// WRITE FILES
+//		String fileName = "unknown";
+//		try {
+//			fileName = getRecognizerFileName();
+//			target.genRecognizerFile(g,outputFileST);
+//			writeHeaderFile();
+//			// write out the vocab interchange file; used by antlr,
+//			// does not change per target
+//			ST tokenVocabSerialization = getTokenVocabOutput();
+//			fileName = getVocabFileName();
+//			if ( fileName!=null ) {
+//				write(tokenVocabSerialization, fileName);
+//			}
+//		}
+//		catch (IOException ioe) {
+//			tool.errMgr.toolError(ErrorType.CANNOT_WRITE_FILE,
+//									ioe,
+//									fileName);
+//		}
+//	}
+
+	public void write(ST code, String fileName) {
+		try {
+			long start = System.currentTimeMillis();
+			Writer w = tool.getOutputFileWriter(g, fileName);
+			STWriter wr = new AutoIndentWriter(w);
+			wr.setLineWidth(lineWidth);
+			code.write(wr);
+			w.close();
+			long stop = System.currentTimeMillis();
+			System.out.println("render time for "+fileName+": "+(int)(stop-start)+"ms");
+		}
+		catch (IOException ioe) {
+			tool.errMgr.toolError(ErrorType.CANNOT_WRITE_FILE,
+								  ioe,
+								  fileName);
+		}
 	}
 
 	/** Generate TParser.java and TLexer.java from T.g if combined, else
@@ -217,6 +313,15 @@ public class CodeGenerator {
 		return recognizerName+extST.render();
 	}
 
+	/** A given grammar T, return the listener name such as
+	 *  TListener.java, if we're using the Java target.
+ 	 */
+	public String getListenerFileName() {
+		ST extST = templates.getInstanceOf("codeFileExtension");
+		String listenerName = g.name + "Listener";
+		return listenerName+extST.render();
+	}
+
 	/** What is the name of the vocab file generated for this grammar?
 	 *  Returns null if no .tokens file should be generated.
 	 */
@@ -226,6 +331,7 @@ public class CodeGenerator {
 
 	public String getHeaderFileName() {
 		ST extST = templates.getInstanceOf("headerFileExtension");
+		if ( extST==null ) return null;
 		String recognizerName = g.getRecognizerName();
 		return recognizerName+extST.render();
 	}

tool/src/org/antlr/v4/codegen/OutputModelController.java

@@ -33,10 +33,13 @@ import org.antlr.runtime.tree.CommonTreeNodeStream;
 import org.antlr.v4.codegen.model.*;
 import org.antlr.v4.codegen.model.ast.*;
 import org.antlr.v4.codegen.model.decl.CodeBlock;
-import org.antlr.v4.parse.*;
+import org.antlr.v4.parse.ANTLRParser;
+import org.antlr.v4.parse.GrammarASTAdaptor;
 import org.antlr.v4.tool.*;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Stack;
 
 /** This receives events from SourceGenTriggers.g and asks factory to do work.
  *  Then runs extensions in order on resulting SrcOps to get final list.
@@ -102,6 +105,11 @@ public class OutputModelController {
 		return file;
 	}
 
+	public OutputModelObject buildListenerOutputModel() {
+		CodeGenerator gen = delegate.getGenerator();
+		return new ListenerFile(delegate, gen.getListenerFileName());
+	}
+
 	public ParserFile parserFile(String fileName) {
 		ParserFile f = delegate.parserFile(fileName);
 		for (CodeGeneratorExtension ext : extensions) f = ext.parserFile(f);
@@ -115,7 +123,7 @@ public class OutputModelController {
 	}
 
 	public LexerFile lexerFile(String fileName) {
-		return new LexerFile(delegate, getGenerator().getRecognizerFileName());
+		return new LexerFile(delegate, fileName);
 	}
 
 	public Lexer lexer(LexerFile file) {

tool/src/org/antlr/v4/codegen/Target.java

@@ -31,12 +31,13 @@ package org.antlr.v4.codegen;
 
 import org.antlr.v4.codegen.model.RuleFunction;
 import org.antlr.v4.parse.ANTLRParser;
-import org.antlr.v4.runtime.*;
-import org.antlr.v4.tool.*;
+import org.antlr.v4.runtime.Lexer;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.tool.Grammar;
+import org.antlr.v4.tool.GrammarAST;
+import org.antlr.v4.tool.Rule;
 import org.stringtemplate.v4.ST;
 
-import java.io.IOException;
-
 /** */
 public class Target {
 	/** For pure strings of Java 16-bit unicode char, how can we display
@@ -66,18 +67,23 @@ public class Target {
 		this.gen = gen;
 	}
 
-	protected void genRecognizerFile(Grammar g,
-									 ST outputFileST)
-		throws IOException
+	protected void genFile(Grammar g,
+						   ST outputFileST,
+						   String fileName)
 	{
-		String fileName = gen.getRecognizerFileName();
+		gen.write(outputFileST, fileName);
+	}
+
+	protected void genListenerFile(Grammar g,
+								   ST outputFileST)
+	{
+		String fileName = gen.getListenerFileName();
 		gen.write(outputFileST, fileName);
 	}
 
 	protected void genRecognizerHeaderFile(Grammar g,
 										   ST headerFileST,
 										   String extName) // e.g., ".h"
-		throws IOException
 	{
 		// no header file by default
 	}

tool/src/org/antlr/v4/codegen/model/ListenerFile.java

@@ -0,0 +1,24 @@
+package org.antlr.v4.codegen.model;
+
+import org.antlr.v4.codegen.OutputModelFactory;
+import org.antlr.v4.tool.Grammar;
+import org.antlr.v4.tool.Rule;
+
+import java.util.Collection;
+
+/** A model object representing a parse tree listener file.
+ *  These are the rules specific events triggered by a parse tree visitor.
+ */
+public class ListenerFile extends OutputModelObject {
+	public String fileName;
+	public String grammarName;
+	public Collection<Rule> rules;
+
+	public ListenerFile(OutputModelFactory factory, String fileName) {
+		super(factory);
+		this.fileName = fileName;
+		Grammar g = factory.getGrammar();
+		grammarName = g.name;
+		rules = g.rules.values();
+	}
+}

tool/test/org/antlr/v4/test/TestActionTranslation.java

@@ -1,12 +1,9 @@
 package org.antlr.v4.test;
 
 import org.antlr.runtime.RecognitionException;
-import org.antlr.v4.automata.*;
 import org.antlr.v4.codegen.CodeGenerator;
 import org.antlr.v4.semantics.SemanticPipeline;
-import org.antlr.v4.tool.*;
 import org.junit.Test;
-import org.stringtemplate.v4.*;
 
 /** */
 public class TestActionTranslation extends BaseTest {
@@ -423,7 +420,7 @@ public class TestActionTranslation extends BaseTest {
 				g.atn = factory.createATN();
 
 				CodeGenerator gen = new CodeGenerator(g);
-				ST outputFileST = gen.generate();
+				ST outputFileST = gen.generateRecognizer();
 				String output = outputFileST.render();
 				//System.out.println(output);
 				String b = "#" + actionName + "#";

tool/test/org/antlr/v4/test/TestLexerAttributes.java

@@ -17,7 +17,7 @@ public class TestLexerAttributes extends BaseTest {
 		Tool antlr = new Tool();
 		antlr.process(g,false);
 		CodeGenerator gen = new CodeGenerator(g);
-		ST outputFileST = gen.generate();
+		ST outputFileST = gen.generateRecognizer();
 		String output = outputFileST.render();
 		int start = output.indexOf('#');
 		int end = output.lastIndexOf('#');