added @ModelElement annotation. model walker now descends into these in order to fill template args.

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8665]

tool/playground/T.java

@@ -1,2 +0,0 @@
-/*
- */

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -132,13 +132,6 @@ RuleFunction(currentRule,code,decls,context,scope,namedActions,finallyAction) ::
 }
 >>
 
-/** Convenience method to call from outside */
-StartRuleFunction(f) ::= <<
-<if(f.modifiers)><f.modifiers:{f | <f> }><else>public final <endif><f.ctxType> <f.name>(<f.args; separator=", ">) throws RecognitionException {
-    return <f.name>(new <f.ctxType>(<f.args:{a | <a.name>, }>LABitSet.EOF_SET));
-}
->>
-
 CodeBlock(c, ops) ::= <<
 <ops; separator="\n">
 >>
@@ -275,11 +268,6 @@ Sync(s) ::= "sync(<s.expecting.name>);"
 
 ThrowNoViableAlt(t) ::= "throw new NoViableAltException(this,_ctx);"
 
-TestSet(s) ::= <<
-_ctx.s = <s.stateNumber>;
-<s.set.name>.member(state.input.LA(1))
->>
-
 TestSetInline(s) ::= <<
 <s.ttypes:{ttype | <s.varName>==<ttype>}; separator=" || ">
 >>
@@ -315,7 +303,7 @@ QRetValueRef(a) ::= "<a.dict>.<a.name>"
 /** How to translate $tokenLabel */
 TokenRef(t) ::= "<t.name>"
 SetAttr(s,rhsChunks) ::= "_ctx.<s.name> = <rhsChunks>;"
-SetQAttr(s,rhsChunks) ::= "<s.dict>.<s.name> = <rhsChunks>;"
+//SetQAttr(s,rhsChunks) ::= "<s.dict>.<s.name> = <rhsChunks>;"
 
 TokenPropertyRef_text(t) ::= "(<t.label>!=null?<t.label>.getText():null)"
 TokenPropertyRef_type(t) ::= "(<t.label>!=null?<t.label>.getType():0)"

tool/resources/org/antlr/v4/tool/templates/messages/languages/en.stg

@@ -36,7 +36,7 @@ INVALID() ::= <<this can never be printed>>
 
 // TOOL ERRORS
 // file errors
-CANNOT_WRITE_FILE(arg,arg2) ::= <<
+CANNOT_WRITE_FILE(arg,arg2,exception,stackTrace) ::= <<
 cannot write file <arg>: <exception>
 <stackTrace; separator="\n">
 >>
@@ -84,7 +84,7 @@ CODE_GEN_TEMPLATES_INCOMPLETE(arg) ::=
 CANNOT_CREATE_TARGET_GENERATOR(arg,exception,stackTrace) ::=
 	"cannot create target <arg> code generator: <exception>"
 CODE_TEMPLATE_ARG_ISSUE(arg,arg2) ::=
-    "code generation template <arg> has missing, misnamed, or incomplete arg list: <arg2>"
+    "code generation template <arg> has missing, misnamed, or incomplete arg list; missing <arg2>"
 NO_MODEL_TO_TEMPLATE_MAPPING(arg) ::=
     "no mapping to template name for output model class <arg>"
 

tool/src/org/antlr/v4/codegen/CodeGenerator.java

@@ -125,9 +125,12 @@ public class CodeGenerator {
 
 	public void write(ST outputFileST) {
 		// WRITE FILES
+		String fileName = "unknown";
 		try {
+			fileName = getRecognizerFileName();
 			target.genRecognizerFile(this,g,outputFileST);
 			if ( templates.isDefined("headerFile") ) {
+				fileName = getHeaderFileName();
 				ST extST = templates.getInstanceOf("headerFileExtension");
 				ST headerFileST = null;
 				target.genRecognizerHeaderFile(this,g,headerFileST,extST.render(lineWidth));
@@ -135,15 +138,15 @@ public class CodeGenerator {
 			// write out the vocab interchange file; used by antlr,
 			// does not change per target
 			ST tokenVocabSerialization = getTokenVocabOutput();
-			String vocabFileName = getVocabFileName();
+			fileName = getVocabFileName();
-			if ( vocabFileName!=null ) {
+			if ( fileName!=null ) {
-				write(tokenVocabSerialization, vocabFileName);
+				write(tokenVocabSerialization, fileName);
 			}
 		}
 		catch (IOException ioe) {
 			g.tool.errMgr.toolError(ErrorType.CANNOT_WRITE_FILE,
-						 getVocabFileName(),
+									fileName,
-						 ioe);
+									ioe);
 		}
 	}
 
@@ -174,4 +177,9 @@ public class CodeGenerator {
 		return g.name+VOCAB_FILE_EXTENSION;
 	}
 
+	public String getHeaderFileName() {
+		ST extST = templates.getInstanceOf("headerFileExtension");
+		String recognizerName = g.getRecognizerName();
+		return recognizerName+extST.render();
+	}
 }

tool/src/org/antlr/v4/codegen/OutputModelWalker.java

@@ -6,6 +6,7 @@ import org.antlr.v4.tool.ErrorType;
 import org.stringtemplate.v4.*;
 import org.stringtemplate.v4.compiler.FormalArgument;
 
+import java.lang.annotation.Annotation;
 import java.lang.reflect.Field;
 import java.util.*;
 
@@ -39,9 +40,10 @@ public class OutputModelWalker {
 
 	public ST walk(OutputModelObject omo) {
 		// CREATE TEMPLATE FOR THIS OUTPUT OBJECT
-		String templateName = omo.getClass().getSimpleName();
+		Class<? extends OutputModelObject> cl = omo.getClass();
+		String templateName = cl.getSimpleName();
 		if ( templateName == null ) {
-			tool.errMgr.toolError(ErrorType.NO_MODEL_TO_TEMPLATE_MAPPING, omo.getClass().getSimpleName());
+			tool.errMgr.toolError(ErrorType.NO_MODEL_TO_TEMPLATE_MAPPING, cl.getSimpleName());
 			return new ST("["+templateName+" invalid]");
 		}
 		ST st = templates.getInstanceOf(templateName);
@@ -55,21 +57,22 @@ public class OutputModelWalker {
 		}
 
 		Map<String,FormalArgument> formalArgs = st.impl.formalArguments;
+
+		// PASS IN OUTPUT MODEL OBJECT TO TEMPLATE AS FIRST ARG
 		Set<String> argNames = formalArgs.keySet();
 		Iterator<String> arg_it = argNames.iterator();
-
-		// PASS IN OUTPUT MODEL OBJECT TO TEMPLATE
 		String modelArgName = arg_it.next(); // ordered so this is first arg
 		st.add(modelArgName, omo);
 
-		// COMPUTE STs FOR EACH NESTED MODEL OBJECT NAMED AS ARG BY TEMPLATE
+		// COMPUTE STs FOR EACH NESTED MODEL OBJECT MARKED WITH @ModelElement AND MAKE ST ATTRIBUTE
-		while ( arg_it.hasNext() ) {
+		Field fields[] = cl.getFields();
-			String fieldName = arg_it.next();
+		for (Field fi : fields) {
-			if ( fieldName.equals("actions") ) {
+			Annotation[] annotations = fi.getAnnotations();
-				System.out.println("computing ST for field "+fieldName+" of "+omo.getClass());
+			if ( annotations.length==0 ) continue;
-			}
+			String fieldName = fi.getName();
+			// Just don't set @ModelElement fields w/o formal arg in target ST
+			if ( formalArgs.get(fieldName)==null ) continue;
 			try {
-				Field fi = omo.getClass().getField(fieldName);
 				Object o = fi.get(omo);
 				if ( o instanceof OutputModelObject ) {  // SINGLE MODEL OBJECT?
 					OutputModelObject nestedOmo = (OutputModelObject)o;
@@ -83,9 +86,7 @@ public class OutputModelWalker {
 					}
 					Collection<? extends OutputModelObject> nestedOmos = (Collection)o;
 					for (OutputModelObject nestedOmo : nestedOmos) {
-						if ( nestedOmo==null ) {
+						if ( nestedOmo==null ) continue;
-							System.out.println("collection has nulls: "+nestedOmos);
-						}
 						ST nestedST = walk(nestedOmo);
 						st.add(fieldName, nestedST);
 					}
@@ -100,12 +101,9 @@ public class OutputModelWalker {
 					st.add(fieldName, m);
 				}
 				else if ( o!=null ) {
-					tool.errMgr.toolError(ErrorType.CODE_TEMPLATE_ARG_ISSUE, templateName, fieldName);
+					tool.errMgr.toolError(ErrorType.INTERNAL_ERROR, "not recognized nested model element: "+fieldName);
 				}
 			}
-			catch (NoSuchFieldException nsfe) {
-				tool.errMgr.toolError(ErrorType.CODE_TEMPLATE_ARG_ISSUE, templateName, nsfe.getMessage());
-			}
 			catch (IllegalAccessException iae) {
 				tool.errMgr.toolError(ErrorType.CODE_TEMPLATE_ARG_ISSUE, templateName, fieldName);
 			}

tool/src/org/antlr/v4/codegen/model/Action.java

@@ -8,7 +8,8 @@ import java.util.List;
 
 /** */
 public class Action extends RuleElement {
-	public List<ActionChunk> chunks;
+	@ModelElement public List<ActionChunk> chunks;
+
 	public Action(OutputModelFactory factory, GrammarAST ast) {
 		super(factory,ast);
 		RuleFunction rf = null;

tool/src/org/antlr/v4/codegen/model/AltBlock.java

@@ -7,7 +7,7 @@ import org.antlr.v4.tool.GrammarAST;
 import java.util.List;
 
 public class AltBlock extends Choice {
-	public ThrowNoViableAlt error;
+	@ModelElement public ThrowNoViableAlt error;
 
 	public AltBlock(OutputModelFactory factory,
 					GrammarAST blkOrEbnfRootAST,

tool/src/org/antlr/v4/codegen/model/Choice.java

@@ -18,8 +18,9 @@ import java.util.*;
  */
 public abstract class Choice extends RuleElement {
 	public int decision = -1;
-	public List<CodeBlock> alts;
+
-	public List<SrcOp> preamble;
+	@ModelElement public List<CodeBlock> alts;
+	@ModelElement public List<SrcOp> preamble;
 
 	public Choice(OutputModelFactory factory,
 				  GrammarAST blkOrEbnfRootAST,

tool/src/org/antlr/v4/codegen/model/CodeBlock.java

@@ -6,7 +6,7 @@ import java.util.*;
 
 /** */
 public class CodeBlock extends SrcOp {
-	public List<SrcOp> ops;
+	@ModelElement public List<SrcOp> ops;
 
 	public CodeBlock(OutputModelFactory factory) { this.factory = factory; }
 

tool/src/org/antlr/v4/codegen/model/LL1Choice.java

@@ -8,7 +8,7 @@ import java.util.List;
 public abstract class LL1Choice extends Choice {
 	/** Token names for each alt 0..n-1 */
 	public List<String[]> altLook;
-	public ThrowNoViableAlt error;
+	@ModelElement public ThrowNoViableAlt error;
 
 	public LL1Choice(OutputModelFactory factory, GrammarAST blkAST,
 					 List<CodeBlock> alts)

tool/src/org/antlr/v4/codegen/model/LL1Loop.java

@@ -8,9 +8,9 @@ import java.util.*;
 
 /** */
 public abstract class LL1Loop extends Choice {
-	public OutputModelObject loopExpr;
+	@ModelElement public OutputModelObject loopExpr;
-	public List<SrcOp> iteration;
+	@ModelElement public List<SrcOp> iteration;
-	public Sync sync;
+	@ModelElement public Sync sync;
 
 	public LL1Loop(OutputModelFactory factory,
 				   GrammarAST blkAST,

tool/src/org/antlr/v4/codegen/model/LL1OptionalBlockSingleAlt.java

@@ -9,8 +9,8 @@ import java.util.List;
 
 /** (A B C)? */
 public class LL1OptionalBlockSingleAlt extends LL1Choice {
-	public OutputModelObject expr;
+	@ModelElement public OutputModelObject expr;
-	public OutputModelObject followExpr;
+	@ModelElement public OutputModelObject followExpr;
 
 	public LL1OptionalBlockSingleAlt(OutputModelFactory factory,
 									 GrammarAST blkAST,

tool/src/org/antlr/v4/codegen/model/LL1PlusBlock.java

@@ -12,22 +12,21 @@ public class LL1PlusBlock extends LL1Loop {
 	/** Token names for each alt 0..n-1 */
 	public List<String[]> altLook;
 
-	public Sync iterationSync;
+	@ModelElement public Sync iterationSync;
 	public String loopLabel;
 	public String loopCounterVar;
 	public String[] exitLook;
-	public SrcOp loopExpr;
+
-	public ThrowNoViableAlt error;
+	@ModelElement public SrcOp loopExpr;
+	@ModelElement public ThrowNoViableAlt error;
 
 	public LL1PlusBlock(OutputModelFactory factory, GrammarAST plusRoot, List<CodeBlock> alts) {
 		super(factory, plusRoot, alts);
 
 		PlusBlockStartState blkStart = (PlusBlockStartState)plusRoot.atnState;
-//		BlockStartState blkStart = (BlockStartState)plus.transition(0).target;
 
 		this.decision = blkStart.decision;
 		/** Lookahead for each alt 1..n */
-//		IntervalSet[] altLookSets = LinearApproximator.getLL1LookaheadSets(dfa);
 		IntervalSet[] altLookSets = factory.g.decisionLOOK.get(decision);
 		altLook = getAltLookaheadAsStringLists(altLookSets);
 		IntervalSet all = new IntervalSet();

tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java

@@ -9,7 +9,7 @@ import java.util.List;
 
 /** */
 public class LL1PlusBlockSingleAlt extends LL1Loop {
-	public Sync iterationSync;
+	@ModelElement public Sync iterationSync;
 
 	public LL1PlusBlockSingleAlt(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlock> alts) {
 		super(factory, blkAST, alts);

tool/src/org/antlr/v4/codegen/model/Lexer.java

@@ -12,9 +12,10 @@ public class Lexer extends OutputModelObject {
 	public String[] tokenNames;
 	public Set<String> ruleNames;
 	public Collection<String> modes;
-	public LinkedHashMap<Integer, Action> actions;
+
-	public LinkedHashMap<Integer, Action> sempreds;
+	@ModelElement public SerializedATN atn;
-	public SerializedATN atn;
+	@ModelElement public LinkedHashMap<Integer, Action> actions;
+	@ModelElement public LinkedHashMap<Integer, Action> sempreds;
 
 	public Lexer(OutputModelFactory factory, LexerFile file) {
 		this.factory = factory;

tool/src/org/antlr/v4/codegen/model/LexerFile.java

@@ -7,8 +7,9 @@ import java.util.*;
 
 public class LexerFile extends OutputModelObject {
 	public String fileName;
-	public Lexer lexer;
+
-	public Map<String, Action> namedActions;
+	@ModelElement public Lexer lexer;
+	@ModelElement public Map<String, Action> namedActions;
 
 	public LexerFile(OutputModelFactory factory, String fileName) {
 		super(factory);

tool/src/org/antlr/v4/codegen/model/ModelElement.java

@@ -0,0 +1,10 @@
+package org.antlr.v4.codegen.model;
+
+import java.lang.annotation.*;
+
+/** Indicate field of OutputModelObject is an element to be walked by
+ *  OutputModelWalker.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ModelElement {
+}

tool/src/org/antlr/v4/codegen/model/Parser.java

@@ -11,12 +11,13 @@ public class Parser extends OutputModelObject {
 	public Map<String,Integer> tokens;
 	public String[] tokenNames;
 	public Set<String> ruleNames;
-	public List<RuleFunction> funcs = new ArrayList<RuleFunction>();
-	public List<DynamicScopeStruct> scopes = new ArrayList<DynamicScopeStruct>();
 	public ParserFile file;
-	public SerializedATN atn;
+
-	public LinkedHashMap<Integer, ForcedAction> actions;
+	@ModelElement public List<DynamicScopeStruct> scopes = new ArrayList<DynamicScopeStruct>();
-	public LinkedHashMap<Integer, Action> sempreds;
+	@ModelElement public List<RuleFunction> funcs = new ArrayList<RuleFunction>();
+	@ModelElement public SerializedATN atn;
+	@ModelElement public LinkedHashMap<Integer, ForcedAction> actions;
+	@ModelElement public LinkedHashMap<Integer, Action> sempreds;
 
 	public Parser(OutputModelFactory factory, ParserFile file) {
 		this.factory = factory;

tool/src/org/antlr/v4/codegen/model/ParserFile.java

@@ -8,12 +8,11 @@ import java.util.*;
 /** */
 public class ParserFile extends OutputModelObject {
 	public String fileName;
-	public Parser parser;
-//	public List<DFADecl> dfaDecls = new ArrayList<DFADecl>();
-//	public OrderedHashSet<BitSetDecl> bitSetDecls = new OrderedHashSet<BitSetDecl>();
 	public String TokenLabelType;
 	public String ASTLabelType;
-	public Map<String, Action> namedActions;
+
+	@ModelElement public Parser parser;
+	@ModelElement public Map<String, Action> namedActions;
 
 	public ParserFile(OutputModelFactory factory, String fileName) {
 		super(factory);

tool/src/org/antlr/v4/codegen/model/PlusBlock.java

@@ -7,7 +7,8 @@ import org.antlr.v4.tool.GrammarAST;
 import java.util.List;
 
 public class PlusBlock extends Loop {
-	public ThrowNoViableAlt error;
+	@ModelElement public ThrowNoViableAlt error;
+
 	public PlusBlock(OutputModelFactory factory,
 					 GrammarAST ebnfRootAST,
 					 List<CodeBlock> alts)

tool/src/org/antlr/v4/codegen/model/RuleFunction.java

@@ -19,19 +19,16 @@ public class RuleFunction extends OutputModelObject {
 	public Collection<String> tokenLabels;
 	public List<String> elementsReferencedInRewrite;
 	public List<String> exceptions;
-	public Action finallyAction;
-	public Map<String, Action> namedActions;
-
 	public ATNState startState;
-
-	public StructDecl context;
-	public DynamicScopeStruct scope;
-
 	public int index;
-
 	public Collection<Attribute> args = null;
-	public OrderedHashSet<Decl> decls;
+
-	public SrcOp code;
+	@ModelElement public SrcOp code;
+	@ModelElement public OrderedHashSet<Decl> decls;
+	@ModelElement public StructDecl context;
+	@ModelElement public DynamicScopeStruct scope;
+	@ModelElement public Map<String, Action> namedActions;
+	@ModelElement public Action finallyAction;
 
 	public RuleFunction(OutputModelFactory factory) {
 		super(factory);

tool/src/org/antlr/v4/codegen/model/StartRuleFunction.java

@@ -1,26 +0,0 @@
-package org.antlr.v4.codegen.model;
-
-import org.antlr.v4.codegen.OutputModelFactory;
-import org.antlr.v4.misc.Utils;
-import org.antlr.v4.tool.*;
-
-import java.util.ArrayList;
-
-/** */
-public class StartRuleFunction extends RuleFunction {
-	public StartRuleFunction(OutputModelFactory factory, Rule r) {
-		super(factory);
-		this.name = r.name;
-		if ( r.modifiers!=null && r.modifiers.size()>0 ) {
-			this.modifiers = new ArrayList<String>();
-			for (GrammarAST t : r.modifiers) modifiers.add(t.getText());
-		}
-		modifiers = Utils.nodesToStrings(r.modifiers);
-
-		ctxType = factory.gen.target.getRuleFunctionContextStructName(r);
-
-		if ( r.args!=null ) {
-			args = r.args.attributes.values();
-		}
-	}
-}

tool/src/org/antlr/v4/codegen/model/StructDecl.java

@@ -7,8 +7,8 @@ import java.util.*;
 
 /** */
 public class StructDecl extends Decl {
-	public List<Decl> attrs = new ArrayList<Decl>();
+	@ModelElement public List<Decl> attrs = new ArrayList<Decl>();
-	public Collection<Attribute> ctorAttrs;
+	@ModelElement public Collection<Attribute> ctorAttrs;
 
 	public StructDecl(OutputModelFactory factory, String name, Collection<Attribute> attrList) {
 		super(factory, name);

tool/src/org/antlr/v4/codegen/model/actions/DynScopeAttrRef_index.java

@@ -1,10 +1,13 @@
 package org.antlr.v4.codegen.model.actions;
 
+import org.antlr.v4.codegen.model.ModelElement;
+
 import java.util.List;
 
 /** */
 public class DynScopeAttrRef_index extends DynScopeAttrRef {
-	public List<ActionChunk> indexChunks;
+	@ModelElement public List<ActionChunk> indexChunks;
+
 	public DynScopeAttrRef_index(String scope, String attr, List<ActionChunk> indexChunks) {
 		super(scope, attr);
 		this.indexChunks = indexChunks;

tool/src/org/antlr/v4/codegen/model/actions/SetAttr.java

@@ -1,11 +1,13 @@
 package org.antlr.v4.codegen.model.actions;
 
+import org.antlr.v4.codegen.model.ModelElement;
+
 import java.util.List;
 
 /** */
 public class SetAttr extends ActionChunk {
 	public String name;
-	public List<ActionChunk> rhsChunks;
+	@ModelElement public List<ActionChunk> rhsChunks;
 
 	public SetAttr(String name, List<ActionChunk> rhsChunks) {
 		this.name = name;

tool/src/org/antlr/v4/codegen/model/actions/SetDynScopeAttr.java

@@ -1,12 +1,14 @@
 package org.antlr.v4.codegen.model.actions;
 
+import org.antlr.v4.codegen.model.ModelElement;
+
 import java.util.List;
 
 /** */
 public class SetDynScopeAttr extends ActionChunk {
 	public String scope;
 	public String attr;
-	public List<ActionChunk> rhsChunks;
+	@ModelElement public List<ActionChunk> rhsChunks;
 
 	public SetDynScopeAttr(String scope, String attr, List<ActionChunk> rhsChunks) {
 		this.scope = scope;

tool/src/org/antlr/v4/codegen/model/actions/SetDynScopeAttr_index.java

@@ -1,10 +1,13 @@
 package org.antlr.v4.codegen.model.actions;
 
+import org.antlr.v4.codegen.model.ModelElement;
+
 import java.util.List;
 
 /** */
 public class SetDynScopeAttr_index extends SetDynScopeAttr {
-	public List<ActionChunk> indexChunks;
+	@ModelElement public List<ActionChunk> indexChunks;
+
 	public SetDynScopeAttr_index(String scope, String attr, List<ActionChunk> indexChunks, List<ActionChunk> rhsChunks) {
 		super(scope, attr, rhsChunks);
 		this.indexChunks = indexChunks;