got simple LL1 (..) and (..)? in; added Decl concept

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 6851]

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -32,16 +32,54 @@ CodeBlock(c, ops) ::= <<
 <ops; separator="\n">
 >>
 
-LL1Choice(c, alts) ::= <<
+LL1Choice(choice, alts) ::= <<
 switch ( input.LA(1) ) {
-	<alts; separator="\n">
+	<choice.altLook,alts:{look,alt| <look:cases()><alt>}; separator="\n">
+	default :
+	    error
 }
 >>
 
-MatchToken(m) ::= <<
+LL1OptionalBlock(choice, expr, alts) ::= <<
-match(<m.name>, <m.follow.name>);
+switch ( <expr> ) {
+	<choice.altLook,alts:{look,alt| <look:cases()><alt>}; separator="\n">
+}
 >>
 
+LL1OptionalBlockSingleAlt(choice, expr, alts, decls) ::= <<
+<decls>
+if ( <expr> ) {
+    <alts; separator="\n">
+}
+else {
+    NoViableAltException nvae = new NoViableAltException("", 4, 0, input);
+}
+>>
+
+TestSet(s) ::= <<
+<s.set.name>.member(input.LA(1))
+>>
+
+TestSetInline(s) ::= <<
+<s.ttypes:{ttype | <first(choice.decls).varName>==<ttype>}; separator=" || ">
+>>
+
+cases(look) ::= <<
+<look:{case <it>:<\n>}>
+>>
+
+InvokeRule(r) ::= <<
+pushFollow(<r.follow.name>);
+<if(r.label)><r.label> = <endif><r.name>();
+state._fsp--;
+>>
+
+MatchToken(m) ::= <<
+<if(m.label)><m.label> = <endif>match(<m.name>, <m.follow.name>);
+>>
+
+NextTokenDecl(d) ::= "Token <d.varName> = input.LA(1);"
+
 codeFileExtension() ::= ".java"
 
 true() ::= "true"

tool/src/org/antlr/v4/analysis/LinearApproximator.java

@@ -154,6 +154,16 @@ public class LinearApproximator {
 		return dfa;
 	}
 
+	/** From linear approximate LL(1) DFA, get lookahead per alt; 1..n */
+	public static IntervalSet[] getLL1LookaheadSets(DFA dfa) {
+		IntervalSet[] look = new IntervalSet[dfa.nAlts+1];
+		DFAState s0 = dfa.startState;
+		for (int a=1; a<=dfa.nAlts; a++) {
+			look[a] = s0.edges.get(a-1).label;
+		}
+		return look;
+	}
+
 	/** From an NFA state, s, find the set of all labels reachable from s at
 	 *  depth k.
 	 */

tool/src/org/antlr/v4/automata/ParserNFAFactory.java

@@ -241,7 +241,7 @@ public class ParserNFAFactory implements NFAFactory {
 		Handle first = els.get(0);
 		Handle last = els.get(els.size()-1);
 		if ( first==null || last==null ) {
-			System.out.println("huh?");
+			g.tool.errMgr.toolError(ErrorType.INTERNAL_ERROR, "alt Handle has first|last == null");
 		}
 		return new Handle(first.left, last.right);
 	}

tool/src/org/antlr/v4/codegen/CodeGenerator.java

@@ -1,10 +1,10 @@
 package org.antlr.v4.codegen;
 
-import org.antlr.v4.codegen.src.BitSetDef;
+import org.antlr.v4.automata.DFA;
-import org.antlr.v4.codegen.src.OutputModelObject;
+import org.antlr.v4.codegen.src.*;
-import org.antlr.v4.codegen.src.ParserFile;
 import org.antlr.v4.misc.IntSet;
 import org.antlr.v4.parse.ANTLRParser;
+import org.antlr.v4.tool.BlockAST;
 import org.antlr.v4.tool.ErrorType;
 import org.antlr.v4.tool.Grammar;
 import org.antlr.v4.tool.GrammarAST;
@@ -12,6 +12,7 @@ import org.stringtemplate.v4.*;
 
 import java.io.IOException;
 import java.io.Writer;
+import java.util.List;
 
 /** */
 public abstract class CodeGenerator {
@@ -105,6 +106,27 @@ public abstract class CodeGenerator {
 		}
 	}
 
+	public Choice getChoiceBlock(BlockAST blkAST, GrammarAST ebnfRoot, List<CodeBlock> alts) {
+		// TODO: assumes LL1
+		int ebnf = 0;
+		if ( ebnfRoot!=null ) ebnf = ebnfRoot.getType();
+		Choice c = null;
+		switch ( ebnf ) {
+			case ANTLRParser.OPTIONAL :
+				if ( alts.size()==1 ) c = new LL1OptionalBlockSingleAlt(this, ebnfRoot, alts);
+				else c = new LL1OptionalBlock(this, ebnfRoot, alts);
+				break;
+			case ANTLRParser.CLOSURE :
+				break;
+			case ANTLRParser.POSITIVE_CLOSURE :
+				break;
+			default :
+				c = new LL1Choice(this, blkAST, alts);
+				break;
+		}
+		return c;
+	}
+
 	public void write(ST code, String fileName) throws IOException {
 		long start = System.currentTimeMillis();
 		Writer w = g.tool.getOutputFile(g, fileName);
@@ -136,14 +158,28 @@ public abstract class CodeGenerator {
 		return g.name+VOCAB_FILE_EXTENSION;
 	}
 
-	public BitSetDef defineBitSet(GrammarAST ast, IntSet follow) {
+	public DFADef defineDFA(GrammarAST ast, DFA dfa) {
+		return null;
+//		DFADef d = new DFADef(name, dfa);
+//		outputModel.dfaDefs.add(d);
+	}
+
+	public BitSetDef defineFollowBitSet(GrammarAST ast, IntSet set) {
 		String inRuleName = ast.nfaState.rule.name;
 		String elementName = ast.getText(); // assume rule ref
 		if ( ast.getType() == ANTLRParser.TOKEN_REF ) {
-			target.getTokenTypeAsTargetLabel(g, ast.getType() );
+			elementName = target.getTokenTypeAsTargetLabel(g, g.tokenNameToTypeMap.get(elementName));
 		}
-		String name = "FOLLOW_"+elementName+"_in_"+inRuleName+ast.token.getTokenIndex();
+		String name = "FOLLOW_"+elementName+"_in_"+inRuleName+"_"+ast.token.getTokenIndex();
-		BitSetDef b = new BitSetDef(this, name, follow);
+		BitSetDef b = new BitSetDef(this, name, set);
+		outputModel.bitSetDefs.add(b);
+		return b;
+	}
+
+	public BitSetDef defineTestBitSet(GrammarAST ast, IntSet set) {
+		String inRuleName = ast.nfaState.rule.name;
+		String name = "LOOK_in_"+inRuleName+"_"+ast.token.getTokenIndex();
+		BitSetDef b = new BitSetDef(this, name, set);
 		outputModel.bitSetDefs.add(b);
 		return b;
 	}

tool/src/org/antlr/v4/codegen/OutputModelWalker.java

@@ -73,9 +73,16 @@ public class OutputModelWalker {
 					ST nestedST = walk(nestedOmo);
 					st.add(fieldName, nestedST);
 				}
-				else if ( o instanceof Collection ) {    // LIST OF MODEL OBJECTS?
+				else if ( o instanceof Collection || o instanceof OutputModelObject[] ) {
+					// LIST OF MODEL OBJECTS?
+					if ( o instanceof OutputModelObject[] ) {
+						o = Arrays.asList((OutputModelObject[])o);
+					}
 					Collection<? extends OutputModelObject> nestedOmos = (Collection)o;
 					for (OutputModelObject nestedOmo : nestedOmos) {
+						if ( nestedOmo==null ) {
+							System.out.println("collection has nulls: "+nestedOmos);
+						}
 						ST nestedST = walk(nestedOmo);
 						st.add(fieldName, nestedST);
 					}

tool/src/org/antlr/v4/codegen/SourceGenTriggers.g

@@ -24,14 +24,14 @@ import java.util.HashMap;
     }
 }
 
-block returns [CodeBlock omo]
+block[GrammarAST label, GrammarAST ebnfRoot] returns [SrcOp omo]
-    :	^(	BLOCK (^(OPTIONS .+))?
+    :	^( blk=BLOCK (^(OPTIONS .+))?
 		{List<CodeBlock> alts = new ArrayList<CodeBlock>();}
     		( alternative {alts.add($alternative.omo);} )+
     	)
     	{
-    	Choice c = new LL1Choice(gen, alts); // TODO: assumes LL1
+    	if ( alts.size()==1 && ebnfRoot==null) return alts.get(0);
-		$omo = new CodeBlock(gen, c);
+    	$omo = gen.getChoiceBlock((BlockAST)$blk, $ebnfRoot, alts);
     	}
     ;
 
@@ -43,9 +43,9 @@ alternative returns [CodeBlock omo]
     ;
 
 element returns [SrcOp omo]
-	:	labeledElement				
+	:	labeledElement					{$omo = $labeledElement.omo;}
-	|	atom						{$omo = $atom.omo;}
+	|	atom[null]						{$omo = $atom.omo;}
-	|	ebnf						
+	|	ebnf							{$omo = $ebnf.omo;}						
 	|   ACTION						
 	|   SEMPRED					
 	|	GATED_SEMPRED	
@@ -53,10 +53,10 @@ element returns [SrcOp omo]
 	;
 	
 labeledElement returns [SrcOp omo]
-	:	^(ASSIGN ID  atom  )
+	:	^(ASSIGN ID atom[$ID] )				{$omo = $atom.omo;}
-	|	^(ASSIGN ID block)			
+	|	^(ASSIGN ID block[$ID,null])		{$omo = $block.omo;}
-	|	^(PLUS_ASSIGN ID atom)		
+	|	^(PLUS_ASSIGN ID atom[$ID])			{$omo = $atom.omo;}
-	|	^(PLUS_ASSIGN ID block)		
+	|	^(PLUS_ASSIGN ID block[$ID,null])	{$omo = $block.omo;}	
 	;
 
 treeSpec returns [SrcOp omo]
@@ -64,14 +64,12 @@ treeSpec returns [SrcOp omo]
     ;
 
 ebnf returns [SrcOp omo]
-	:	^(astBlockSuffix block)		
+	:	^(astBlockSuffix block[null,null])		
-	|	^(OPTIONAL block)			
+	|	^(OPTIONAL block[null,$OPTIONAL])	{$omo = $block.omo;}		
-		
+	|	^(CLOSURE block[null,$CLOSURE])		{$omo = $block.omo;}		
-	|	^(CLOSURE block)			
+	|	^(POSITIVE_CLOSURE block[null,$POSITIVE_CLOSURE])
-		
+										    {$omo = $block.omo;}
-	|	^(POSITIVE_CLOSURE block)
+	| 	block[null, null]					{$omo = $block.omo;}
-		
-	| 	block						
     ;
 
 astBlockSuffix
@@ -80,43 +78,44 @@ astBlockSuffix
     | BANG
     ;
 
-atom returns [SrcOp omo]
+// TODO: combine ROOT/BANG into one then just make new op ref'ing return value of atom/terminal...
-	:	^(ROOT range)			
+// TODO: same for NOT
-	|	^(BANG range)			
+atom[GrammarAST label] returns [SrcOp omo]
-	|	^(ROOT notSet)			
+	:	^(ROOT range[label])			
-	|	^(BANG notSet)			
+	|	^(BANG range[label])		{$omo = $range.omo;}	
-	|	notSet					
+	|	^(ROOT notSet[label])			
-	|	range					
+	|	^(BANG notSet[label])		{$omo = $notSet.omo;}	
-	|	^(DOT ID terminal)		
+	|	notSet[label]					
-	|	^(DOT ID ruleref)		
+	|	range[label]				{$omo = $range.omo;}	
+	|	^(DOT ID terminal[label])
+	|	^(DOT ID ruleref[label])
     |	^(WILDCARD .)				
     |	WILDCARD				
-    |   terminal				{$omo = $terminal.omo;}
+    |   terminal[label]				{$omo = $terminal.omo;}
-    |   ruleref					{$omo = $ruleref.omo;}
+    |   ruleref[label]				{$omo = $ruleref.omo;}
     ;
 
-notSet returns [SrcOp omo]
+notSet[GrammarAST label] returns [SrcOp omo]
-    : ^(NOT terminal)		
+    : ^(NOT terminal[label])		
-    | ^(NOT block)			
+    | ^(NOT block[label,null])			
     ;
 
-ruleref returns [SrcOp omo]
+ruleref[GrammarAST label] returns [SrcOp omo]
     :	^(ROOT ^(RULE_REF ARG_ACTION?))
-    |	^(BANG ^(RULE_REF ARG_ACTION?))	
+    |	^(BANG ^(RULE_REF ARG_ACTION?))	{$omo = new InvokeRule(gen, $RULE_REF, $label);}
-    |	^(RULE_REF ARG_ACTION?)			
+    |	^(RULE_REF ARG_ACTION?)			{$omo = new InvokeRule(gen, $RULE_REF, $label);}
     ;
 
-range returns [SrcOp omo]
+range[GrammarAST label] returns [SrcOp omo]
     :	^(RANGE a=STRING_LITERAL b=STRING_LITERAL)    	
-    	
     ;
 
-terminal returns [MatchToken omo]
+terminal[GrammarAST label] returns [MatchToken omo]
-    :  ^(STRING_LITERAL .)			
+    :  ^(STRING_LITERAL .)			{$omo = new MatchToken(gen, (TerminalAST)$STRING_LITERAL, $label);}
-    |	STRING_LITERAL				
+    |	STRING_LITERAL				{$omo = new MatchToken(gen, (TerminalAST)$STRING_LITERAL, $label);}
-    |	^(TOKEN_REF ARG_ACTION .)	
+    |	^(TOKEN_REF ARG_ACTION .)	{$omo = new MatchToken(gen, (TerminalAST)$TOKEN_REF, $label);}
-    |	^(TOKEN_REF .)				
+    |	^(TOKEN_REF .)				{$omo = new MatchToken(gen, (TerminalAST)$TOKEN_REF, $label);}
-    |	TOKEN_REF					{$omo = new MatchToken(gen, (TerminalAST)$TOKEN_REF);}
+    |	TOKEN_REF					{$omo = new MatchToken(gen, (TerminalAST)$TOKEN_REF, $label);}
-    |	^(ROOT terminal)			
+    |	^(ROOT terminal[label])			
-    |	^(BANG terminal)			
+    |	^(BANG terminal[label])			
     ;

tool/src/org/antlr/v4/codegen/Target.java

@@ -74,6 +74,14 @@ public class Target {
 		return name;
 	}
 
+	public String[] getTokenTypeAsTargetLabel(Grammar g, int[] ttypes) {
+		String[] labels = new String[ttypes.length];
+		for (int i=0; i<ttypes.length; i++) {
+			labels[i] = getTokenTypeAsTargetLabel(g, ttypes[i]);
+		}
+		return labels;
+	}
+
 	/** Convert from an ANTLR char literal found in a grammar file to
 	 *  an equivalent char literal in the target language.  For most
 	 *  languages, this means leaving 'x' as 'x'.  Actually, we need
@@ -112,4 +120,5 @@ public class Target {
 		return buf.toString();
 	}
 
+	public int getInlineTestsVsBitsetThreshold() { return 20; }
 }

tool/src/org/antlr/v4/codegen/src/Choice.java

@@ -1,22 +1,28 @@
 package org.antlr.v4.codegen.src;
 
+import org.antlr.v4.automata.BlockStartState;
 import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.tool.GrammarAST;
 
 import java.util.ArrayList;
 import java.util.List;
 
 /** */
 public abstract class Choice extends SrcOp {
-	public DFADef dfaDef;
+	public int decision;
 	public List<CodeBlock> alts;
+	public List<Decl> decls;
 
-	public Choice(CodeGenerator gen, List<CodeBlock> alts) {
+	public Choice(CodeGenerator gen, GrammarAST blkOrEbnfRootAST, List<CodeBlock> alts) {
 		this.gen = gen;
+		this.ast = blkOrEbnfRootAST;
 		this.alts = alts;
+		this.decision = ((BlockStartState)blkOrEbnfRootAST.nfaState).decision;
 	}
 
 	@Override
 	public List<String> getChildren() {
-		return new ArrayList<String>() {{ add("alts"); }};
+		final List<String> sup = super.getChildren();
+		return new ArrayList<String>() {{ if ( sup!=null ) addAll(sup); add("alts"); add("decls"); }};
 	}
 }

tool/src/org/antlr/v4/codegen/src/CodeBlock.java

@@ -22,6 +22,7 @@ public class CodeBlock extends SrcOp {
 
 	@Override
 	public List<String> getChildren() {
-		return new ArrayList<String>() {{ add("ops"); }};
+		final List<String> sup = super.getChildren();
+		return new ArrayList<String>() {{ if ( sup!=null ) addAll(sup); add("ops"); }};
 	}
 }

tool/src/org/antlr/v4/codegen/src/Decl.java

@@ -0,0 +1,7 @@
+package org.antlr.v4.codegen.src;
+
+/** */
+public class Decl extends SrcOp {
+	public String varName;
+	public Decl(String varName) { this.varName = varName; }
+}

tool/src/org/antlr/v4/codegen/src/InvokeRule.java

@@ -1,7 +1,9 @@
 package org.antlr.v4.codegen.src;
 
+import org.antlr.v4.analysis.LinearApproximator;
 import org.antlr.v4.codegen.CodeGenerator;
 import org.antlr.v4.misc.IntervalSet;
+import org.antlr.v4.tool.GrammarAST;
 
 import java.util.List;
 
@@ -10,17 +12,21 @@ public class InvokeRule extends SrcOp {
 	public String name;
 	public String label;
 	public List<String> args;
-	public IntervalSet[] follow;
+	public BitSetDef follow;
 
-	public InvokeRule(CodeGenerator gen, String name, String argAction, IntervalSet[] follow) {
+	public InvokeRule(CodeGenerator gen, GrammarAST ast, GrammarAST labelAST) {
 		this.gen = gen;
+		this.ast = ast;
+		this.name = ast.getText();
+		if ( labelAST!=null ) this.label = labelAST.getText();
+		if ( ast.getChildCount()>0 ) {
+			String argAction = ast.getChild(0).getText();
 			// split and translate argAction
-		// compute follow
 		}
-
-	public InvokeRule(CodeGenerator gen, String name, IntervalSet[] follow) {
-		this.gen = gen;
-		// split and translate argAction
 		// compute follow
+		LinearApproximator approx = new LinearApproximator(gen.g, -1);
+		IntervalSet fset = approx.LOOK(ast.nfaState.transition(0).target);
+		System.out.println("follow="+follow);
+		follow = gen.defineFollowBitSet(ast, fset);
 	}
 }

tool/src/org/antlr/v4/codegen/src/LL1Choice.java

@@ -1,12 +1,28 @@
 package org.antlr.v4.codegen.src;
 
+import org.antlr.v4.analysis.LinearApproximator;
+import org.antlr.v4.automata.DFA;
 import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.misc.IntervalSet;
+import org.antlr.v4.tool.GrammarAST;
 
+import java.util.ArrayList;
 import java.util.List;
 
-/** */
+/** (A | B | C) */
 public class LL1Choice extends Choice {
-	public LL1Choice(CodeGenerator gen, List<CodeBlock> alts) {
+	/** Token names for each alt 0..n-1 */
-		super(gen, alts);
+	public List<String[]> altLook;
+	/** Lookahead for each alt 1..n */
+	public IntervalSet[] altLookSets;
+	public LL1Choice(CodeGenerator gen, GrammarAST blkAST, List<CodeBlock> alts) {
+		super(gen, blkAST, alts);
+		DFA dfa = gen.g.decisionDFAs.get(decision);
+		altLookSets = LinearApproximator.getLL1LookaheadSets(dfa);
+		altLook = new ArrayList<String[]>();
+		for (int a=1; a<altLookSets.length; a++) {
+			IntervalSet s = altLookSets[a];
+			altLook.add(gen.target.getTokenTypeAsTargetLabel(gen.g, s.toArray()));
+		}
 	}
 }

tool/src/org/antlr/v4/codegen/src/LL1OptionalBlock.java

@@ -1,12 +1,13 @@
 package org.antlr.v4.codegen.src;
 
 import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.tool.GrammarAST;
 
 import java.util.List;
 
 /** */
-public class LL1OptionalBlock extends OptionalBlock {
+public class LL1OptionalBlock extends LL1Choice {
-	public LL1OptionalBlock(CodeGenerator gen, List<CodeBlock> alts) {
+	public LL1OptionalBlock(CodeGenerator gen, GrammarAST blkAST, List<CodeBlock> alts) {
-		super(gen, alts);
+		super(gen, blkAST, alts);
 	}
 }

tool/src/org/antlr/v4/codegen/src/LL1OptionalBlockSingleAlt.java

@@ -1,12 +1,31 @@
 package org.antlr.v4.codegen.src;
 
 import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.misc.IntervalSet;
+import org.antlr.v4.tool.GrammarAST;
 
+import java.util.ArrayList;
 import java.util.List;
 
 /** */
-public class LL1OptionalBlockSingleAlt extends OptionalBlock {
+public class LL1OptionalBlockSingleAlt extends LL1OptionalBlock {
-	public LL1OptionalBlockSingleAlt(CodeGenerator gen, List<CodeBlock> alts) {
+	public Object expr;
-		super(gen, alts);
+	public LL1OptionalBlockSingleAlt(CodeGenerator gen, GrammarAST blkAST, List<CodeBlock> alts) {
+		super(gen, blkAST, alts);
+		IntervalSet look = altLookSets[1];
+		if ( look.size() < gen.target.getInlineTestsVsBitsetThreshold() ) {
+			expr = new TestSetInline(gen, blkAST, look);
+			decls = new ArrayList<Decl>();
+			decls.add(new NextTokenDecl("la34"));
+		}
+		else {
+			expr = new TestSet(gen, blkAST, look);
+		}
+	}
+
+	@Override
+	public List<String> getChildren() {
+		final List<String> sup = super.getChildren();
+		return new ArrayList<String>() {{ if ( sup!=null ) addAll(sup); add("expr"); }};
 	}
 }

tool/src/org/antlr/v4/codegen/src/LLStarOptionalBlock.java

@@ -1,12 +1,15 @@
 package org.antlr.v4.codegen.src;
 
 import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.tool.BlockAST;
 
 import java.util.List;
 
 /** */
 public class LLStarOptionalBlock extends OptionalBlock {
-	public LLStarOptionalBlock(CodeGenerator gen, List<CodeBlock> alts) {
+	public DFADef dfaDef;
-		super(gen, alts);
+	public LLStarOptionalBlock(CodeGenerator gen, BlockAST blkAST, List<CodeBlock> alts) {
+		super(gen, blkAST, alts);
+		dfaDef = gen.defineDFA(ast, gen.g.decisionDFAs.get(decision));
 	}
 }

tool/src/org/antlr/v4/codegen/src/LLkOptionalBlock.java

@@ -1,12 +1,13 @@
 package org.antlr.v4.codegen.src;
 
 import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.tool.BlockAST;
 
 import java.util.List;
 
 /** */
 public class LLkOptionalBlock extends OptionalBlock {
-	public LLkOptionalBlock(CodeGenerator gen, List<CodeBlock> alts) {
+	public LLkOptionalBlock(CodeGenerator gen, BlockAST blkAST, List<CodeBlock> alts) {
-		super(gen, alts);
+		super(gen, blkAST, alts);
 	}
 }

tool/src/org/antlr/v4/codegen/src/MatchToken.java

@@ -3,20 +3,23 @@ package org.antlr.v4.codegen.src;
 import org.antlr.v4.analysis.LinearApproximator;
 import org.antlr.v4.codegen.CodeGenerator;
 import org.antlr.v4.misc.IntervalSet;
+import org.antlr.v4.tool.GrammarAST;
 import org.antlr.v4.tool.TerminalAST;
 
 /** */
 public class MatchToken extends SrcOp {
 	public String name;
 	public BitSetDef follow;
+	public String label;
 
-	public MatchToken(CodeGenerator gen, TerminalAST ast) {
+	public MatchToken(CodeGenerator gen, TerminalAST ast, GrammarAST labelAST) {
 		this.gen = gen;
 		name = ast.getText();
+		if ( labelAST!=null ) this.label = labelAST.getText();
 
 		LinearApproximator approx = new LinearApproximator(gen.g, -1);
 		IntervalSet fset = approx.LOOK(ast.nfaState.transition(0).target);
 		System.out.println("follow="+follow);
-		follow = gen.defineBitSet(ast, fset);
+		follow = gen.defineFollowBitSet(ast, fset);
 	}
 }

tool/src/org/antlr/v4/codegen/src/NextTokenDecl.java

@@ -0,0 +1,6 @@
+package org.antlr.v4.codegen.src;
+
+/** */
+public class NextTokenDecl extends Decl {
+	public NextTokenDecl(String varName) { super(varName); }
+}

tool/src/org/antlr/v4/codegen/src/OptionalBlock.java

@@ -1,17 +1,13 @@
 package org.antlr.v4.codegen.src;
 
 import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.tool.GrammarAST;
 
 import java.util.List;
 
 /** */
-public class OptionalBlock extends Choice {
+public abstract class OptionalBlock extends Choice {
-	public OptionalBlock(CodeGenerator gen, List<CodeBlock> alts) {
+	public OptionalBlock(CodeGenerator gen, GrammarAST blkAST, List<CodeBlock> alts) {
-		super(gen, alts);
+		super(gen, blkAST, alts);
-	}
-
-	@Override
-	public List<String> getChildren() {
-		return super.getChildren();
 	}
 }

tool/src/org/antlr/v4/codegen/src/Parser.java

@@ -27,6 +27,7 @@ public class Parser extends OutputModelObject {
 
 	@Override
 	public List<String> getChildren() {
-		return new ArrayList<String>() {{ add("funcs"); }};
+		final List<String> sup = super.getChildren();
+		return new ArrayList<String>() {{ if ( sup!=null ) addAll(sup); add("funcs"); }};
 	}
 }

tool/src/org/antlr/v4/codegen/src/ParserFile.java

@@ -19,7 +19,9 @@ public class ParserFile extends OutputModelObject {
 
 	@Override
 	public List<String> getChildren() {
+		final List<String> sup = super.getChildren();
 		return new ArrayList<String>() {{
+			if ( sup!=null ) addAll(sup);
 			add("parser");
 			add("dfaDefs");
 			add("bitSetDefs");

tool/src/org/antlr/v4/codegen/src/RuleFunction.java

@@ -28,7 +28,7 @@ public class RuleFunction extends OutputModelObject {
 	public List<String> exceptions;
 	public String finallyAction;
 
-	public CodeBlock code;
+	public SrcOp code;
 
 	public RuleFunction(CodeGenerator gen, Rule r) {
 		this.gen = gen;
@@ -52,7 +52,7 @@ public class RuleFunction extends OutputModelObject {
 		CommonTreeNodeStream nodes = new CommonTreeNodeStream(adaptor,blk);
 		SourceGenTriggers genTriggers = new SourceGenTriggers(nodes, gen);
 		try {
-			code = genTriggers.block(); // GEN Instr OBJECTS
+			code = genTriggers.block(null,null); // GEN Instr OBJECTS
 		}
 		catch (Exception e){
 			e.printStackTrace(System.err);
@@ -61,6 +61,7 @@ public class RuleFunction extends OutputModelObject {
 
 	@Override
 	public List<String> getChildren() {
-		return new ArrayList<String>() {{ add("code"); }};
+		final List<String> sup = super.getChildren();
+		return new ArrayList<String>() {{ if ( sup!=null ) addAll(sup); add("code"); }};
 	}
 }

tool/src/org/antlr/v4/codegen/src/TestSet.java

@@ -0,0 +1,13 @@
+package org.antlr.v4.codegen.src;
+
+import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.misc.IntervalSet;
+import org.antlr.v4.tool.GrammarAST;
+
+/** */
+public class TestSet extends OutputModelObject {
+	public BitSetDef set;
+	public TestSet(CodeGenerator gen, GrammarAST blkAST, IntervalSet set) {
+		this.set = gen.defineTestBitSet(blkAST, set);
+	}
+}

tool/src/org/antlr/v4/codegen/src/TestSetInline.java

@@ -0,0 +1,13 @@
+package org.antlr.v4.codegen.src;
+
+import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.misc.IntervalSet;
+import org.antlr.v4.tool.GrammarAST;
+
+/** */
+public class TestSetInline extends OutputModelObject {
+	public String[] ttypes;
+	public TestSetInline(CodeGenerator gen, GrammarAST blkAST, IntervalSet set) {
+		this.ttypes = gen.target.getTokenTypeAsTargetLabel(gen.g, set.toArray());
+	}
+}