reorg

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 6862]

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -74,7 +74,7 @@ LL1StarBlockSingleAlt(choice, expr, alts, preamble) ::= <<
 <preamble>
 while ( <expr> ) {
     <alts; separator="\n">
-    <choice.expr.nextToken.varName> = input.LA(1);
+    <choice.loop
 }
 >>
 
@@ -95,12 +95,11 @@ while (true) {
 }
 >>
 
-//LL1PlusBlockSingleAlt ::= LL1PlusBlock
-
 LL1PlusBlockSingleAlt(choice, expr, alts, preamble) ::= <<
 <preamble>
 do {
     <alts; separator="\n">
+    // TODO: only if !set
     <choice.expr.nextToken.varName> = input.LA(1);
 } while ( <expr> );
 >>
@@ -110,7 +109,7 @@ TestSet(s) ::= <<
 >>
 
 TestSetInline(s) ::= <<
-<s.ttypes:{ttype | <s.nextToken.varName>==<ttype>}; separator=" || ">
+<s.ttypes:{ttype | <s.varName>==<ttype>}; separator=" || ">
 >>
 
 cases(ttypes) ::= <<

tool/src/org/antlr/v4/codegen/CodeGenPipeline.java

@@ -17,7 +17,7 @@ public class CodeGenPipeline {
 	}
 
 	void processParser() {
-		ParserGenerator gen = new ParserGenerator(g);
+		CodeGenerator gen = new CodeGenerator(g);
 		gen.write();
 	}
 

tool/src/org/antlr/v4/codegen/CodeGenerator.java

@@ -1,22 +1,16 @@
 package org.antlr.v4.codegen;
 
-import org.antlr.v4.automata.DFA;
+import org.antlr.v4.codegen.src.OutputModelObject;
-import org.antlr.v4.codegen.src.*;
-import org.antlr.v4.misc.IntSet;
-import org.antlr.v4.misc.IntervalSet;
-import org.antlr.v4.parse.ANTLRParser;
-import org.antlr.v4.tool.BlockAST;
 import org.antlr.v4.tool.ErrorType;
 import org.antlr.v4.tool.Grammar;
-import org.antlr.v4.tool.GrammarAST;
 import org.stringtemplate.v4.*;
 
 import java.io.IOException;
 import java.io.Writer;
-import java.util.List;
 
-/** */
+/** General controller for code gen.  Can instantiate sub generator(s).
-public abstract class CodeGenerator {
+ */
+public class CodeGenerator {
 	public static final String TEMPLATE_ROOT = "org/antlr/v4/tool/templates/codegen";
 	public static final String VOCAB_FILE_EXTENSION = ".tokens";
 	public final static String vocabFilePattern =
@@ -26,7 +20,6 @@ public abstract class CodeGenerator {
 	public Grammar g;
 	public Target target;
 	public STGroup templates;
-	public ParserFile outputModel;
 
 	public int lineWidth = 72;
 
@@ -74,14 +67,14 @@ public abstract class CodeGenerator {
 //		}		
 	}
 
-	/** The parser, tree parser, etc... variants know to build the model */
-	public abstract OutputModelObject buildOutputModel();
-
 	public void write() {
-		OutputModelObject root = buildOutputModel();
+		OutputModelFactory factory = null;
+		if ( g.isParser() ) factory = new ParserFactory(this);
+		// ...
+		OutputModelObject outputModel = factory.buildOutputModel();
 
 		OutputModelWalker walker = new OutputModelWalker(g.tool, templates);
-		ST outputFileST = walker.walk(root);
+		ST outputFileST = walker.walk(outputModel);
 		
 		// WRITE FILES
 		try {
@@ -107,42 +100,6 @@ public abstract class CodeGenerator {
 		}
 	}
 
-	public Choice getChoiceBlock(BlockAST blkAST, GrammarAST ebnfRoot, List<CodeBlock> alts) {
-		// TODO: assumes LL1
-		int ebnf = 0;
-		if ( ebnfRoot!=null ) ebnf = ebnfRoot.getType();
-		Choice c = null;
-		switch ( ebnf ) {
-			case ANTLRParser.OPTIONAL :
-				if ( alts.size()==1 ) c = new LL1OptionalBlockSingleAlt(this, ebnfRoot, alts);
-				else c = new LL1OptionalBlock(this, ebnfRoot, alts);
-				break;
-			case ANTLRParser.CLOSURE :
-				if ( alts.size()==1 ) c = new LL1StarBlockSingleAlt(this, ebnfRoot, alts);
-				else c = new LL1StarBlock(this, ebnfRoot, alts);
-				break;
-			case ANTLRParser.POSITIVE_CLOSURE :
-				if ( alts.size()==1 ) c = new LL1PlusBlockSingleAlt(this, ebnfRoot, alts);
-				else c = new LL1PlusBlock(this, ebnfRoot, alts);
-				break;
-			default :
-				c = new LL1Choice(this, blkAST, alts);
-				break;
-		}
-		return c;
-	}
-
-	public OutputModelObject getLL1Test(Choice choice, IntervalSet look, GrammarAST blkAST) {
-		OutputModelObject expr;
-		if ( look.size() < target.getInlineTestsVsBitsetThreshold() ) {
-			expr = new TestSetInline(this, choice, blkAST, look);
-		}
-		else {
-			expr = new TestSet(this, blkAST, look);
-		}
-		return expr;
-	}
-
 	public void write(ST code, String fileName) throws IOException {
 		long start = System.currentTimeMillis();
 		Writer w = g.tool.getOutputFile(g, fileName);
@@ -174,37 +131,4 @@ public abstract class CodeGenerator {
 		return g.name+VOCAB_FILE_EXTENSION;
 	}
 
-	public DFADef defineDFA(GrammarAST ast, DFA dfa) {
-		return null;
-//		DFADef d = new DFADef(name, dfa);
-//		outputModel.dfaDefs.add(d);
-	}
-
-	public BitSetDef defineFollowBitSet(GrammarAST ast, IntSet set) {
-		String inRuleName = ast.nfaState.rule.name;
-		String elementName = ast.getText(); // assume rule ref
-		if ( ast.getType() == ANTLRParser.TOKEN_REF ) {
-			elementName = target.getTokenTypeAsTargetLabel(g, g.tokenNameToTypeMap.get(elementName));
-		}
-		String name = "FOLLOW_"+elementName+"_in_"+inRuleName+"_"+ast.token.getTokenIndex();
-		BitSetDef b = new BitSetDef(this, name, set);
-		outputModel.bitSetDefs.add(b);
-		return b;
-	}
-
-	public BitSetDef defineTestBitSet(GrammarAST ast, IntSet set) {
-		String inRuleName = ast.nfaState.rule.name;
-		String name = "LOOK_in_"+inRuleName+"_"+ast.token.getTokenIndex();
-		BitSetDef b = new BitSetDef(this, name, set);
-		outputModel.bitSetDefs.add(b);
-		return b;
-	}
-
-	public String getLoopLabel(GrammarAST ast) {
-		return "loop"+ ast.token.getTokenIndex();
-	}
-
-	public String getLoopCounter(GrammarAST ast) {
-		return "cnt"+ ast.token.getTokenIndex();
-	}
 }

tool/src/org/antlr/v4/codegen/OutputModelFactory.java

@@ -0,0 +1,109 @@
+package org.antlr.v4.codegen;
+
+import org.antlr.v4.automata.DFA;
+import org.antlr.v4.codegen.src.*;
+import org.antlr.v4.misc.IntSet;
+import org.antlr.v4.misc.IntervalSet;
+import org.antlr.v4.parse.ANTLRParser;
+import org.antlr.v4.tool.BlockAST;
+import org.antlr.v4.tool.Grammar;
+import org.antlr.v4.tool.GrammarAST;
+
+import java.util.List;
+
+/** */
+public abstract class OutputModelFactory {
+	public Grammar g;
+	public CodeGenerator gen;
+
+	public OutputModelObject root;	
+
+	protected OutputModelFactory(CodeGenerator gen) {
+		this.gen = gen;
+		this.g = gen.g;
+	}
+
+	public abstract OutputModelObject buildOutputModel();
+
+	public abstract ParserFile outputFile(String fileName);
+
+	public abstract Parser parser(ParserFile pf);
+
+	public CodeBlock epsilon() { return new CodeBlock(this); }
+
+	public CodeBlock alternative(List<SrcOp> elems) { return new CodeBlock(this, elems); }
+
+	public SrcOp action(GrammarAST ast) { return new Action(this, ast); }
+	public SrcOp sempred(GrammarAST ast) { return new SemPred(this, ast); }
+
+	public Choice getChoiceBlock(BlockAST blkAST, GrammarAST ebnfRoot, List<CodeBlock> alts) {
+		// TODO: assumes LL1
+		int ebnf = 0;
+		if ( ebnfRoot!=null ) ebnf = ebnfRoot.getType();
+		Choice c = null;
+		switch ( ebnf ) {
+			case ANTLRParser.OPTIONAL :
+				if ( alts.size()==1 ) c = new LL1OptionalBlockSingleAlt(this, ebnfRoot, alts);
+				else c = new LL1OptionalBlock(this, ebnfRoot, alts);
+				break;
+			case ANTLRParser.CLOSURE :
+				if ( alts.size()==1 ) c = new LL1StarBlockSingleAlt(this, ebnfRoot, alts);
+				else c = new LL1StarBlock(this, ebnfRoot, alts);
+				break;
+			case ANTLRParser.POSITIVE_CLOSURE :
+				if ( alts.size()==1 ) c = new LL1PlusBlockSingleAlt(this, ebnfRoot, alts);
+				else c = new LL1PlusBlock(this, ebnfRoot, alts);
+				break;
+			default :
+				c = new LL1Choice(this, blkAST, alts);
+				break;
+		}
+		return c;
+	}
+
+
+	public abstract void defineBitSet(BitSetDef b);
+
+	public OutputModelObject getLL1Test(IntervalSet look, GrammarAST blkAST) {
+		OutputModelObject expr;
+		if ( look.size() < gen.target.getInlineTestsVsBitsetThreshold() ) {
+			expr = new TestSetInline(this, blkAST, look);
+		}
+		else {
+			expr = new TestSet(this, blkAST, look);
+		}
+		return expr;
+	}
+
+	public DFADef defineDFA(GrammarAST ast, DFA dfa) {
+		return null;
+//		DFADef d = new DFADef(name, dfa);
+//		outputModel.dfaDefs.add(d);
+	}
+
+	public String getLoopLabel(GrammarAST ast) {
+		return "loop"+ ast.token.getTokenIndex();
+	}
+
+	public String getLoopCounter(GrammarAST ast) {
+		return "cnt"+ ast.token.getTokenIndex();
+	}
+
+	public BitSetDef createFollowBitSet(GrammarAST ast, IntSet set) {
+		String inRuleName = ast.nfaState.rule.name;
+		String elementName = ast.getText(); // assume rule ref
+		if ( ast.getType() == ANTLRParser.TOKEN_REF ) {
+			elementName = gen.target.getTokenTypeAsTargetLabel(g, g.tokenNameToTypeMap.get(elementName));
+		}
+		String name = "FOLLOW_"+elementName+"_in_"+inRuleName+"_"+ast.token.getTokenIndex();
+		BitSetDef b = new BitSetDef(this, name, set);
+		return b;
+	}
+
+	public BitSetDef createTestBitSet(GrammarAST ast, IntSet set) {
+		String inRuleName = ast.nfaState.rule.name;
+		String name = "LOOK_in_"+inRuleName+"_"+ast.token.getTokenIndex();
+		BitSetDef b = new BitSetDef(this, name, set);
+		return b;
+	}
+}

tool/src/org/antlr/v4/codegen/ParserFactory.java

@@ -0,0 +1,45 @@
+package org.antlr.v4.codegen;
+
+import org.antlr.v4.codegen.src.*;
+
+import java.util.Stack;
+
+/** */
+public class ParserFactory extends OutputModelFactory {
+//	public static final Map<Class, String> modelToTemplateMap = new HashMap<Class, String>() {{
+//		put(ParserFile.class, "parserFile");
+//		put(Parser.class, "parser");
+//		put(RuleFunction.class, "parserFunction");
+//		put(DFADef.class, "DFA");
+//		put(CodeBlock.class, "codeBlock");
+//		put(LL1Choice.class, "switch");
+//		put(MatchToken.class, "matchToken");
+//	}};
+
+	// Context ptrs
+	ParserFile file;
+	Parser parser;
+	Stack<RuleFunction> currentRule;
+
+	public ParserFactory(CodeGenerator gen) {
+		super(gen);
+	}
+
+	public OutputModelObject buildOutputModel() {
+		root = file = new ParserFile(this, gen.getRecognizerFileName());
+		file.parser = new Parser(this, file);
+
+		// side-effect: fills pf dfa and bitset defs
+		return file;
+	}
+
+	public ParserFile outputFile(String fileName) {
+		return new ParserFile(this, fileName);
+	}
+
+	public Parser parser(ParserFile pf) {
+		return new Parser(this, pf);
+	}
+
+	public void defineBitSet(BitSetDef b) { file.defineBitSet(b); }
+}

tool/src/org/antlr/v4/codegen/ParserGenerator.java

@@ -1,31 +0,0 @@
-package org.antlr.v4.codegen;
-
-import org.antlr.v4.codegen.src.OutputModelObject;
-import org.antlr.v4.codegen.src.Parser;
-import org.antlr.v4.codegen.src.ParserFile;
-import org.antlr.v4.tool.Grammar;
-
-/** */
-public class ParserGenerator extends CodeGenerator {
-//	public static final Map<Class, String> modelToTemplateMap = new HashMap<Class, String>() {{
-//		put(ParserFile.class, "parserFile");
-//		put(Parser.class, "parser");
-//		put(RuleFunction.class, "parserFunction");
-//		put(DFADef.class, "DFA");
-//		put(CodeBlock.class, "codeBlock");
-//		put(LL1Choice.class, "switch");
-//		put(MatchToken.class, "matchToken");
-//	}};
-
-	public ParserGenerator(Grammar g) {
-		super(g);
-	}
-
-	public OutputModelObject buildOutputModel() {
-		ParserFile pf = new ParserFile(this, getRecognizerFileName());
-		outputModel = pf;
-		pf.parser = new Parser(this, pf); // side-effect: fills pf dfa and bitset defs
-		// at this point, model is built
-		return outputModel;
-	}
-}

tool/src/org/antlr/v4/codegen/SourceGenTriggers.g

@@ -3,7 +3,6 @@ options {
 	language     = Java;
 	tokenVocab   = ANTLRParser;
 	ASTLabelType = GrammarAST;
-//	superClass   = NFABytecodeGenerator;
 }
 
 @header {
@@ -17,10 +16,10 @@ import java.util.HashMap;
 
 @members {
 // TODO: identical grammar to NFABytecodeTriggers; would be nice to combine
-	public CodeGenerator gen;
+	public OutputModelFactory factory;
-    public SourceGenTriggers(TreeNodeStream input, CodeGenerator gen) {
+    public SourceGenTriggers(TreeNodeStream input, OutputModelFactory factory) {
     	this(input);
-    	this.gen = gen;
+    	this.factory = factory;
     }
 }
 
@@ -31,23 +30,23 @@ block[GrammarAST label, GrammarAST ebnfRoot] returns [SrcOp omo]
     	)
     	{
     	if ( alts.size()==1 && ebnfRoot==null) return alts.get(0);
-    	$omo = gen.getChoiceBlock((BlockAST)$blk, $ebnfRoot, alts);
+    	$omo = factory.getChoiceBlock((BlockAST)$blk, $ebnfRoot, alts);
     	}
     ;
 
 alternative returns [CodeBlock omo]
 @init {List<SrcOp> elems = new ArrayList<SrcOp>();}
     :	^(ALT_REWRITE a=alternative .)	
-    |	^(ALT EPSILON) {$omo = new CodeBlock(gen);}
+    |	^(ALT EPSILON) {$omo = factory.epsilon();}
-    |   ^( ALT ( element {elems.add($element.omo);} )+ ) {$omo = new CodeBlock(gen, elems);}
+    |   ^( ALT ( element {elems.add($element.omo);} )+ ) {$omo = factory.alternative(elems);}
     ;
 
 element returns [SrcOp omo]
 	:	labeledElement					{$omo = $labeledElement.omo;}
 	|	atom[null]						{$omo = $atom.omo;}
 	|	ebnf							{$omo = $ebnf.omo;}						
-	|   ACTION							{$omo = new Action(gen, $ACTION);}
+	|   ACTION							{$omo = factory.action($ACTION);}
-	|   SEMPRED							{$omo = new SemPred(gen, $SEMPRED);}
+	|   SEMPRED							{$omo = factory.sempred($SEMPRED);}
 	|	GATED_SEMPRED	
 	|	treeSpec					
 	;
@@ -102,8 +101,8 @@ notSet[GrammarAST label] returns [SrcOp omo]
 
 ruleref[GrammarAST label] returns [SrcOp omo]
     :	^(ROOT ^(RULE_REF ARG_ACTION?))
-    |	^(BANG ^(RULE_REF ARG_ACTION?))	{$omo = new InvokeRule(gen, $RULE_REF, $label);}
+    |	^(BANG ^(RULE_REF ARG_ACTION?))	{$omo = new InvokeRule(factory, $RULE_REF, $label);}
-    |	^(RULE_REF ARG_ACTION?)			{$omo = new InvokeRule(gen, $RULE_REF, $label);}
+    |	^(RULE_REF ARG_ACTION?)			{$omo = new InvokeRule(factory, $RULE_REF, $label);}
     ;
 
 range[GrammarAST label] returns [SrcOp omo]
@@ -111,11 +110,11 @@ range[GrammarAST label] returns [SrcOp omo]
     ;
 
 terminal[GrammarAST label] returns [MatchToken omo]
-    :  ^(STRING_LITERAL .)			{$omo = new MatchToken(gen, (TerminalAST)$STRING_LITERAL, $label);}
+    :  ^(STRING_LITERAL .)			{$omo = new MatchToken(factory, (TerminalAST)$STRING_LITERAL, $label);}
-    |	STRING_LITERAL				{$omo = new MatchToken(gen, (TerminalAST)$STRING_LITERAL, $label);}
+    |	STRING_LITERAL				{$omo = new MatchToken(factory, (TerminalAST)$STRING_LITERAL, $label);}
-    |	^(TOKEN_REF ARG_ACTION .)	{$omo = new MatchToken(gen, (TerminalAST)$TOKEN_REF, $label);}
+    |	^(TOKEN_REF ARG_ACTION .)	{$omo = new MatchToken(factory, (TerminalAST)$TOKEN_REF, $label);}
-    |	^(TOKEN_REF .)				{$omo = new MatchToken(gen, (TerminalAST)$TOKEN_REF, $label);}
+    |	^(TOKEN_REF .)				{$omo = new MatchToken(factory, (TerminalAST)$TOKEN_REF, $label);}
-    |	TOKEN_REF					{$omo = new MatchToken(gen, (TerminalAST)$TOKEN_REF, $label);}
+    |	TOKEN_REF					{$omo = new MatchToken(factory, (TerminalAST)$TOKEN_REF, $label);}
     |	^(ROOT terminal[label])			
     |	^(BANG terminal[label])			
     ;

tool/src/org/antlr/v4/codegen/src/Action.java

@@ -1,9 +1,9 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.tool.GrammarAST;
 
 /** */
 public class Action extends SrcOp {
-	public Action(CodeGenerator gen, GrammarAST ast) { super(gen,ast); }
+	public Action(OutputModelFactory factory, GrammarAST ast) { super(factory,ast); }
 }

tool/src/org/antlr/v4/codegen/src/BitSetDef.java

@@ -1,14 +1,14 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.misc.IntSet;
 
 /** */
 public class BitSetDef extends OutputModelObject {
 	public String name;
 	public IntSet fset;
-	public BitSetDef(CodeGenerator gen, String name, IntSet fset) {
+	public BitSetDef(OutputModelFactory factory, String name, IntSet fset) {
-		this.gen = gen;
+		this.factory = factory;
 		this.name = name;
 		this.fset = fset;
 	}

tool/src/org/antlr/v4/codegen/src/Choice.java

@@ -1,7 +1,7 @@
 package org.antlr.v4.codegen.src;
 
 import org.antlr.v4.automata.BlockStartState;
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.tool.GrammarAST;
 
 import java.util.ArrayList;
@@ -13,15 +13,15 @@ public abstract class Choice extends SrcOp {
 	public List<CodeBlock> alts;
 	public List<SrcOp> preamble;
 
-	public Choice(CodeGenerator gen, GrammarAST blkOrEbnfRootAST, List<CodeBlock> alts) {
+	public Choice(OutputModelFactory factory, GrammarAST blkOrEbnfRootAST, List<CodeBlock> alts) {
-		this.gen = gen;
+		this.factory = factory;
 		this.ast = blkOrEbnfRootAST;
 		this.alts = alts;
 		this.decision = ((BlockStartState)blkOrEbnfRootAST.nfaState).decision;
 	}
 
 	public void addPreambleOp(SrcOp op) {
-		preamble = new ArrayList<SrcOp>();
+		if ( preamble==null ) preamble = new ArrayList<SrcOp>();
 		preamble.add(op);
 	}
 

tool/src/org/antlr/v4/codegen/src/CodeBlock.java

@@ -1,6 +1,6 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -9,15 +9,15 @@ import java.util.List;
 public class CodeBlock extends SrcOp {
 	public List<SrcOp> ops;
 
-	public CodeBlock(CodeGenerator gen) { this.gen = gen; }
+	public CodeBlock(OutputModelFactory factory) { this.factory = factory; }
 	
-	public CodeBlock(CodeGenerator gen, List<SrcOp> ops) {
+	public CodeBlock(OutputModelFactory factory, List<SrcOp> ops) {
-		this.gen = gen;
+		super(factory);
 		this.ops = ops;
 	}
 
-	public CodeBlock(CodeGenerator gen, final SrcOp elem) {
+	public CodeBlock(OutputModelFactory factory, final SrcOp elem) {
-		this(gen, new ArrayList<SrcOp>() {{ add(elem); }});
+		this(factory, new ArrayList<SrcOp>() {{ add(elem); }});
 	}
 
 	@Override

tool/src/org/antlr/v4/codegen/src/InvokeRule.java

@@ -1,7 +1,7 @@
 package org.antlr.v4.codegen.src;
 
 import org.antlr.v4.analysis.LinearApproximator;
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.misc.IntervalSet;
 import org.antlr.v4.tool.GrammarAST;
 
@@ -14,8 +14,8 @@ public class InvokeRule extends SrcOp {
 	public List<String> args;
 	public BitSetDef follow;
 
-	public InvokeRule(CodeGenerator gen, GrammarAST ast, GrammarAST labelAST) {
+	public InvokeRule(OutputModelFactory factory, GrammarAST ast, GrammarAST labelAST) {
-		this.gen = gen;
+		this.factory = factory;
 		this.ast = ast;
 		this.name = ast.getText();
 		if ( labelAST!=null ) this.label = labelAST.getText();
@@ -24,9 +24,10 @@ public class InvokeRule extends SrcOp {
 			// split and translate argAction
 		}
 		// compute follow
-		LinearApproximator approx = new LinearApproximator(gen.g, -1);
+		LinearApproximator approx = new LinearApproximator(factory.g, -1);
 		IntervalSet fset = approx.LOOK(ast.nfaState.transition(0).target);
 		System.out.println("follow="+follow);
-		follow = gen.defineFollowBitSet(ast, fset);
+		follow = factory.createFollowBitSet(ast, fset);
+		factory.defineBitSet(follow);
 	}
 }

tool/src/org/antlr/v4/codegen/src/LL1Choice.java

@@ -2,7 +2,7 @@ package org.antlr.v4.codegen.src;
 
 import org.antlr.v4.analysis.LinearApproximator;
 import org.antlr.v4.automata.DFA;
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.misc.IntervalSet;
 import org.antlr.v4.tool.GrammarAST;
 
@@ -15,14 +15,14 @@ public class LL1Choice extends Choice {
 	public List<String[]> altLook;
 	/** Lookahead for each alt 1..n */
 	public IntervalSet[] altLookSets;
-	public LL1Choice(CodeGenerator gen, GrammarAST blkAST, List<CodeBlock> alts) {
+	public LL1Choice(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlock> alts) {
-		super(gen, blkAST, alts);
+		super(factory, blkAST, alts);
-		DFA dfa = gen.g.decisionDFAs.get(decision);
+		DFA dfa = factory.g.decisionDFAs.get(decision);
 		altLookSets = LinearApproximator.getLL1LookaheadSets(dfa);
 		altLook = new ArrayList<String[]>();
 		for (int a=1; a<altLookSets.length; a++) {
 			IntervalSet s = altLookSets[a];
-			altLook.add(gen.target.getTokenTypesAsTargetLabels(gen.g, s.toArray()));
+			altLook.add(factory.gen.target.getTokenTypesAsTargetLabels(factory.g, s.toArray()));
 		}
 	}
 }

tool/src/org/antlr/v4/codegen/src/LL1OptionalBlock.java

@@ -1,13 +1,13 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.tool.GrammarAST;
 
 import java.util.List;
 
 /** */
 public class LL1OptionalBlock extends LL1Choice {
-	public LL1OptionalBlock(CodeGenerator gen, GrammarAST blkAST, List<CodeBlock> alts) {
+	public LL1OptionalBlock(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlock> alts) {
-		super(gen, blkAST, alts);
+		super(factory, blkAST, alts);
 	}
 }

tool/src/org/antlr/v4/codegen/src/LL1OptionalBlockSingleAlt.java

@@ -1,6 +1,6 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.misc.IntervalSet;
 import org.antlr.v4.tool.GrammarAST;
 
@@ -10,10 +10,15 @@ import java.util.List;
 /** */
 public class LL1OptionalBlockSingleAlt extends LL1Choice {
 	public Object expr;
-	public LL1OptionalBlockSingleAlt(CodeGenerator gen, GrammarAST blkAST, List<CodeBlock> alts) {
+	public LL1OptionalBlockSingleAlt(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlock> alts) {
-		super(gen, blkAST, alts);
+		super(factory, blkAST, alts);
 		IntervalSet look = altLookSets[1];
-		expr = gen.getLL1Test(this, look, blkAST);
+		expr = factory.getLL1Test(look, blkAST);
+		if ( expr instanceof TestSetInline ) {
+			TestSetInline e = (TestSetInline)expr;
+			CaptureNextToken nextToken = new CaptureNextToken(e.varName);
+			addPreambleOp(nextToken);
+		}
 	}
 
 	@Override

tool/src/org/antlr/v4/codegen/src/LL1PlusBlock.java

@@ -2,7 +2,7 @@ package org.antlr.v4.codegen.src;
 
 import org.antlr.v4.automata.DFA;
 import org.antlr.v4.automata.PlusBlockStartState;
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.misc.IntervalSet;
 import org.antlr.v4.tool.GrammarAST;
 
@@ -13,16 +13,16 @@ public class LL1PlusBlock extends LL1Choice {
 	public String loopLabel;
 	public String loopCounterVar;
 	public String[] exitLook;
-	public LL1PlusBlock(CodeGenerator gen, GrammarAST blkAST, List<CodeBlock> alts) {
+	public LL1PlusBlock(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlock> alts) {
-		super(gen, blkAST, alts);
+		super(factory, blkAST, alts);
 		PlusBlockStartState plusStart = (PlusBlockStartState)blkAST.nfaState;
 		int enterExitDecision = plusStart.decision;
 
-		DFA dfa = gen.g.decisionDFAs.get(enterExitDecision);
+		DFA dfa = factory.g.decisionDFAs.get(enterExitDecision);
 		IntervalSet exitLook = dfa.startState.edge(1).label;
-		this.exitLook = gen.target.getTokenTypesAsTargetLabels(gen.g, exitLook.toArray());
+		this.exitLook = factory.gen.target.getTokenTypesAsTargetLabels(factory.g, exitLook.toArray());
 
-		loopLabel = gen.getLoopLabel(blkAST);
+		loopLabel = factory.getLoopLabel(blkAST);
-		loopCounterVar = gen.getLoopCounter(blkAST);
+		loopCounterVar = factory.getLoopCounter(blkAST);
 	}
 }

tool/src/org/antlr/v4/codegen/src/LL1PlusBlockSingleAlt.java

@@ -1,6 +1,6 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.misc.IntervalSet;
 import org.antlr.v4.tool.GrammarAST;
 
@@ -10,10 +10,16 @@ import java.util.List;
 /** */
 public class LL1PlusBlockSingleAlt extends LL1Choice {
 	public Object expr;
-	public LL1PlusBlockSingleAlt(CodeGenerator gen, GrammarAST blkAST, List<CodeBlock> alts) {
+	public List<SrcOp> loopIteration = new ArrayList<SrcOp>();
-		super(gen, blkAST, alts);
+	public LL1PlusBlockSingleAlt(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlock> alts) {
+		super(factory, blkAST, alts);
 		IntervalSet loopBackLook = altLookSets[2]; // loop exit is alt 1
-		expr = gen.getLL1Test(this, loopBackLook, blkAST);
+		expr = factory.getLL1Test(loopBackLook, blkAST);
+		if ( expr instanceof TestSetInline ) {
+			CaptureNextToken nextToken = new CaptureNextToken("la"+blkAST.token.getTokenIndex());
+			addPreambleOp(nextToken);
+			loopIteration.add(nextToken);
+		}
 	}
 
 	@Override

tool/src/org/antlr/v4/codegen/src/LL1StarBlock.java

@@ -2,7 +2,7 @@ package org.antlr.v4.codegen.src;
 
 import org.antlr.v4.automata.DFA;
 import org.antlr.v4.automata.StarBlockStartState;
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.misc.IntervalSet;
 import org.antlr.v4.tool.GrammarAST;
 
@@ -12,20 +12,20 @@ import java.util.List;
 public class LL1StarBlock extends LL1Choice {
 	public String loopLabel;
 	public String[] exitLook;
-	public LL1StarBlock(CodeGenerator gen, GrammarAST blkAST, List<CodeBlock> alts) {
+	public LL1StarBlock(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlock> alts) {
 		// point at choice block inside outermost enter-exit choice
-		super(gen, ((StarBlockStartState)blkAST.nfaState).transition(0).target.ast, alts);
+		super(factory, ((StarBlockStartState)blkAST.nfaState).transition(0).target.ast, alts);
 		StarBlockStartState starStart = (StarBlockStartState)blkAST.nfaState;
 		int enterExitDecision = starStart.decision;
 //		BlockStartState blkStart = (BlockStartState)starStart.transition(0).target;
 //		this.decision = blkStart.decision;
 		int loopbackDecision = starStart.loopBackState.decision;
 
-		DFA dfa = gen.g.decisionDFAs.get(enterExitDecision);
+		DFA dfa = factory.g.decisionDFAs.get(enterExitDecision);
 		IntervalSet exitLook = dfa.startState.edge(1).label;
-		this.exitLook = gen.target.getTokenTypesAsTargetLabels(gen.g, exitLook.toArray());
+		this.exitLook = factory.gen.target.getTokenTypesAsTargetLabels(factory.g, exitLook.toArray());
 
-		loopLabel = gen.getLoopLabel(blkAST);
+		loopLabel = factory.getLoopLabel(blkAST);
 	}
 
 }

tool/src/org/antlr/v4/codegen/src/LL1StarBlockSingleAlt.java

@@ -1,6 +1,6 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.misc.IntervalSet;
 import org.antlr.v4.tool.GrammarAST;
 
@@ -10,10 +10,17 @@ import java.util.List;
 /** */
 public class LL1StarBlockSingleAlt extends LL1Choice {
 	public Object expr;
-	public LL1StarBlockSingleAlt(CodeGenerator gen, GrammarAST blkAST, List<CodeBlock> alts) {
+	public List<SrcOp> loopIteration = new ArrayList<SrcOp>();
-		super(gen, blkAST, alts);
+	public LL1StarBlockSingleAlt(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlock> alts) {
+		super(factory, blkAST, alts);
 		IntervalSet look = altLookSets[1];
-		expr = gen.getLL1Test(this, look, blkAST);
+		expr = factory.getLL1Test(look, blkAST);
+		if ( expr instanceof TestSetInline ) {
+			TestSetInline e = (TestSetInline)expr;
+			CaptureNextToken nextToken = new CaptureNextToken(e.varName);
+			addPreambleOp(nextToken);
+			loopIteration.add(nextToken);
+		}
 	}
 
 	@Override

tool/src/org/antlr/v4/codegen/src/LLStarOptionalBlock.java

@@ -1,6 +1,6 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.tool.BlockAST;
 
 import java.util.List;
@@ -8,8 +8,8 @@ import java.util.List;
 /** */
 public class LLStarOptionalBlock extends OptionalBlock {
 	public DFADef dfaDef;
-	public LLStarOptionalBlock(CodeGenerator gen, BlockAST blkAST, List<CodeBlock> alts) {
+	public LLStarOptionalBlock(OutputModelFactory factory, BlockAST blkAST, List<CodeBlock> alts) {
-		super(gen, blkAST, alts);
+		super(factory, blkAST, alts);
-		dfaDef = gen.defineDFA(ast, gen.g.decisionDFAs.get(decision));
+		dfaDef = factory.defineDFA(ast, factory.g.decisionDFAs.get(decision));
 	}
 }

tool/src/org/antlr/v4/codegen/src/LLkOptionalBlock.java

@@ -1,13 +1,13 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.tool.BlockAST;
 
 import java.util.List;
 
 /** */
 public class LLkOptionalBlock extends OptionalBlock {
-	public LLkOptionalBlock(CodeGenerator gen, BlockAST blkAST, List<CodeBlock> alts) {
+	public LLkOptionalBlock(OutputModelFactory factory, BlockAST blkAST, List<CodeBlock> alts) {
-		super(gen, blkAST, alts);
+		super(factory, blkAST, alts);
 	}
 }

tool/src/org/antlr/v4/codegen/src/MatchToken.java

@@ -1,7 +1,7 @@
 package org.antlr.v4.codegen.src;
 
 import org.antlr.v4.analysis.LinearApproximator;
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.misc.IntervalSet;
 import org.antlr.v4.tool.GrammarAST;
 import org.antlr.v4.tool.TerminalAST;
@@ -12,14 +12,15 @@ public class MatchToken extends SrcOp {
 	public BitSetDef follow;
 	public String label;
 
-	public MatchToken(CodeGenerator gen, TerminalAST ast, GrammarAST labelAST) {
+	public MatchToken(OutputModelFactory factory, TerminalAST ast, GrammarAST labelAST) {
-		this.gen = gen;
+		this.factory = factory;
 		name = ast.getText();
 		if ( labelAST!=null ) this.label = labelAST.getText();
 
-		LinearApproximator approx = new LinearApproximator(gen.g, -1);
+		LinearApproximator approx = new LinearApproximator(factory.g, -1);
 		IntervalSet fset = approx.LOOK(ast.nfaState.transition(0).target);
 		System.out.println("follow="+follow);
-		follow = gen.defineFollowBitSet(ast, fset);
+		follow = factory.createFollowBitSet(ast, fset);
+		factory.defineBitSet(follow);
 	}
 }

tool/src/org/antlr/v4/codegen/src/OptionalBlock.java

@@ -1,13 +1,13 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.tool.GrammarAST;
 
 import java.util.List;
 
 /** */
 public abstract class OptionalBlock extends Choice {
-	public OptionalBlock(CodeGenerator gen, GrammarAST blkAST, List<CodeBlock> alts) {
+	public OptionalBlock(OutputModelFactory factory, GrammarAST blkAST, List<CodeBlock> alts) {
-		super(gen, blkAST, alts);
+		super(factory, blkAST, alts);
 	}
 }

tool/src/org/antlr/v4/codegen/src/OutputModelObject.java

@@ -1,13 +1,13 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.tool.GrammarAST;
 
 import java.util.List;
 
 /** */
 public abstract class OutputModelObject {
-	public CodeGenerator gen;
+	public OutputModelFactory factory;
 	public GrammarAST ast;
 
 	/** If the output model object encloses some other model objects,
@@ -24,10 +24,10 @@ public abstract class OutputModelObject {
 
 	public OutputModelObject() {;}
 	
-	public OutputModelObject(CodeGenerator gen) { this.gen = gen; }
+	public OutputModelObject(OutputModelFactory factory) { this.factory = factory; }
 
-	public OutputModelObject(CodeGenerator gen, GrammarAST ast) {
+	public OutputModelObject(OutputModelFactory factory, GrammarAST ast) {
-		this.gen = gen;
+		this.factory = factory;
 		this.ast = ast;
 	}
 }

tool/src/org/antlr/v4/codegen/src/Parser.java

@@ -1,6 +1,6 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.tool.Rule;
 
 import java.util.ArrayList;
@@ -12,11 +12,11 @@ public class Parser extends OutputModelObject {
 	public List<RuleFunction> funcs = new ArrayList<RuleFunction>();
 	ParserFile file;
 
-	public Parser(CodeGenerator gen, ParserFile file) {
+	public Parser(OutputModelFactory factory, ParserFile file) {
-		this.gen = gen;
+		this.factory = factory;
 		this.file = file; // who contains us?
-		name = gen.g.getRecognizerName();
+		name = factory.g.getRecognizerName();
-		for (Rule r : gen.g.rules.values()) funcs.add( new RuleFunction(gen, r) );
+		for (Rule r : factory.g.rules.values()) funcs.add( new RuleFunction(factory, r) );
 
 		// We create dfa and bitsets during rule function construction.
 		// They get stored in code gen for convenience as we walk rule block tree

tool/src/org/antlr/v4/codegen/src/ParserFile.java

@@ -1,6 +1,6 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -12,11 +12,15 @@ public class ParserFile extends OutputModelObject {
 	public List<DFADef> dfaDefs = new ArrayList<DFADef>();
 	public List<BitSetDef> bitSetDefs = new ArrayList<BitSetDef>();
 	
-	public ParserFile(CodeGenerator gen, String fileName) {
+	public ParserFile(OutputModelFactory factory, String fileName) {
-		this.gen = gen;
+		super(factory);
 		this.fileName = fileName; 
 	}
 
+	public void defineBitSet(BitSetDef b) {
+		bitSetDefs.add(b);
+	}
+
 	@Override
 	public List<String> getChildren() {
 		final List<String> sup = super.getChildren();

tool/src/org/antlr/v4/codegen/src/RuleFunction.java

@@ -1,7 +1,7 @@
 package org.antlr.v4.codegen.src;
 
 import org.antlr.runtime.tree.CommonTreeNodeStream;
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.codegen.SourceGenTriggers;
 import org.antlr.v4.misc.Utils;
 import org.antlr.v4.parse.ANTLRParser;
@@ -30,8 +30,8 @@ public class RuleFunction extends OutputModelObject {
 
 	public SrcOp code;
 
-	public RuleFunction(CodeGenerator gen, Rule r) {
+	public RuleFunction(OutputModelFactory factory, Rule r) {
-		this.gen = gen;
+		super(factory);
 		this.name = r.name;
 		if ( r.modifiers!=null && r.modifiers.size()>0 ) {
 			this.modifiers = new ArrayList<String>();
@@ -50,7 +50,7 @@ public class RuleFunction extends OutputModelObject {
 		GrammarASTAdaptor adaptor = new GrammarASTAdaptor(r.ast.token.getInputStream());
 		GrammarAST blk = (GrammarAST)r.ast.getFirstChildWithType(ANTLRParser.BLOCK);
 		CommonTreeNodeStream nodes = new CommonTreeNodeStream(adaptor,blk);
-		SourceGenTriggers genTriggers = new SourceGenTriggers(nodes, gen);
+		SourceGenTriggers genTriggers = new SourceGenTriggers(nodes, factory);
 		try {
 			code = genTriggers.block(null,null); // GEN Instr OBJECTS
 		}

tool/src/org/antlr/v4/codegen/src/SemPred.java

@@ -1,9 +1,9 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.tool.GrammarAST;
 
 /** */
 public class SemPred extends SrcOp {
-	public SemPred(CodeGenerator gen, GrammarAST ast) { super(gen,ast); }
+	public SemPred(OutputModelFactory factory, GrammarAST ast) { super(factory,ast); }
 }

tool/src/org/antlr/v4/codegen/src/SrcOp.java

@@ -1,11 +1,11 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.tool.GrammarAST;
 
 /** */
 public abstract class SrcOp extends OutputModelObject {
 	public SrcOp() {;}
-	public SrcOp(CodeGenerator gen) { super(gen); }
+	public SrcOp(OutputModelFactory factory) { super(factory); }
-	public SrcOp(CodeGenerator gen, GrammarAST ast) { super(gen,ast); }
+	public SrcOp(OutputModelFactory factory, GrammarAST ast) { super(factory,ast); }
 }

tool/src/org/antlr/v4/codegen/src/TestSet.java

@@ -1,13 +1,14 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.misc.IntervalSet;
 import org.antlr.v4.tool.GrammarAST;
 
 /** */
 public class TestSet extends OutputModelObject {
 	public BitSetDef set;
-	public TestSet(CodeGenerator gen, GrammarAST blkAST, IntervalSet set) {
+	public TestSet(OutputModelFactory factory, GrammarAST blkAST, IntervalSet set) {
-		this.set = gen.defineTestBitSet(blkAST, set);
+		this.set = factory.createTestBitSet(blkAST, set);
+		factory.defineBitSet(this.set);
 	}
 }

tool/src/org/antlr/v4/codegen/src/TestSetInline.java

@@ -1,20 +1,21 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.codegen.CodeGenerator;
+import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.misc.IntervalSet;
 import org.antlr.v4.tool.GrammarAST;
 
 /** */
 public class TestSetInline extends OutputModelObject {
+	public String varName;
 	public String[] ttypes;
-	public CaptureNextToken nextToken;
+//	public CaptureNextToken nextToken;
-	public Choice choice;
+//	public Choice choice;
-	public TestSetInline(CodeGenerator gen, Choice choice, GrammarAST blkAST, IntervalSet set) {
+	public TestSetInline(OutputModelFactory factory, GrammarAST blkAST, IntervalSet set) {
-		this.gen = gen;
+		super(factory, blkAST);
-		this.ast = blkAST;
+		this.ttypes = factory.gen.target.getTokenTypesAsTargetLabels(factory.g, set.toArray());
-		this.ttypes = gen.target.getTokenTypesAsTargetLabels(gen.g, set.toArray());
+		this.varName = "la"+blkAST.token.getTokenIndex();
-		this.choice = choice;
+//		this.choice = choice;
-		nextToken = new CaptureNextToken("la"+blkAST.token.getTokenIndex());
+//		nextToken = new CaptureNextToken();
-		choice.addPreambleOp(nextToken);
+//		choice.addPreambleOp(nextToken);
 	}
 }