track AST nodes -> NFA now and add more code gen

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 6846]

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -23,6 +23,20 @@ parserFunction(f,code) ::= <<
 }
 >>
 
+codeBlock(c, ops) ::= <<
+<ops>
+>>
+
+switch(c, alts) ::= <<
+switch ( input.LA(1) ) {
+	<alts>
+}
+>>
+
+matchToken(m) ::= <<
+match(<m.ttype>);
+>>
+
 codeFileExtension() ::= ".java"
 
 true() ::= "true"

tool/src/org/antlr/v4/automata/ParserNFAFactory.java

@@ -65,34 +65,10 @@ public class ParserNFAFactory implements NFAFactory {
 		Handle h = new Handle(start, stop);
 //		FASerializer ser = new FASerializer(g, h.left);
 //		System.out.println(ruleAST.toStringTree()+":\n"+ser);
+		ruleAST.nfaState = start;
 		return h;
 	}
 
-	public NFAState newState(Class nodeType, GrammarAST node) {
-		try {
-			Constructor ctor = nodeType.getConstructor(NFA.class);
-			NFAState s = (NFAState)ctor.newInstance(nfa);
-			s.ast = node;
-			s.rule = currentRule;
-			nfa.addState(s);
-			return s;
-		}
-		catch (Exception e) {
-			ErrorManager.internalError("can't create NFA node: "+nodeType.getName(), e);
-		}
-		return null;
-	}
-
-	public BasicState newState(GrammarAST node) {
-		BasicState n = new BasicState(nfa);
-		n.rule = currentRule;
-		n.ast = node;
-		nfa.addState(n);
-		return n;
-	}
-
-	public BasicState newState() { return newState(null); }
-
 	/** From label A build Graph o-A->o */
 	public Handle tokenRef(TerminalAST node) {
 		BasicState left = newState(node);
@@ -100,7 +76,7 @@ public class ParserNFAFactory implements NFAFactory {
 		int ttype = g.getTokenType(node.getText());
 		left.transition = new AtomTransition(ttype, right);
 		right.incidentTransition = left.transition;
-		
+		node.nfaState = left;
 		return new Handle(left, right);
 	}
 
@@ -112,7 +88,7 @@ public class ParserNFAFactory implements NFAFactory {
 		BasicState right = newState(associatedAST);
 		left.transition = new SetTransition(set, right);
 		right.incidentTransition = left.transition;
-
+		associatedAST.nfaState = left;
 		return new Handle(left, right);
 	}
 
@@ -168,6 +144,7 @@ public class ParserNFAFactory implements NFAFactory {
 		RuleStopState stop = nfa.ruleToStopState.get(r);
 		epsilon(stop, right);
 
+		node.nfaState = left;
 		return new Handle(left, right);
 	}
 
@@ -176,6 +153,7 @@ public class ParserNFAFactory implements NFAFactory {
 		BasicState left = newState(node);
 		BasicState right = newState(node);
 		epsilon(left, right);
+		node.nfaState = left;
 		return new Handle(left, right);
 	}
 
@@ -188,6 +166,7 @@ public class ParserNFAFactory implements NFAFactory {
 		BasicState left = newState(pred);
 		NFAState right = newState(pred);
 		left.transition = new PredicateTransition(pred, right);
+		pred.nfaState = left;
 		return new Handle(left, right);
 	}
 
@@ -205,6 +184,7 @@ public class ParserNFAFactory implements NFAFactory {
 		BasicState left = newState(action);
 		NFAState right = newState(action);
 		left.transition = new ActionTransition(action, right);  
+		action.nfaState = left;
 		return new Handle(left, right);
 	}
 
@@ -248,6 +228,7 @@ public class ParserNFAFactory implements NFAFactory {
 		Handle h = new Handle(start, end);
 //		FASerializer ser = new FASerializer(g, h.left);
 //		System.out.println(blkAST.toStringTree()+":\n"+ser);
+		blkAST.nfaState = start;		
 		return h;
 	}
 
@@ -293,6 +274,7 @@ public class ParserNFAFactory implements NFAFactory {
 		Handle h = new Handle(start, end);
 //		FASerializer ser = new FASerializer(g, h.left);
 //		System.out.println(optAST.toStringTree()+":\n"+ser);
+		optAST.nfaState = start;
 		return h;
 	}
 
@@ -315,6 +297,7 @@ public class ParserNFAFactory implements NFAFactory {
 		epsilon(blk.right, loop);
 		epsilon(loop, end);
 		nfa.defineDecisionState(loop);
+		plusAST.nfaState = start;
 		return new Handle(start, end);
 	}
 
@@ -351,6 +334,7 @@ public class ParserNFAFactory implements NFAFactory {
 		epsilon(loop, end);
 		nfa.defineDecisionState(start);
 		nfa.defineDecisionState(loop);
+		starAST.nfaState = start;
 		return new Handle(start, end);
 	}
 
@@ -404,4 +388,29 @@ public class ParserNFAFactory implements NFAFactory {
 		}
 		return n;
 	}
+
+	public NFAState newState(Class nodeType, GrammarAST node) {
+		try {
+			Constructor ctor = nodeType.getConstructor(NFA.class);
+			NFAState s = (NFAState)ctor.newInstance(nfa);
+			s.ast = node;
+			s.rule = currentRule;
+			nfa.addState(s);
+			return s;
+		}
+		catch (Exception e) {
+			ErrorManager.internalError("can't create NFA node: "+nodeType.getName(), e);
+		}
+		return null;
+	}
+
+	public BasicState newState(GrammarAST node) {
+		BasicState n = new BasicState(nfa);
+		n.rule = currentRule;
+		n.ast = node;
+		nfa.addState(n);
+		return n;
+	}
+
+	public BasicState newState() { return newState(null); }	
 }

tool/src/org/antlr/v4/codegen/CodeGenerator.java

@@ -12,15 +12,15 @@ import java.io.Writer;
 public abstract class CodeGenerator {
 	public static final String TEMPLATE_ROOT = "org/antlr/v4/tool/templates/codegen";
 	public static final String VOCAB_FILE_EXTENSION = ".tokens";
-	protected final static String vocabFilePattern =
+	public final static String vocabFilePattern =
 		"<tokens:{<attr.name>=<attr.type>\n}>" +
 		"<literals:{<attr.name>=<attr.type>\n}>";
 
-	Grammar g;
-	Target target;
-	STGroup templates;
+	public Grammar g;
+	public Target target;
+	public STGroup templates;
 
-	int lineWidth = 72;
+	public int lineWidth = 72;
 
 	public CodeGenerator(Grammar g) {
 		this.g = g;

tool/src/org/antlr/v4/codegen/ParserGenerator.java

@@ -14,6 +14,9 @@ public class ParserGenerator extends CodeGenerator {
 		put(Parser.class, "parser");
 		put(RuleFunction.class, "parserFunction");
 		put(DFADef.class, "DFA");
+		put(CodeBlock.class, "codeBlock");
+		put(LL1Choice.class, "switch");
+		put(MatchToken.class, "matchToken");
 	}};
 
 	public ParserGenerator(Grammar g) {
@@ -21,7 +24,7 @@ public class ParserGenerator extends CodeGenerator {
 	}
 
 	public OutputModelObject buildOutputModel() {
-		Parser p = new Parser(g);
-		return new ParserFile(p, getRecognizerFileName());
+		Parser p = new Parser(this);
+		return new ParserFile(this, p, getRecognizerFileName());
 	}
 }

tool/src/org/antlr/v4/codegen/SourceGenTriggers.g

@@ -8,31 +8,43 @@ options {
 
 @header {
 package org.antlr.v4.codegen;
-import org.antlr.v4.codegen.nfa.*;
-import org.antlr.v4.tool.GrammarAST;
-import org.antlr.v4.tool.GrammarASTWithOptions;
-import org.antlr.v4.tool.LexerGrammar;
+import org.antlr.v4.codegen.src.*;
+import org.antlr.v4.tool.*;
 import java.util.Collections;
 import java.util.Map;
 import java.util.HashMap;
 }
 
-block
+@members {
+// TODO: identical grammar to NFABytecodeTriggers; would be nice to combine
+	public CodeGenerator gen;
+    public SourceGenTriggers(TreeNodeStream input, CodeGenerator gen) {
+    	this(input);
+    	this.gen = gen;
+    }
+}
+
+block returns [CodeBlock omo]
     :	^(	BLOCK (^(OPTIONS .+))?
-    		(	alternative    			
-    		)+
+		{List<CodeBlock> alts = new ArrayList<CodeBlock>();}
+    		( alternative {alts.add($alternative.omo);} )+
     	)
+    	{
+    	Choice c = new LL1Choice(gen, alts); // TODO: assumes LL1
+		$omo = new CodeBlock(gen, c);
+    	}
     ;
 
-alternative
+alternative returns [CodeBlock omo]
+@init {List<SrcOp> elems = new ArrayList<SrcOp>();}
     :	^(ALT_REWRITE a=alternative .)	
-    |	^(ALT EPSILON)					
-    |   ^(ALT (e=element )+)    									
+    |	^(ALT EPSILON) {$omo = new CodeBlock(gen);}
+    |   ^( ALT ( element {elems.add($element.omo);} )+ ) {$omo = new CodeBlock(gen, elems);}
     ;
 
-element
+element returns [SrcOp omo]
 	:	labeledElement				
-	|	atom						
+	|	atom						{$omo = $atom.omo;}
 	|	ebnf						
 	|   ACTION						
 	|   SEMPRED					
@@ -40,22 +52,18 @@ element
 	|	treeSpec					
 	;
 	
-labeledElement
+labeledElement returns [SrcOp omo]
 	:	^(ASSIGN ID  atom  )
 	|	^(ASSIGN ID block)			
 	|	^(PLUS_ASSIGN ID atom)		
 	|	^(PLUS_ASSIGN ID block)		
 	;
 
-treeSpec
+treeSpec returns [SrcOp omo]
     : ^(TREE_BEGIN  (e=element )+)	
     ;
 
-ebnf
-@init {
-	GrammarASTWithOptions blk = (GrammarASTWithOptions)$start.getChild(0);
-	String greedyOption = blk.getOption("greedy");
-}
+ebnf returns [SrcOp omo]
 	:	^(astBlockSuffix block)		
 	|	^(OPTIONAL block)			
 		
@@ -72,7 +80,7 @@ astBlockSuffix
     | BANG
     ;
 
-atom
+atom returns [SrcOp omo]
 	:	^(ROOT range)			
 	|	^(BANG range)			
 	|	^(ROOT notSet)			
@@ -83,32 +91,32 @@ atom
 	|	^(DOT ID ruleref)		
     |	^(WILDCARD .)				
     |	WILDCARD				
-    |   terminal				
-    |   ruleref					
+    |   terminal				{$omo = $terminal.omo;}
+    |   ruleref					{$omo = $ruleref.omo;}
     ;
 
-notSet
+notSet returns [SrcOp omo]
     : ^(NOT terminal)		
     | ^(NOT block)			
     ;
 
-ruleref
+ruleref returns [SrcOp omo]
     :	^(ROOT ^(RULE_REF ARG_ACTION?))	
     |	^(BANG ^(RULE_REF ARG_ACTION?))	
     |	^(RULE_REF ARG_ACTION?)			
     ;
 
-range
+range returns [SrcOp omo]
     :	^(RANGE a=STRING_LITERAL b=STRING_LITERAL)
     	
     ;
 
-terminal
+terminal returns [MatchToken omo]
     :  ^(STRING_LITERAL .)			
     |	STRING_LITERAL				
     |	^(TOKEN_REF ARG_ACTION .)	
     |	^(TOKEN_REF .)				
-    |	TOKEN_REF					
+    |	TOKEN_REF					{$omo = new MatchToken(gen, (TerminalAST)$TOKEN_REF);}
     |	^(ROOT terminal)			
     |	^(BANG terminal)			
     ;

tool/src/org/antlr/v4/codegen/src/Choice.java

@@ -1,17 +1,18 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.tool.Alternative;
+import org.antlr.v4.codegen.CodeGenerator;
 
 import java.util.ArrayList;
 import java.util.List;
 
 /** */
-public class Choice extends SrcOp {
+public abstract class Choice extends SrcOp {
 	public DFADef dfaDef;
 	public List<CodeBlock> alts;
 
-	public Choice(Alternative[] alts) {
-		
+	public Choice(CodeGenerator gen, List<CodeBlock> alts) {
+		this.gen = gen;
+		this.alts = alts;
 	}
 
 	@Override

tool/src/org/antlr/v4/codegen/src/CodeBlock.java

@@ -1,5 +1,7 @@
 package org.antlr.v4.codegen.src;
 
+import org.antlr.v4.codegen.CodeGenerator;
+
 import java.util.ArrayList;
 import java.util.List;
 
@@ -7,6 +9,17 @@ import java.util.List;
 public class CodeBlock extends SrcOp {
 	public List<SrcOp> ops;
 
+	public CodeBlock(CodeGenerator gen) { this.gen = gen; }
+	
+	public CodeBlock(CodeGenerator gen, List<SrcOp> ops) {
+		this.gen = gen;
+		this.ops = ops;
+	}
+
+	public CodeBlock(CodeGenerator gen, final SrcOp elem) {
+		this(gen, new ArrayList<SrcOp>() {{ add(elem); }});
+	}
+
 	@Override
 	public List<String> getChildren() {
 		return new ArrayList<String>() {{ add("ops"); }};

tool/src/org/antlr/v4/codegen/src/InvokeRule.java

@@ -1,5 +1,6 @@
 package org.antlr.v4.codegen.src;
 
+import org.antlr.v4.codegen.CodeGenerator;
 import org.antlr.v4.misc.IntervalSet;
 
 import java.util.List;
@@ -11,12 +12,14 @@ public class InvokeRule extends SrcOp {
 	public List<String> args;
 	public IntervalSet[] follow;
 
-	public InvokeRule(String name, String argAction, IntervalSet[] follow) {
+	public InvokeRule(CodeGenerator gen, String name, String argAction, IntervalSet[] follow) {
+		this.gen = gen;
 		// split and translate argAction
 		// compute follow
 	}
 
-	public InvokeRule(String name, IntervalSet[] follow) {
+	public InvokeRule(CodeGenerator gen, String name, IntervalSet[] follow) {
+		this.gen = gen;
 		// split and translate argAction
 		// compute follow
 	}

tool/src/org/antlr/v4/codegen/src/LL1Choice.java

@@ -0,0 +1,12 @@
+package org.antlr.v4.codegen.src;
+
+import org.antlr.v4.codegen.CodeGenerator;
+
+import java.util.List;
+
+/** */
+public class LL1Choice extends Choice {
+	public LL1Choice(CodeGenerator gen, List<CodeBlock> alts) {
+		super(gen, alts);
+	}
+}

tool/src/org/antlr/v4/codegen/src/LL1OptionalBlock.java

@@ -1,10 +1,12 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.tool.Alternative;
+import org.antlr.v4.codegen.CodeGenerator;
+
+import java.util.List;
 
 /** */
 public class LL1OptionalBlock extends OptionalBlock {
-	public LL1OptionalBlock(Alternative[] alts) {
-		super(alts);
+	public LL1OptionalBlock(CodeGenerator gen, List<CodeBlock> alts) {
+		super(gen, alts);
 	}
 }

tool/src/org/antlr/v4/codegen/src/LL1OptionalBlockSingleAlt.java

@@ -1,10 +1,12 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.tool.Alternative;
+import org.antlr.v4.codegen.CodeGenerator;
+
+import java.util.List;
 
 /** */
 public class LL1OptionalBlockSingleAlt extends OptionalBlock {
-	public LL1OptionalBlockSingleAlt(Alternative[] alts) {
-		super(alts);
+	public LL1OptionalBlockSingleAlt(CodeGenerator gen, List<CodeBlock> alts) {
+		super(gen, alts);
 	}
 }

tool/src/org/antlr/v4/codegen/src/LLStarOptionalBlock.java

@@ -1,10 +1,12 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.tool.Alternative;
+import org.antlr.v4.codegen.CodeGenerator;
+
+import java.util.List;
 
 /** */
 public class LLStarOptionalBlock extends OptionalBlock {
-	public LLStarOptionalBlock(Alternative[] alts) {
-		super(alts);
+	public LLStarOptionalBlock(CodeGenerator gen, List<CodeBlock> alts) {
+		super(gen, alts);
 	}
 }

tool/src/org/antlr/v4/codegen/src/LLkOptionalBlock.java

@@ -1,10 +1,12 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.tool.Alternative;
+import org.antlr.v4.codegen.CodeGenerator;
+
+import java.util.List;
 
 /** */
 public class LLkOptionalBlock extends OptionalBlock {
-	public LLkOptionalBlock(Alternative[] alts) {
-		super(alts);
+	public LLkOptionalBlock(CodeGenerator gen, List<CodeBlock> alts) {
+		super(gen, alts);
 	}
 }

tool/src/org/antlr/v4/codegen/src/MatchToken.java

@@ -1,9 +1,15 @@
 package org.antlr.v4.codegen.src;
 
+import org.antlr.v4.codegen.CodeGenerator;
 import org.antlr.v4.misc.IntervalSet;
+import org.antlr.v4.tool.TerminalAST;
 
 /** */
 public class MatchToken extends SrcOp {
 	public int ttype;
 	public IntervalSet[] follow;
+
+	public MatchToken(CodeGenerator gen, TerminalAST ast) {
+		this.gen = gen;
+	}
 }

tool/src/org/antlr/v4/codegen/src/OptionalBlock.java

@@ -1,13 +1,13 @@
 package org.antlr.v4.codegen.src;
 
-import org.antlr.v4.tool.Alternative;
+import org.antlr.v4.codegen.CodeGenerator;
 
 import java.util.List;
 
 /** */
 public class OptionalBlock extends Choice {
-	public OptionalBlock(Alternative[] alts) {
-		super(alts);
+	public OptionalBlock(CodeGenerator gen, List<CodeBlock> alts) {
+		super(gen, alts);
 	}
 
 	@Override

tool/src/org/antlr/v4/codegen/src/OutputModelObject.java

@@ -1,10 +1,12 @@
 package org.antlr.v4.codegen.src;
 
+import org.antlr.v4.codegen.CodeGenerator;
+
 import java.util.List;
 
 /** */
 public abstract class OutputModelObject {
-	//public ST st;
+	public CodeGenerator gen;
 
 	/** If the output model object encloses some other model objects,
 	 *  we need to be able to walk them. Rather than make each class

tool/src/org/antlr/v4/codegen/src/Parser.java

@@ -1,8 +1,8 @@
 package org.antlr.v4.codegen.src;
 
 import org.antlr.v4.automata.DFA;
+import org.antlr.v4.codegen.CodeGenerator;
 import org.antlr.v4.misc.IntSet;
-import org.antlr.v4.tool.Grammar;
 import org.antlr.v4.tool.Rule;
 
 import java.util.ArrayList;
@@ -15,12 +15,13 @@ public class Parser extends OutputModelObject {
 	public List<DFADef> dfaDefs = new ArrayList<DFADef>();
 	public List<IntSet> bitsetDefs;
 
-	public Parser(Grammar g) {		
-		name = g.getRecognizerName();
-		for (Rule r : g.rules.values()) funcs.add( new RuleFunction(r) );
+	public Parser(CodeGenerator gen) {
+		this.gen = gen;
+		name = gen.g.getRecognizerName();
+		for (Rule r : gen.g.rules.values()) funcs.add( new RuleFunction(gen, r) );
 
 		// build DFA, bitset defs
-		for (DFA dfa : g.decisionDFAs.values()) {
+		for (DFA dfa : gen.g.decisionDFAs.values()) {
 			dfaDefs.add( new DFADef("DFA"+dfa.decision, dfa) );
 		}
 	}

tool/src/org/antlr/v4/codegen/src/ParserFile.java

@@ -1,5 +1,7 @@
 package org.antlr.v4.codegen.src;
 
+import org.antlr.v4.codegen.CodeGenerator;
+
 import java.util.ArrayList;
 import java.util.List;
 
@@ -7,7 +9,12 @@ import java.util.List;
 public class ParserFile extends OutputModelObject {
 	public String fileName;
 	public Parser parser;
-	public ParserFile(Parser p, String fileName) { parser = p; this.fileName = fileName; }
+	
+	public ParserFile(CodeGenerator gen, Parser p, String fileName) {
+		this.gen = gen;
+		parser = p;
+		this.fileName = fileName; 
+	}
 
 	@Override
 	public List<String> getChildren() {

tool/src/org/antlr/v4/codegen/src/RuleFunction.java

@@ -1,6 +1,7 @@
 package org.antlr.v4.codegen.src;
 
 import org.antlr.runtime.tree.CommonTreeNodeStream;
+import org.antlr.v4.codegen.CodeGenerator;
 import org.antlr.v4.codegen.SourceGenTriggers;
 import org.antlr.v4.misc.Utils;
 import org.antlr.v4.parse.ANTLRParser;
@@ -29,7 +30,8 @@ public class RuleFunction extends OutputModelObject {
 
 	public CodeBlock code;
 
-	public RuleFunction(Rule r) {
+	public RuleFunction(CodeGenerator gen, Rule r) {
+		this.gen = gen;
 		this.name = r.name;
 		if ( r.modifiers!=null && r.modifiers.size()>0 ) {
 			this.modifiers = new ArrayList<String>();
@@ -48,9 +50,9 @@ public class RuleFunction extends OutputModelObject {
 		GrammarASTAdaptor adaptor = new GrammarASTAdaptor(r.ast.token.getInputStream());
 		GrammarAST blk = (GrammarAST)r.ast.getFirstChildWithType(ANTLRParser.BLOCK);
 		CommonTreeNodeStream nodes = new CommonTreeNodeStream(adaptor,blk);
-		SourceGenTriggers gen = new SourceGenTriggers(nodes);
+		SourceGenTriggers genTriggers = new SourceGenTriggers(nodes, gen);
 		try {
-			gen.block(); // GEN Instr OBJECTS
+			code = genTriggers.block(); // GEN Instr OBJECTS
 		}
 		catch (Exception e){
 			e.printStackTrace(System.err);

tool/src/org/antlr/v4/codegen/src/SrcOp.java

@@ -3,6 +3,6 @@ package org.antlr.v4.codegen.src;
 import org.antlr.v4.tool.GrammarAST;
 
 /** */
-public class SrcOp extends OutputModelObject {
+public abstract class SrcOp extends OutputModelObject {
 	public GrammarAST ast;	
 }

tool/src/org/antlr/v4/tool/GrammarAST.java

@@ -4,6 +4,7 @@ import org.antlr.runtime.BitSet;
 import org.antlr.runtime.CommonToken;
 import org.antlr.runtime.Token;
 import org.antlr.runtime.tree.Tree;
+import org.antlr.v4.automata.NFAState;
 import org.antlr.v4.parse.ANTLRParser;
 import org.antlr.v4.runtime.tree.CommonTree;
 
@@ -12,6 +13,9 @@ import java.util.LinkedList;
 import java.util.List;
 
 public class GrammarAST extends CommonTree {
+	/** If we build an NFA, we make AST node point at left edge of NFA construct */
+	public NFAState nfaState;
+
     public GrammarAST() {;}
     public GrammarAST(Token t) { super(t); }
     public GrammarAST(GrammarAST node) { super(node); }