rule invoke follow was off

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 6886]

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -37,6 +37,7 @@ public class <parser.name> extends Parser {
 		TokenType(int type) { this.type = type; }
 	}
 !>	
+    public static final int EOF=-1;
     <parser.tokens.keys:{k | public static final int <k>=<parser.tokens.(k)>;}; separator="\n">
     <scopes>
     <namedActions.members>
@@ -246,10 +247,12 @@ SetDynScopeAttr_index(s, indexChunks, rhsChunks)    ::=
 AddToList(a) ::= "<a.listName>.add(<first(a.opWithResultToAdd.labels)>);"
 
 TokenDecl(t) ::= "Token <t.name>;"
+TokenTypeDecl(t) ::= "int <t.name>;"
 TokenListDecl(t) ::= "List\<Token> <t.name> = new ArrayList\<Token>();"
 RuleContextDecl(r) ::= "<r.ctxName> <r.name>;"
 
-CaptureNextToken(d) ::= "<d.varName> = state.input.LA(1);"
+CaptureNextToken(d) ::= "<d.varName> = state.input.LT(1);"
+CaptureNextTokenType(d) ::= "<d.varName> = state.input.LA(1);"
 
 StructDecl(s,attrs) ::= <<
 public static class <s.name> extends ParserRuleContext {
@@ -276,7 +279,7 @@ DFADecl(dfa) ::= <<
 >>
 
 BitSetDecl(b) ::= <<
-public static final LABitSet <b.name>=new LABitSet(new long[]{<b.fset.bits:{<it>L};separator=",">}<if(b.fset.EOF)>, true<endif>);
+public static final LABitSet <b.name>=new LABitSet(new long[]{<b.hexWords:{<it>L};separator=",">}<if(b.fset.EOF)>, true<endif>);
 >>
 
 LexerFile(fileName, lexer) ::= <<

tool/src/org/antlr/v4/codegen/Target.java

@@ -122,6 +122,24 @@ public class Target {
 		return buf.toString();
 	}
 
+	/** Convert long to 0xNNNNNNNNNNNNNNNN by default for spitting out
+	 *  with bitsets.  I.e., convert bytes to hex string.
+	 */
+	public String getTarget64BitStringFromValue(long word) {
+		int numHexDigits = 8*2;
+		StringBuffer buf = new StringBuffer(numHexDigits+2);
+		buf.append("0x");
+		String digits = Long.toHexString(word);
+		digits = digits.toUpperCase();
+		int padding = numHexDigits - digits.length();
+		// pad left with zeros
+		for (int i=1; i<=padding; i++) {
+			buf.append('0');
+		}
+		buf.append(digits);
+		return buf.toString();
+	}
+	
 	public String getLoopLabel(GrammarAST ast) {
 		return "loop"+ ast.token.getTokenIndex();
 	}

tool/src/org/antlr/v4/codegen/src/BitSetDecl.java

@@ -4,11 +4,22 @@ import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.misc.IntervalSet;
 import org.antlr.v4.runtime.misc.LABitSet;
 
+import java.util.ArrayList;
+import java.util.List;
+
 /** */
 public class BitSetDecl extends Decl {
 	public LABitSet fset; // runtime bitset
+	public List<String> hexWords;
 	public BitSetDecl(OutputModelFactory factory, String name, IntervalSet fset) {
 		super(factory, name);
 		this.fset = fset.toRuntimeBitSet();
+		long[] words = this.fset.bits;
+
+		hexWords = new ArrayList<String>();
+		for (long w : words) {
+			String h = factory.gen.target.getTarget64BitStringFromValue(w);
+			hexWords.add(h);
+		}
 	}
 }

tool/src/org/antlr/v4/codegen/src/CaptureNextTokenType.java

@@ -0,0 +1,7 @@
+package org.antlr.v4.codegen.src;
+
+/** */
+public class CaptureNextTokenType extends SrcOp {
+	public String varName;
+	public CaptureNextTokenType(String varName) { this.varName = varName; }	
+}

tool/src/org/antlr/v4/codegen/src/InvokeRule.java

@@ -2,6 +2,7 @@ package org.antlr.v4.codegen.src;
 
 import org.antlr.v4.analysis.LinearApproximator;
 import org.antlr.v4.automata.NFA;
+import org.antlr.v4.automata.RuleTransition;
 import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.codegen.RuleContextDecl;
 import org.antlr.v4.misc.IntervalSet;
@@ -49,8 +50,9 @@ public class InvokeRule extends SrcOp implements LabeledOp {
 		}
 
 		LinearApproximator approx = new LinearApproximator(factory.g, NFA.INVALID_DECISION_NUMBER);
-		IntervalSet fset = approx.LOOK(ast.nfaState.transition(0).target);
-		System.out.println("follow="+follow);
+		RuleTransition call = (RuleTransition)ast.nfaState.transition(0);
+		IntervalSet fset = approx.LOOK(call.followState);
+		System.out.println("follow rule ref "+name+"="+fset);
 		follow = factory.createFollowBitSet(ast, fset);
 		factory.defineBitSet(follow);
 	}

tool/src/org/antlr/v4/codegen/src/LL1Loop.java

@@ -27,11 +27,11 @@ public abstract class LL1Loop extends Choice {
 		expr = factory.getLL1Test(look, ast);
 		if ( expr instanceof TestSetInline ) {
 			TestSetInline e = (TestSetInline) expr;
-			Decl d = new TokenDecl(factory, e.varName);
+			Decl d = new TokenTypeDecl(factory, e.varName);
 			factory.currentRule.peek().addDecl(d);
-			CaptureNextToken nextToken = new CaptureNextToken(e.varName);
-			addPreambleOp(nextToken);
-			addIterationOp(nextToken);
+			CaptureNextTokenType nextType = new CaptureNextTokenType(e.varName);
+			addPreambleOp(nextType);
+			addIterationOp(nextType);
 		}
 	}
 

tool/src/org/antlr/v4/codegen/src/LL1OptionalBlockSingleAlt.java

@@ -21,8 +21,10 @@ public class LL1OptionalBlockSingleAlt extends LL1Choice {
 		expr = factory.getLL1Test(look, blkAST);
 		if ( expr instanceof TestSetInline ) {
 			TestSetInline e = (TestSetInline)expr;
-			CaptureNextToken nextToken = new CaptureNextToken(e.varName);
-			addPreambleOp(nextToken);
+			Decl d = new TokenTypeDecl(factory, e.varName);
+			factory.currentRule.peek().addDecl(d);
+			CaptureNextTokenType nextType = new CaptureNextTokenType(e.varName);
+			addPreambleOp(nextType);
 		}
 	}
 }

tool/src/org/antlr/v4/codegen/src/MatchToken.java

@@ -42,7 +42,7 @@ public class MatchToken extends SrcOp implements LabeledOp {
 
 		LinearApproximator approx = new LinearApproximator(factory.g, NFA.INVALID_DECISION_NUMBER);
 		IntervalSet fset = approx.LOOK(ast.nfaState.transition(0).target);
-		System.out.println("follow="+fset);
+		System.out.println("follow match "+name+"="+fset);
 		follow = factory.createFollowBitSet(ast, fset);
 		factory.defineBitSet(follow);
 	}

tool/src/org/antlr/v4/codegen/src/ParserFile.java

@@ -1,6 +1,7 @@
 package org.antlr.v4.codegen.src;
 
 import org.antlr.v4.codegen.OutputModelFactory;
+import org.antlr.v4.misc.OrderedHashSet;
 import org.antlr.v4.tool.GrammarAST;
 
 import java.util.ArrayList;
@@ -13,7 +14,7 @@ public class ParserFile extends OutputModelObject {
 	public String fileName;
 	public Parser parser;
 	public List<DFADecl> dfaDecls = new ArrayList<DFADecl>();
-	public List<BitSetDecl> bitSetDecls = new ArrayList<BitSetDecl>();
+	public OrderedHashSet<BitSetDecl> bitSetDecls = new OrderedHashSet<BitSetDecl>();
 	public String TokenLabelType;
 	public String ASTLabelType;
 	public Map<String, Action> namedActions;
@@ -35,16 +36,4 @@ public class ParserFile extends OutputModelObject {
 	public void defineBitSet(BitSetDecl b) {
 		bitSetDecls.add(b);
 	}
-
-//	@Override
-//	public List<String> getChildren() {
-//		final List<String> sup = super.getChildren();
-//		return new ArrayList<String>() {{
-//			if ( sup!=null ) addAll(sup);
-//			add("parser");
-//			add("dfaDecls");
-//			add("namedActions");
-//			add("bitSetDecls");
-//		}};
-//	}
 }

tool/src/org/antlr/v4/codegen/src/ThrowRecognitionException.java

@@ -14,7 +14,7 @@ public class ThrowRecognitionException extends SrcOp {
 
 	public ThrowRecognitionException(OutputModelFactory factory, GrammarAST ast, IntervalSet expecting) {
 		super(factory, ast);
-//		this.decision = ((BlockStartState)ast.nfaState).decision;
+		//this.decision = ((BlockStartState)ast.nfaState).decision;
 		grammarLine = ast.getLine();
 		grammarLine = ast.getCharPositionInLine();
 		grammarFile = factory.g.fileName;

tool/src/org/antlr/v4/codegen/src/TokenTypeDecl.java

@@ -0,0 +1,10 @@
+package org.antlr.v4.codegen.src;
+
+import org.antlr.v4.codegen.OutputModelFactory;
+
+/** */
+public class TokenTypeDecl extends Decl {
+	public TokenTypeDecl(OutputModelFactory factory, String name) {
+		super(factory, name);
+	}
+}

tool/src/org/antlr/v4/misc/Utils.java

@@ -119,14 +119,16 @@ public class Utils {
 		return x;
 	}
 
+	/** apply methodName to list and return list of results. method has
+	 *  no args.  This pulls data out of a list essentially.
+	 */
 	public static <From,To> List<To> apply(List<From> list, String methodName) {
 		if ( list==null ) return null;
 		List<To> b = new ArrayList<To>();
 		for (From f : list) {
 			try {
 				Method m = f.getClass().getMethod(methodName, (Class[])null);
-				To r = (To)m.invoke(f, (Object[])null);
-				b.add(r);
+				b.add( (To)m.invoke(f, (Object[])null) );
 			}
 			catch (Exception e) {
 				e.printStackTrace(System.err);
@@ -134,4 +136,5 @@ public class Utils {
 		}
 		return b;
 	}
+
 }