refactored RuleContext to move all noncritical fields down into ParserRuleContext. Widespread but minor changes. Using the more specific ParserRuleContext where appropriate.

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9509]

runtime/Java/src/org/antlr/v4/runtime/BaseRecognizer.java

@@ -204,7 +204,7 @@ public abstract class BaseRecognizer<Symbol> extends Recognizer<Symbol, ParserAT
 		// if we have new localctx, make sure we replace existing ctx
 		// that is previous child of parse tree
 		if ( buildParseTrees && _ctx != localctx ) {
-			RuleContext parent = _ctx.parent;
+			ParserRuleContext parent = (ParserRuleContext)_ctx.parent;
 			parent.removeLastChild();
 			if ( parent!=null )	parent.addChild(localctx);
 		}
@@ -241,7 +241,7 @@ public abstract class BaseRecognizer<Symbol> extends Recognizer<Symbol, ParserAT
 	}
 
 	protected void addContextToParseTree() {
-		RuleContext parent = _ctx.parent;
+		ParserRuleContext parent = (ParserRuleContext)_ctx.parent;
 		// add current context to parent if we have a parent
 		if ( parent!=null )	{
 			parent.addChild(_ctx);
@@ -275,7 +275,7 @@ public abstract class BaseRecognizer<Symbol> extends Recognizer<Symbol, ParserAT
     public boolean isExpectedToken(int symbol) {
 //   		return getInterpreter().atn.nextTokens(_ctx);
         ATN atn = getInterpreter().atn;
-        RuleContext ctx = _ctx;
+		ParserRuleContext ctx = _ctx;
         ATNState s = atn.states.get(ctx.s);
         IntervalSet following = atn.nextTokens(s);
         if (following.contains(symbol)) {
@@ -292,7 +292,7 @@ public abstract class BaseRecognizer<Symbol> extends Recognizer<Symbol, ParserAT
                 return true;
             }
 
-            ctx = ctx.parent;
+            ctx = (ParserRuleContext)ctx.parent;
         }
 
         if ( following.contains(Token.EPSILON) && symbol == Token.EOF ) {
@@ -302,10 +302,12 @@ public abstract class BaseRecognizer<Symbol> extends Recognizer<Symbol, ParserAT
         return false;
     }
 
+	/** Compute the set of valid tokens reachable from the current
+	 *  position in the parse.
+	 */
     public IntervalSet getExpectedTokens() {
-//   		return getInterpreter().atn.nextTokens(_ctx);
         ATN atn = getInterpreter().atn;
-        RuleContext ctx = _ctx;
+		ParserRuleContext ctx = _ctx;
         ATNState s = atn.states.get(ctx.s);
         IntervalSet following = atn.nextTokens(s);
 //        System.out.println("following "+s+"="+following);
@@ -319,7 +321,7 @@ public abstract class BaseRecognizer<Symbol> extends Recognizer<Symbol, ParserAT
             following = atn.nextTokens(rt.followState);
             expected.addAll(following);
             expected.remove(Token.EPSILON);
-            ctx = ctx.parent;
+            ctx = (ParserRuleContext)ctx.parent;
         }
         if ( following.contains(Token.EPSILON) ) {
             expected.add(Token.EOF);
@@ -333,6 +335,16 @@ public abstract class BaseRecognizer<Symbol> extends Recognizer<Symbol, ParserAT
    		return atn.nextTokens(s);
    	}
 
+//	/** Compute the set of valid tokens reachable from the current
+//	 *  position in the parse.
+//	 */
+//	public IntervalSet nextTokens(@NotNull RuleContext ctx) {
+//		ATN atn = getInterpreter().atn;
+//		ATNState s = atn.states.get(ctx.s);
+//		if ( s == null ) return null;
+//		return atn.nextTokens(s, ctx);
+//	}
+
 	/** Return List<String> of the rule names in your parser instance
 	 *  leading up to a call to the current rule.  You could override if
 	 *  you want more details such as the file/line info of where

runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java

@@ -60,7 +60,7 @@ public class NoViableAltException extends RecognitionException {
 										 Token offendingToken,
 										 Symbol offendingNode,
 										 OrderedHashSet<ATNConfig> deadEndConfigs,
-										 RuleContext ctx)
+										 ParserRuleContext ctx)
 	{
 		super(recognizer, input, ctx);
 		this.deadEndConfigs = deadEndConfigs;

runtime/Java/src/org/antlr/v4/runtime/NoViableTreeGrammarAltException.java

@@ -31,7 +31,8 @@ package org.antlr.v4.runtime;
 
 import org.antlr.v4.runtime.atn.ATNConfig;
 import org.antlr.v4.runtime.misc.OrderedHashSet;
-import org.antlr.v4.runtime.tree.*;
+import org.antlr.v4.runtime.tree.ASTNodeStream;
+import org.antlr.v4.runtime.tree.TreeParser;
 
 public class NoViableTreeGrammarAltException extends NoViableAltException {
 	protected Object startNode;
@@ -50,7 +51,7 @@ public class NoViableTreeGrammarAltException extends NoViableAltException {
 													Symbol startNode,
 													Symbol offendingNode,
 													OrderedHashSet<ATNConfig> deadEndConfigs,
-													RuleContext ctx) {
+													ParserRuleContext ctx) {
 		super(recognizer, input,
 			  input.getTreeAdaptor().getToken(startNode),
 			  input.getTreeAdaptor().getToken(offendingNode),

runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java

@@ -28,18 +28,29 @@
  */
 package org.antlr.v4.runtime;
 
+import org.antlr.v4.runtime.atn.ATN;
+import org.antlr.v4.runtime.atn.ATNState;
+import org.antlr.v4.runtime.misc.NotNull;
+import org.antlr.v4.runtime.misc.Nullable;
+import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeListener;
 import org.stringtemplate.v4.ST;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-/** Rules return values in an object containing all the values.
+/** A rule invocation record for parsing and tree parsing.
- *  Besides the properties defined in
+ *
- *  RuleLabelScope.predefinedRulePropertiesScope there may be user-defined
+ *  Contains all of the information about the current rule not stored in the
- *  return values.  This class simply defines the minimum properties that
+ *  RuleContext. It handles parse tree children list, Any ATN state
- *  are always defined and methods to access the others that might be
+ *  tracing, and the default values available for rule indications:
- *  available depending on output option such as template and tree.
+ *  start, stop, ST, rule index, current alt number, current
+ *  ATN state.
+ *
+ *  Subclasses made for each rule and grammar track the parameters,
+ *  return values, locals, and labels specific to that rule. These
+ *  are the objects that are returned from rules.
  *
  *  Note text is not an actual property of the return value, it is computed
  *  from start and stop using the input stream's toString() method.  I
@@ -53,6 +64,43 @@ import java.util.List;
  *  satisfy the superclass interface.
  */
 public class ParserRuleContext<Symbol> extends RuleContext {
+	public static final ParserRuleContext EMPTY = new ParserRuleContext();
+
+	/** If we are debugging or building a parse tree for a visitor,
+	 *  we need to track all of the tokens and rule invocations associated
+	 *  with this rule's context. This is empty for normal parsing
+	 *  operation because we don't the need to track the details about
+	 *  how we parse this rule.
+	 */
+	public List<ParseTree> children;
+
+	/** For debugging/tracing purposes, we want to track all of the nodes in
+	 *  the ATN traversed by the parser for a particular rule.
+	 *  This list indicates the sequence of ATN nodes used to match
+	 *  the elements of the children list. This list does not include
+	 *  ATN nodes and other rules used to match rule invocations. It
+	 *  traces the rule invocation node itself but nothing inside that
+	 *  other rule's ATN submachine.
+	 *
+	 *  There is NOT a one-to-one correspondence between the children and
+	 *  states list. There are typically many nodes in the ATN traversed
+	 *  for each element in the children list. For example, for a rule
+	 *  invocation there is the invoking state and the following state.
+	 *
+	 *  The parser setState() method updates field s and adds it to this list
+	 *  if we are debugging/tracing.
+     *
+     *  This does not trace states visited during prediction.
+	 */
+	public List<Integer> states;
+
+	/** Current ATN state number we are executing.
+	 *
+	 *  Not used during ATN simulation/prediction; only used during parse that updates
+	 *  current location in ATN.
+	 */
+	public int s = -1;
+
 	public Symbol start, stop;
 	public ST st;
 
@@ -64,8 +112,7 @@ public class ParserRuleContext<Symbol> extends RuleContext {
 
 	public ParserRuleContext() { }
 
-	/** COPY a ctx
+	/** COPY a ctx (it deliberately not using copy constructor) */
-	 */
 	public void copyFrom(ParserRuleContext<Symbol> ctx) {
 		// from RuleContext
 		this.parent = ctx.parent;
@@ -78,8 +125,13 @@ public class ParserRuleContext<Symbol> extends RuleContext {
 		this.ruleIndex = ctx.ruleIndex;
 	}
 
-	public ParserRuleContext(RuleContext parent, int stateNumber) {
+	public ParserRuleContext(@Nullable ParserRuleContext parent, int invokingStateNumber, int stateNumber) {
-		super(parent, stateNumber);
+		super(parent, invokingStateNumber);
+		this.s = stateNumber;
+	}
+
+	public ParserRuleContext(@Nullable ParserRuleContext parent, int stateNumber) {
+		this(parent, parent!=null ? parent.s : -1 /* invoking state */, stateNumber);
 	}
 
 //	@Override
@@ -96,6 +148,49 @@ public class ParserRuleContext<Symbol> extends RuleContext {
 	public void enterRule(ParseTreeListener<Symbol> listener) { }
 	public void exitRule(ParseTreeListener<Symbol> listener) { }
 
+	public void addChild(TerminalNode<?> t) {
+		if ( children==null ) children = new ArrayList<ParseTree>();
+		children.add(t);
+	}
+
+	public void addChild(RuleContext ruleInvocation) {
+		if ( children==null ) children = new ArrayList<ParseTree>();
+		children.add(ruleInvocation);
+	}
+
+	/** Used by enterOuterAlt to toss out a RuleContext previously added as
+	 *  we entered a rule. If we have # label, we will need to remove
+	 *  generic ruleContext object.
+ 	 */
+	public void removeLastChild() {
+		if ( children!=null ) {
+			children.remove(children.size()-1);
+		}
+	}
+
+	public void trace(int s) {
+		if ( states==null ) states = new ArrayList<Integer>();
+		states.add(s);
+	}
+
+	public void addChild(Token matchedToken) {
+		TerminalNodeImpl<?> t = new TerminalNodeImpl<Token>(matchedToken);
+		addChild(t);
+	}
+
+	public void addErrorNode(Token badToken) {
+		TerminalNodeImpl<?> t = new ErrorNodeImpl<Token>(badToken);
+		addChild(t);
+	}
+
+	@Override
+	public ParseTree getChild(int i) {
+		return children!=null ? children.get(i) : null;
+	}
+
+	@Override
+	public int getChildCount() { return children!=null ? children.size() : 0; }
+
 	@Override
 	public int getRuleIndex() { return ruleIndex; }
 
@@ -103,6 +198,27 @@ public class ParserRuleContext<Symbol> extends RuleContext {
 	public Symbol getStart() { return start; }
 	public Symbol getStop() { return stop; }
 
+	@Override
+	public String toString(@NotNull Recognizer<?,?> recog, RuleContext stop) {
+		if ( recog==null ) return super.toString(recog, stop);
+		StringBuilder buf = new StringBuilder();
+		ParserRuleContext p = this;
+		buf.append("[");
+		while ( p != null && p != stop ) {
+			ATN atn = recog.getATN();
+			ATNState s = atn.states.get(p.s);
+			String ruleName = recog.getRuleNames()[s.ruleIndex];
+			buf.append(ruleName);
+			if ( p.parent != null ) buf.append(" ");
+//				ATNState invoker = atn.states.get(ctx.invokingState);
+//				RuleTransition rt = (RuleTransition)invoker.transition(0);
+//				buf.append(recog.getRuleNames()[rt.target.ruleIndex]);
+			p = (ParserRuleContext)p.parent;
+		}
+		buf.append("]");
+		return buf.toString();
+	}
+
     /** Used for rule context info debugging during runtime, not so much for ATN debugging */
     public String toInfoString(BaseRecognizer<Symbol> recognizer) {
         List<String> rules = recognizer.getRuleInvocationStack();

runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java

@@ -29,6 +29,7 @@
 package org.antlr.v4.runtime;
 
 import org.antlr.v4.runtime.misc.IntervalSet;
+import org.antlr.v4.runtime.misc.Nullable;
 
 /** The root of the ANTLR exception hierarchy. In general, ANTLR tracks just
  *  3 kinds of errors: prediction errors, failed predicate errors, and
@@ -64,8 +65,8 @@ public class RecognitionException extends RuntimeException {
 
 	protected int offendingState;
 
-	public RecognitionException(Recognizer<?, ?> recognizer, IntStream input,
+	public RecognitionException(@Nullable Recognizer<?, ?> recognizer, IntStream input,
-								RuleContext ctx)
+								@Nullable ParserRuleContext ctx)
 	{
 		this.recognizer = recognizer;
 		this.input = input;
@@ -84,7 +85,7 @@ public class RecognitionException extends RuntimeException {
 	public IntervalSet getExpectedTokens() {
         // TODO: do we really need this type check?
 		if ( recognizer!=null && recognizer instanceof BaseRecognizer<?> ) {
-			return recognizer.getInterpreter().atn.nextTokens(ctx);
+			return ((BaseRecognizer) recognizer).getExpectedTokens();
 		}
 		return null;
 	}

runtime/Java/src/org/antlr/v4/runtime/RuleContext.java

@@ -28,68 +28,39 @@
  */
 package org.antlr.v4.runtime;
 
-import org.antlr.v4.runtime.atn.ATN;
-import org.antlr.v4.runtime.atn.ATNState;
 import org.antlr.v4.runtime.misc.Interval;
-import org.antlr.v4.runtime.misc.NotNull;
+import org.antlr.v4.runtime.misc.Nullable;
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.Trees;
 import org.antlr.v4.runtime.tree.gui.TreeViewer;
 
 import javax.print.PrintException;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
 
-/** Rules can return start/stop info as well as possible trees and templates.
+/** A rule context is a record of a single rule invocation. It knows
- *  Each context knows about invoking context and pointer into ATN so we
+ *  which context invoked it, if any. If there is no parent context, then
- *  can compute FOLLOW for errors and lookahead.
+ *  naturally the invoking state is not valid.  The parent link
+ *  provides a chain upwards from the current rule invocation to the root
+ *  of the invocation tree, forming a stack. We actually carry no
+ *  information about the rule associated with this context (except
+ *  when parsing). We keep only the state number of the invoking state from
+ *  the ATN submachine that invoked this. Contrast this with the s
+ *  pointer inside ParserRuleContext that tracks the current state
+ *  being "executed" for the current rule.
  *
- *  Used during parse to record stack of rule invocations and during
+ *  The parent contexts are useful for computing lookahead sets and
- *  ATN simulation to record invoking states.
+ *  getting error information.
+ *
+ *  These objects are used during lexing, parsing, and prediction.
+ *  For the special case of parsers and tree parsers, we use the subclass
+ *  ParserRuleContext.
+ *
+ *  @see ParserRuleContext
  */
 public class RuleContext implements ParseTree.RuleNode {
-	@NotNull
-	public static final RuleContext EMPTY = new RuleContext();
-
 	/** What context invoked this rule? */
 	public RuleContext parent;
 
-	/** If we are debugging or building a parse tree for a visitor,
-	 *  we need to track all of the tokens and rule invocations associated
-	 *  with this rule's context. This is empty for normal parsing
-	 *  operation because we don't the need to track the details about
-	 *  how we parse this rule.
-	 */
-	public List<ParseTree> children;
-
-	/** For debugging/tracing purposes, we want to track all of the nodes in
-	 *  the ATN traversed by the parser for a particular rule.
-	 *  This list indicates the sequence of ATN nodes used to match
-	 *  the elements of the children list. This list does not include
-	 *  ATN nodes and other rules used to match rule invocations. It
-	 *  traces the rule invocation node itself but nothing inside that
-	 *  other rule's ATN submachine.
-	 *
-	 *  There is NOT a one-to-one correspondence between the children and
-	 *  states list. There are typically many nodes in the ATN traversed
-	 *  for each element in the children list. For example, for a rule
-	 *  invocation there is the invoking state and the following state.
-	 *
-	 *  The parser setState() method updates field s and adds it to this list
-	 *  if we are debugging/tracing.
-     *
-     *  This does not trace states visited during prediction.
-	 */
-	public List<Integer> states;
-
-	/** Current ATN state number we are executing.
-	 *
-	 *  Not used during ATN simulation; only used during parse that updates
-	 *  current location in ATN.
-	 */
-	public int s = -1;
-
 	/** What state invoked the rule associated with this context?
 	 *  The "return address" is the followState of invokingState
 	 *  If parent is null, this should be -1.
@@ -109,20 +80,19 @@ public class RuleContext implements ParseTree.RuleNode {
 
 	public RuleContext() {}
 
-	public RuleContext(RuleContext parent) {
+//	public RuleContext(RuleContext parent) {
-		this(parent, -1);
+//		this(parent, -1);
-	}
+//	}
+//
+//	public RuleContext(RuleContext parent, int stateNumber) {
+//		// capture state that called us as we create this context; use later for
+//		// return state in closure
+//		this(parent, parent != null ? parent.s : -1, stateNumber);
+//	}
 
-	public RuleContext(RuleContext parent, int stateNumber) {
+	public RuleContext(RuleContext parent, int invokingState) {
-		// capture state that called us as we create this context; use later for
-		// return state in closure
-		this(parent, parent!=null ? parent.s : -1, stateNumber);
-	}
-
-	public RuleContext(RuleContext parent, int invokingState, int stateNumber) {
 		this.parent = parent;
 		//if ( parent!=null ) System.out.println("invoke "+stateNumber+" from "+parent);
-		this.s = stateNumber;
 		this.invokingState = invokingState;
 
 		this.cachedHashCode = invokingState;
@@ -153,41 +123,6 @@ public class RuleContext implements ParseTree.RuleNode {
 		return n;
 	}
 
-	public void addChild(Token matchedToken) {
-		TerminalNodeImpl<?> t = new TerminalNodeImpl<Token>(matchedToken);
-		addChild(t);
-	}
-
-	public void addErrorNode(Token badToken) {
-		TerminalNodeImpl<?> t = new ErrorNodeImpl<Token>(badToken);
-		addChild(t);
-	}
-
-	public void addChild(TerminalNode<?> t) {
-		if ( children==null ) children = new ArrayList<ParseTree>();
-		children.add(t);
-	}
-
-	public void addChild(RuleContext ruleInvocation) {
-		if ( children==null ) children = new ArrayList<ParseTree>();
-		children.add(ruleInvocation);
-	}
-
-	/** Used by enterOuterAlt to toss out a RuleContext previously added as
-	 *  we entered a rule. If we have # label, we will need to remove
-	 *  generic ruleContext object.
- 	 */
-	public void removeLastChild() {
-		if ( children!=null ) {
-			children.remove(children.size()-1);
-		}
-	}
-
-	public void trace(int s) {
-		if ( states==null ) states = new ArrayList<Integer>();
-		states.add(s);
-	}
-
 	/** Two contexts are equals() if both have
 	 *  same call stack; walk upwards to the root.
 	 *  Note that you may be comparing contexts in different alt trees.
@@ -290,11 +225,6 @@ public class RuleContext implements ParseTree.RuleNode {
 
 	// satisfy the ParseTree interface
 
-	@Override
-	public ParseTree getChild(int i) {
-		return children!=null ? children.get(i) : null;
-	}
-
 	@Override
 	public RuleContext getRuleContext() { return this; }
 
@@ -304,11 +234,18 @@ public class RuleContext implements ParseTree.RuleNode {
 	@Override
 	public RuleContext getPayload() { return this; }
 
-	@Override
-	public int getChildCount() { return children!=null ? children.size() : 0; }
-
 	public int getRuleIndex() { return -1; }
 
+	@Override
+	public ParseTree getChild(int i) {
+		return null;
+	}
+
+	@Override
+	public int getChildCount() {
+		return 0;
+	}
+
 	@Override
 	public Interval getSourceInterval() {
 		if ( getChildCount()==0 ) return Interval.INVALID;
@@ -351,29 +288,18 @@ public class RuleContext implements ParseTree.RuleNode {
 		return toString(null);
 	}
 
-	public String toString(BaseRecognizer<?> recog) {
+	public String toString(@Nullable Recognizer<?,?> recog) {
-		return toString(recog, RuleContext.EMPTY);
+		return toString(recog, ParserRuleContext.EMPTY);
 	}
 
-	public String toString(BaseRecognizer<?> recog, RuleContext stop) {
+	// recog null unless ParserRuleContext, in which case we use subclass toString(...)
+	public String toString(@Nullable Recognizer<?,?> recog, RuleContext stop) {
 		StringBuilder buf = new StringBuilder();
 		RuleContext p = this;
 		buf.append("[");
 		while ( p != null && p != stop ) {
-			if ( recog!=null ) {
-				ATN atn = recog.getATN();
-				ATNState s = atn.states.get(p.s);
-				String ruleName = recog.getRuleNames()[s.ruleIndex];
-				buf.append(ruleName);
-				if ( p.parent != null ) buf.append(" ");
-//				ATNState invoker = atn.states.get(ctx.invokingState);
-//				RuleTransition rt = (RuleTransition)invoker.transition(0);
-//				buf.append(recog.getRuleNames()[rt.target.ruleIndex]);
-			}
-			else {
 			if ( !p.isEmpty() ) buf.append(p.invokingState);
 			if ( p.parent != null && !p.parent.isEmpty() ) buf.append(" ");
-			}
 			p = p.parent;
 		}
 		buf.append("]");

runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java

@@ -79,15 +79,6 @@ public class ATN {
 	/** Used for runtime deserialization of ATNs from strings */
 	public ATN() { }
 
-	/** Compute the set of valid tokens reachable from the current
-	 *  position in the parse.
-	 */
-	public IntervalSet nextTokens(@NotNull RuleContext ctx) {
-		ATNState s = states.get(ctx.s);
-		if ( s == null ) return null;
-		return nextTokens(s, ctx);
-	}
-
 	/** Compute the set of valid tokens that can occur starting in s.
 	 *  If ctx is null, the set of tokens will not include what can follow
 	 *  the rule surrounding s. In other words, the set will be

runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java

@@ -172,7 +172,7 @@ public class ATNConfig {
         }
         if ( context!=null ) {
             buf.append("|");
-            buf.append(context);
+            buf.append(context.toString(recog));
         }
         if ( semanticContext!=null && semanticContext != SemanticContext.NONE ) {
             buf.append("|");

runtime/Java/src/org/antlr/v4/runtime/atn/LL1Analyzer.java

@@ -29,6 +29,7 @@
 
 package org.antlr.v4.runtime.atn;
 
+import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.RuleContext;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.misc.IntervalSet;
@@ -62,7 +63,7 @@ public class LL1Analyzer {
 			Set<ATNConfig> lookBusy = new HashSet<ATNConfig>();
 			boolean seeThruPreds = false; // fail to get lookahead upon pred
 			_LOOK(s.transition(alt - 1).target,
-				  RuleContext.EMPTY,
+				  ParserRuleContext.EMPTY,
 				  look[alt], lookBusy, seeThruPreds);
 			if ( look[alt].size()==0 ) look[alt] = null;
 		}
@@ -116,7 +117,7 @@ public class LL1Analyzer {
             Transition t = s.transition(i);
             if ( t.getClass() == RuleTransition.class ) {
                 RuleContext newContext =
-                    new RuleContext(ctx, s.stateNumber,  t.target.stateNumber);
+                    new RuleContext(ctx, s.stateNumber);
                 _LOOK(t.target, newContext, look, lookBusy, seeThruPreds);
             }
             else if ( t.isEpsilon() && seeThruPreds ) {

runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java

@@ -38,11 +38,11 @@ import org.antlr.v4.runtime.misc.OrderedHashSet;
 
 import java.io.IOException;
 import java.io.OutputStream;
-import org.antlr.v4.runtime.misc.Interval;
-import org.antlr.v4.runtime.misc.IntervalSet;
 
 /** "dup" of ParserInterpreter */
 public class LexerATNSimulator extends ATNSimulator {
+	public static final RuleContext EMPTY_LEXER_RULE_CONTEXT = new RuleContext();
+
 	public static boolean debug = false;
 	public static boolean dfa_debug = false;
 	public static final int NUM_EDGES = 255;
@@ -456,7 +456,7 @@ public class LexerATNSimulator extends ATNSimulator {
 	protected OrderedHashSet<ATNConfig> computeStartState(@NotNull IntStream input,
 														  @NotNull ATNState p)
 	{
-		RuleContext initialContext = RuleContext.EMPTY;
+		RuleContext initialContext = EMPTY_LEXER_RULE_CONTEXT;
 		OrderedHashSet<ATNConfig> configs = new OrderedHashSet<ATNConfig>();
 		for (int i=0; i<p.getNumberOfTransitions(); i++) {
 			ATNState target = p.transition(i).target;
@@ -515,7 +515,7 @@ public class LexerATNSimulator extends ATNSimulator {
 		ATNConfig c = null;
 		if ( t.getClass() == RuleTransition.class ) {
 			RuleContext newContext =
-				new RuleContext(config.context, p.stateNumber, t.target.stateNumber);
+				new RuleContext(config.context, p.stateNumber);
 			c = new ATNConfig(config, t.target, newContext);
 		}
 		else if ( t.getClass() == PredicateTransition.class ) {

runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java

@@ -43,7 +43,7 @@ import org.stringtemplate.v4.misc.MultiMap;
 import java.util.*;
 
 public class ParserATNSimulator<Symbol> extends ATNSimulator {
-	public static boolean debug = true;
+	public static boolean debug = false;
 	public static boolean dfa_debug = false;
 
 	public static int ATN_failover = 0;
@@ -73,7 +73,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 	 *  The full stack at any moment is [config.outerContext + config.context].
 	 */
 	@NotNull
-	protected RuleContext outerContext = RuleContext.EMPTY;
+	protected ParserRuleContext outerContext = ParserRuleContext.EMPTY;
 	@Nullable
 	protected ATNConfig prevAccept; // TODO Move down? used to avoid passing int down and back up in method calls
 	protected int prevAcceptIndex = -1;
@@ -93,7 +93,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 //		System.out.println(dot.getDOT(atn.rules.get(1), parser.getRuleNames()));
 	}
 
-	public int adaptivePredict(@NotNull SymbolStream<Symbol> input, int decision, @Nullable RuleContext outerContext) {
+	public int adaptivePredict(@NotNull SymbolStream<Symbol> input, int decision,
+							   @Nullable ParserRuleContext outerContext)
+	{
 		predict_calls++;
 		DFA dfa = decisionToDFA[decision];
 		if ( !buildDFA || dfa==null || dfa.s0==null ) {
@@ -120,21 +122,21 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 
 	public void reset() {
 		userWantsCtxSensitive = false;
-		outerContext = RuleContext.EMPTY;
+		outerContext = ParserRuleContext.EMPTY;
 		prevAccept = null;
 		prevAcceptIndex = -1;
 	}
 
 	public int predictATN(@NotNull DFA dfa, @NotNull SymbolStream<Symbol> input,
-						  @Nullable RuleContext outerContext,
+						  @Nullable ParserRuleContext outerContext,
 						  boolean useContext)
 	{
-		if ( outerContext==null ) outerContext = RuleContext.EMPTY;
+		if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY;
 		this.outerContext = outerContext;
 		if ( debug ) System.out.println("ATN decision "+dfa.decision+
 										" exec LA(1)=="+ getLookaheadName(input) +
 										", outerContext="+outerContext.toString(parser));
-		RuleContext ctx = RuleContext.EMPTY;
+		RuleContext ctx = ParserRuleContext.EMPTY;
 		if ( useContext ) ctx = outerContext;
 		OrderedHashSet<ATNConfig> s0_closure =
 			computeStartState(dfa.decision, dfa.atnStartState, ctx);
@@ -167,7 +169,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 						@NotNull ATNState startState)
 	{
 		DFA dfa = new DFA(startState);
-		RuleContext ctx = RuleContext.EMPTY;
+		RuleContext ctx = ParserRuleContext.EMPTY;
 		OrderedHashSet<ATNConfig> s0_closure =
 			computeStartState(dfa.decision, startState, ctx);
 		return execATN(input, dfa, input.index(), s0_closure, false);
@@ -176,10 +178,10 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 	public int execDFA(@NotNull SymbolStream<Symbol> input, @NotNull DFA dfa,
 					   int startIndex,
 					   @NotNull DFAState s0,
-                       @Nullable RuleContext outerContext)
+                       @Nullable ParserRuleContext outerContext)
     {
 //		dump(dfa);
-		if ( outerContext==null ) outerContext = RuleContext.EMPTY;
+		if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY;
 		this.outerContext = outerContext;
 		if ( dfa_debug ) System.out.println("DFA decision "+dfa.decision+
 											" exec LA(1)=="+ getLookaheadName(input) +
@@ -315,9 +317,10 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 		}
 
 		@NotNull DecisionState decState = null;
-		//if ( atn.decisionToState.size()>0 )
+		if ( atn.decisionToState.size()>0 ) {
         	decState = atn.decisionToState.get(dfa.decision);
 			if ( debug ) System.out.println("decision state = "+decState);
+		}
 
 		prevAccept = null;
 		prevAcceptIndex = -1;
@@ -390,7 +393,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 //								   input.toString(startIndex, input.index()));
 				if ( !userWantsCtxSensitive || useContext ) {
 					// resolve ambiguity
-					if ( decState.isGreedy ) {
+					if ( decState!=null && decState.isGreedy ) {
 						// if greedy, resolve in favor of alt coming first
 						resolveToMinAlt(reach, ambigAlts);
 					}
@@ -531,7 +534,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 	public int retryWithContext(@NotNull SymbolStream<Symbol> input,
 								@NotNull DFA dfa,
 								int startIndex,
-								@NotNull RuleContext originalContext,
+								@NotNull ParserRuleContext originalContext,
 								@NotNull OrderedHashSet<ATNConfig> closure,
 								int t,
 								@NotNull OrderedHashSet<ATNConfig> reach,
@@ -544,7 +547,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 		if ( debug ) System.out.println("RETRY "+ getInputString(input, startIndex) +
 										" with ctx="+ originalContext);
 		int min = getMinAlt(ambigAlts);
-		if ( originalContext==RuleContext.EMPTY ) {
+		if ( originalContext==ParserRuleContext.EMPTY ) {
 			// no point in retrying with ctx since it's same.
 			// this implies that we have a true ambiguity
 			reportAmbiguity(startIndex, input.index(), ambigAlts, reach);
@@ -652,7 +655,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
                            @NotNull SemanticContext semanticContext,
                            boolean collectPredicates)
 	{
-		if ( debug ) System.out.println("closure("+config+")");
+		if ( debug ) System.out.println("closure("+config.toString(parser,true)+")");
 
 		if ( !closureBusy.add(config) ) return; // avoid infinite recursion
 
@@ -738,7 +741,8 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 	}
 
 	@Nullable
-	public ATNConfig predTransition(@NotNull ATNConfig config, @NotNull PredicateTransition pt,
+	public ATNConfig predTransition(@NotNull ATNConfig config,
+									@NotNull PredicateTransition pt,
 									boolean collectPredicates)
 	{
 		if ( debug ) {
@@ -756,7 +760,7 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 		// when the context stack is empty and we've not dipped into
 		// the outer context.
 		boolean inContext =
-			config.context==RuleContext.EMPTY && config.reachesIntoOuterContext==0;
+			config.context==ParserRuleContext.EMPTY && config.reachesIntoOuterContext==0;
 //		RuleContext ctx = null;
 //		if ( inContext ) ctx = outerContext;
 
@@ -804,7 +808,9 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 		}
 		ATNState p = config.state;
 		RuleContext newContext =
-			new RuleContext(config.context, p.stateNumber,  t.target.stateNumber);
+			new ParserRuleContext((ParserRuleContext)config.context,
+								  p.stateNumber,
+								  t.target.stateNumber);
 		return new ATNConfig(config, t.target, newContext);
 	}
 
@@ -918,11 +924,12 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 		return ambigAlts;
 	}
 
-    public SemanticContext[] getPredsForAmbigAlts(@NotNull DecisionState decState,
+    public SemanticContext[] getPredsForAmbigAlts(@Nullable DecisionState decState,
                                                   @NotNull Set<Integer> ambigAlts,
                                                   @NotNull OrderedHashSet<ATNConfig> configs)
     {
         // REACH=[1|1|[]|0:0, 1|2|[]|0:1]
+		if ( decState==null ) return null;
         if ( debug ) System.out.println("getPredsForAmbigAlts decision "+decState.decision);
         int nalts = decState.getNumberOfTransitions();
         SemanticContext[] altToPred = new SemanticContext[nalts +1];
@@ -1093,8 +1100,10 @@ public class ParserATNSimulator<Symbol> extends ATNSimulator {
 	}
 
 	@NotNull
-	public NoViableAltException noViableAlt(@NotNull SymbolStream<Symbol> input, @NotNull RuleContext outerContext,
+	public NoViableAltException noViableAlt(@NotNull SymbolStream<Symbol> input,
-								@NotNull OrderedHashSet<ATNConfig> configs, int startIndex)
+											@NotNull ParserRuleContext outerContext,
+											@NotNull OrderedHashSet<ATNConfig> configs,
+											int startIndex)
 	{
 		if ( parser instanceof TreeParser) {
 			Symbol startNode = null;

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -558,8 +558,8 @@ CaptureNextTokenType(d) ::= "<d.varName> = _input.LA(1);"
 StructDecl(s,attrs,visitorDispatchMethods,interfaces,extensionMembers,superClass={ParserRuleContext\<<InputSymbolType()>>}) ::= <<
 public static class <s.name> extends <superClass><if(interfaces)> implements <interfaces; separator=", "><endif> {
 	<attrs:{a | public <a>;}; separator="\n">
-	<if(s.ctorAttrs)>public <s.name>(RuleContext parent, int state) { super(parent, state); }<endif>
+	<if(s.ctorAttrs)>public <s.name>(ParserRuleContext parent, int state) { super(parent, state); }<endif>
-	public <s.name>(RuleContext parent, int state<s.ctorAttrs:{a | , <a>}>) {
+	public <s.name>(ParserRuleContext parent, int state<s.ctorAttrs:{a | , <a>}>) {
 		super(parent, state);
 		<s.ctorAttrs:{a | this.<a.name> = <a.name>;}; separator="\n">
 	}

tool/src/org/antlr/v4/automata/TreeParserATNFactory.java

@@ -29,7 +29,7 @@
 
 package org.antlr.v4.automata;
 
-import org.antlr.v4.runtime.RuleContext;
+import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.atn.*;
 import org.antlr.v4.runtime.misc.IntervalSet;
@@ -58,9 +58,7 @@ public class TreeParserATNFactory extends ParserATNFactory {
 		for (int i=0; i<firstChildStates.size(); i++) {
 			ATNState firstChild = firstChildStates.get(i);
 			LL1Analyzer analyzer = new LL1Analyzer(atn);
-			IntervalSet look = analyzer.LOOK(firstChild, RuleContext.EMPTY);
+			IntervalSet look = analyzer.LOOK(firstChild, ParserRuleContext.EMPTY);
-			TreePatternAST root = treePatternRootNodes.get(i);
-
 			if ( look.contains(Token.UP) ) {
 				// nullable child list if we can see the UP as the next token.
 				// convert r DN kids UP to r (DN kids UP)?; leave AST alone--

tool/src/org/antlr/v4/codegen/model/MatchTree.java

@@ -31,7 +31,7 @@ package org.antlr.v4.codegen.model;
 
 import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.misc.Utils;
-import org.antlr.v4.runtime.RuleContext;
+import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.atn.ATNState;
 import org.antlr.v4.runtime.atn.LL1Analyzer;
@@ -80,7 +80,7 @@ public class MatchTree extends RuleElement {
 	boolean isNullable(TreePatternAST rootNode) {
 		ATNState firstChildState = rootNode.downState.transition(0).target;
 		LL1Analyzer analyzer = new LL1Analyzer(firstChildState.atn);
-		IntervalSet look = analyzer.LOOK(firstChildState, RuleContext.EMPTY);
+		IntervalSet look = analyzer.LOOK(firstChildState, ParserRuleContext.EMPTY);
 		return look.contains(Token.UP);
 	}
 

tool/test/org/antlr/v4/test/TestATNParserPrediction.java

@@ -1,11 +1,45 @@
+/*
+ [The "BSD license"]
+  Copyright (c) 2011 Terence Parr
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+
+  1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+  2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+  3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
 package org.antlr.v4.test;
 
 import org.antlr.v4.Tool;
 import org.antlr.v4.automata.ParserATNFactory;
-import org.antlr.v4.runtime.*;
+import org.antlr.v4.runtime.NoViableAltException;
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.TokenStream;
 import org.antlr.v4.runtime.atn.*;
 import org.antlr.v4.runtime.dfa.DFA;
-import org.antlr.v4.tool.*;
+import org.antlr.v4.tool.DOTGenerator;
+import org.antlr.v4.tool.Grammar;
+import org.antlr.v4.tool.LexerGrammar;
+import org.antlr.v4.tool.Rule;
 import org.junit.Test;
 
 import java.util.List;
@@ -280,10 +314,10 @@ public class TestATNParserPrediction extends BaseTest {
 		RuleStartState eStart = atn.ruleToStartState[g.getRule("e").index];
 		ATNState a_e_invoke = aStart.transition(0).target; //
 		ATNState b_e_invoke = bStart.transition(0).target; //
-		RuleContext a_ctx = new RuleContext(null, -1, a_e_invoke.stateNumber);
+		ParserRuleContext a_ctx = new ParserRuleContext(null, -1, a_e_invoke.stateNumber);
-		RuleContext b_ctx = new RuleContext(null, -1, b_e_invoke.stateNumber);
+		ParserRuleContext b_ctx = new ParserRuleContext(null, -1, b_e_invoke.stateNumber);
-		RuleContext a_e_ctx = new RuleContext(a_ctx, a_e_invoke.stateNumber, bStart.stateNumber);
+		ParserRuleContext a_e_ctx = new ParserRuleContext(a_ctx, a_e_invoke.stateNumber, bStart.stateNumber);
-		RuleContext b_e_ctx = new RuleContext(b_ctx, b_e_invoke.stateNumber, bStart.stateNumber);
+		ParserRuleContext b_e_ctx = new ParserRuleContext(b_ctx, b_e_invoke.stateNumber, bStart.stateNumber);
 
 		ParserATNSimulator interp = new ParserATNSimulator(atn);
 		interp.setContextSensitive(true);
@@ -516,7 +550,7 @@ public class TestATNParserPrediction extends BaseTest {
 		TokenStream input = new IntTokenStream(types);
 		ATNState startState = atn.decisionToState.get(decision);
 		DFA dfa = new DFA(startState);
-		int alt = interp.predictATN(dfa, input, RuleContext.EMPTY, false);
+		int alt = interp.predictATN(dfa, input, ParserRuleContext.EMPTY, false);
 
 		System.out.println(dot.getDOT(dfa, false));
 
@@ -533,7 +567,7 @@ public class TestATNParserPrediction extends BaseTest {
 	}
 
 	public DFA getDFA(LexerGrammar lg, Grammar g, String ruleName,
-					  String inputString, RuleContext ctx)
+					  String inputString, ParserRuleContext ctx)
 	{
 		Tool.internalOption_ShowATNConfigsInDFA = true;
 		ATN lexatn = createATN(lg);
@@ -589,7 +623,7 @@ public class TestATNParserPrediction extends BaseTest {
 			System.out.println(types);
 			TokenStream input = new IntTokenStream(types);
 			try {
-				interp.adaptivePredict(input, decision, RuleContext.EMPTY);
+				interp.adaptivePredict(input, decision, ParserRuleContext.EMPTY);
 			}
 			catch (NoViableAltException nvae) {
 				nvae.printStackTrace(System.err);