adding state stuff

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 6723]

tool/src/org/antlr/v4/Tool.java

@@ -338,8 +338,7 @@ public class Tool {
         Grammar g = new Grammar(this, ast);
         g.fileName = grammarFileNames.get(0);
         process(g);
-		if ( ast.grammarType==ANTLRParser.COMBINED ) {
+		if ( ast!=null && ast.grammarType==ANTLRParser.COMBINED && !ast.hasErrors ) {
-			// todo: don't process if errors in parser
 			lexerAST = extractImplicitLexer(g); // alters ast
             Grammar lexerg = new Grammar(this, lexerAST);
             lexerg.fileName = grammarFileNames.get(0);
@@ -354,15 +353,21 @@ public class Tool {
         g.loadImportedGrammars();
         if ( g.ast!=null && internalOption_PrintGrammarTree ) System.out.println(g.ast.toStringTree());
         //g.ast.inspect();
+
+		// MAKE SURE GRAMMAR IS SEMANTICALLY CORRECT (FILL IN GRAMMAR OBJECT)
         SemanticPipeline sem = new SemanticPipeline();
         sem.process(g);
-		
+		if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any)
-		// process imported grammars (if any)
-		if ( g.getImportedGrammars()!=null ) {
 			for (Grammar imp : g.getImportedGrammars()) {
 				process(imp);
 			}
 		}
+
+		// BUILD NFA FROM AST
+
+		// PERFORM GRAMMAR ANALYSIS ON NFA: BUILD DECISION DFAs
+
+		// GENERATE CODE
     }
 
     // TODO: Move to ast manipulation class?

tool/src/org/antlr/v4/analysis/ActionLabel.java

@@ -0,0 +1,25 @@
+package org.antlr.v4.analysis;
+
+import org.antlr.v4.tool.Grammar;
+import org.antlr.v4.tool.GrammarAST;
+
+/** */
+public class ActionLabel extends Label {
+	public GrammarAST actionAST;
+
+	public ActionLabel(GrammarAST actionAST) {
+		this.actionAST = actionAST;
+	}
+
+	public boolean isEpsilon() {
+		return true; // we are to be ignored by analysis 'cept for predicates
+	}
+
+	public String toString() {
+		return "{"+actionAST+"}";
+	}
+
+	public String toString(Grammar g) {
+		return toString();
+	}
+}

tool/src/org/antlr/v4/analysis/AtomLabel.java

@@ -0,0 +1,35 @@
+package org.antlr.v4.analysis;
+
+import org.antlr.misc.IntervalSet;
+
+/** */
+public class AtomLabel extends Label {
+	/** The token type or character value; or, signifies special label. */
+	protected int label;
+
+	public AtomLabel(int label) {
+		this.label = label;
+	}
+
+	public int hashCode() { return label; }
+
+	public boolean equals(Object o) {
+		if ( o==null ) return false;
+		if ( this == o ) return true; // equals if same object
+		if ( o.getClass() == SetLabel.class ) {
+			return IntervalSet.of(label).equals(o);
+		}
+		return label!=((AtomLabel)o).label;
+	}
+
+	public boolean intersect(Label other) {
+		if ( other.getClass() == AtomLabel.class ) {
+			return label==((AtomLabel)other).label;
+		}
+		return ((SetLabel)other).label.member(this.label);
+	}
+
+//	public int compareTo(Object o) {
+//		return this.label-((AtomLabel)o).label;
+//	}
+}

tool/src/org/antlr/v4/analysis/Label.java

@@ -2,7 +2,13 @@ package org.antlr.v4.analysis;
 
 import org.antlr.runtime.Token;
 
-public class Label implements Comparable, Cloneable {
+/** A state machine transition label.  A label can be either a simple
+ *  label such as a token or character.  A label can be a set of char or
+ *  tokens.  It can be an epsilon transition.  It can be a semantic predicate
+ *  (which assumes an epsilon transition) or a tree of predicates (in a DFA).
+ *  Special label types have to be < 0 to avoid conflict with char.
+ */
+public abstract class Label implements /*Comparable, */ Cloneable {
     public static final int INVALID = -7;
 
 	public static final int ACTION = -6;

tool/src/org/antlr/v4/analysis/PredicateLabel.java

@@ -0,0 +1,46 @@
+package org.antlr.v4.analysis;
+
+import org.antlr.v4.tool.Grammar;
+import org.antlr.v4.tool.GrammarAST;
+
+/** A tree of semantic predicates from the grammar AST if label==SEMPRED.
+ *  In the NFA, labels will always be exactly one predicate, but the DFA
+ *  may have to combine a bunch of them as it collects predicates from
+ *  multiple NFA configurations into a single DFA state.
+ */
+public class PredicateLabel extends Label {
+	protected SemanticContext semanticContext;
+
+	public PredicateLabel(GrammarAST predicateASTNode) {
+		this.semanticContext = new SemanticContext.Predicate(predicateASTNode);
+	}
+
+	public PredicateLabel(SemanticContext semCtx) {
+		this.semanticContext = semCtx;
+	}
+
+	public int hashCode() {
+		return semanticContext.hashCode();
+	}
+
+	public boolean equals(Object o) {
+		if ( o==null ) {
+			return false;
+		}
+		if ( this == o ) {
+			return true; // equals if same object
+		}
+		if ( !(o instanceof PredicateLabel) ) {
+			return false;
+		}
+		return semanticContext.equals(((PredicateLabel)o).semanticContext);
+	}
+
+	public String toString() {
+		return "{"+semanticContext+"}?";
+	}
+
+	public String toString(Grammar g) {
+		return toString();
+	}	
+}

tool/src/org/antlr/v4/analysis/SemanticContext.java

@@ -0,0 +1,308 @@
+package org.antlr.v4.analysis;
+
+import org.antlr.v4.parse.ANTLRParser;
+import org.antlr.v4.tool.GrammarAST;
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+/** A binary tree structure used to record the semantic context in which
+ *  an NFA configuration is valid.  It's either a single predicate or
+ *  a tree representing an operation tree such as: p1&&p2 or p1||p2.
+ *
+ *  For NFA o-p1->o-p2->o, create tree AND(p1,p2).
+ *  For NFA (1)-p1->(2)
+ *           |       ^
+ *           |       |
+ *          (3)-p2----
+ *  we will have to combine p1 and p2 into DFA state as we will be
+ *  adding NFA configurations for state 2 with two predicates p1,p2.
+ *  So, set context for combined NFA config for state 2: OR(p1,p2).
+ *
+ *  I have scoped the AND, NOT, OR, and Predicate subclasses of
+ *  SemanticContext within the scope of this outer class.
+ *
+ *  July 7, 2006: TJP altered OR to be set of operands. the Binary tree
+ *  made it really hard to reduce complicated || sequences to their minimum.
+ *  Got huge repeated || conditions.
+ */
+public abstract class SemanticContext {
+	/** Create a default value for the semantic context shared among all
+	 *  NFAConfigurations that do not have an actual semantic context.
+	 *  This prevents lots of if!=null type checks all over; it represents
+	 *  just an empty set of predicates.
+	 */
+	public static final SemanticContext EMPTY_SEMANTIC_CONTEXT = new Predicate();
+
+	/** Given a semantic context expression tree, return a tree with all
+	 *  nongated predicates set to true and then reduced.  So p&&(q||r) would
+	 *  return p&&r if q is nongated but p and r are gated.
+	 */
+	public abstract SemanticContext getGatedPredicateContext();
+
+	public abstract boolean isSyntacticPredicate();
+
+	public static class Predicate extends SemanticContext {
+		/** The AST node in tree created from the grammar holding the predicate */
+		public GrammarAST predicateAST;
+
+		/** Is this a {...}?=> gating predicate or a normal disambiguating {..}?
+		 *  If any predicate in expression is gated, then expression is considered
+		 *  gated.
+		 *
+		 *  The simple Predicate object's predicate AST's type is used to set
+		 *  gated to true if type==GATED_SEMPRED.
+		 */
+		protected boolean gated = false;
+
+		/** syntactic predicates are converted to semantic predicates
+		 *  but synpreds are generated slightly differently.
+		 */
+		protected boolean synpred = false;
+
+		public static final int INVALID_PRED_VALUE = -1;
+		public static final int FALSE_PRED = 0;
+		public static final int TRUE_PRED = 1;
+
+		/** sometimes predicates are known to be true or false; we need
+		 *  a way to represent this without resorting to a target language
+		 *  value like true or TRUE.
+		 */
+		protected int constantValue = INVALID_PRED_VALUE;
+
+		public Predicate() {
+			predicateAST = new GrammarAST();
+			this.gated=false;
+		}
+
+		public Predicate(GrammarAST predicate) {
+			this.predicateAST = predicate;
+			this.gated =
+				predicate.getType()== ANTLRParser.GATED_SEMPRED ||
+				predicate.getType()==ANTLRParser.SYN_SEMPRED ;
+			this.synpred =
+				predicate.getType()==ANTLRParser.SYN_SEMPRED ||
+				predicate.getType()== ANTLRParser.BACKTRACK_SEMPRED;
+		}
+
+		public Predicate(Predicate p) {
+			this.predicateAST = p.predicateAST;
+			this.gated = p.gated;
+			this.synpred = p.synpred;
+			this.constantValue = p.constantValue;
+		}
+
+		/** Two predicates are the same if they are literally the same
+		 *  text rather than same node in the grammar's AST.
+		 *  Or, if they have the same constant value, return equal.
+		 *  As of July 2006 I'm not sure these are needed.
+		 */
+		public boolean equals(Object o) {
+			if ( !(o instanceof Predicate) ) {
+				return false;
+			}
+			return predicateAST.getText().equals(((Predicate)o).predicateAST.getText());
+		}
+
+		public int hashCode() {
+			if ( predicateAST ==null ) {
+				return 0;
+			}
+			return predicateAST.getText().hashCode();
+		}
+
+		public SemanticContext getGatedPredicateContext() {
+			if ( gated ) {
+				return this;
+			}
+			return null;
+		}
+
+		public boolean isSyntacticPredicate() {
+			return predicateAST !=null &&
+				( predicateAST.getType()==ANTLRParser.SYN_SEMPRED ||
+				  predicateAST.getType()==ANTLRParser.BACKTRACK_SEMPRED );
+		}
+
+		public String toString() {
+			if ( predicateAST ==null ) {
+				return "<nopred>";
+			}
+			return predicateAST.getText();
+		}
+	}
+
+	public static class TruePredicate extends Predicate {
+		public TruePredicate() {
+			super();
+			this.constantValue = TRUE_PRED;
+		}
+
+		public String toString() {
+			return "true"; // not used for code gen, just DOT and print outs
+		}
+	}
+
+	public static class AND extends SemanticContext {
+		protected SemanticContext left,right;
+		public AND(SemanticContext a, SemanticContext b) {
+			this.left = a;
+			this.right = b;
+		}
+		public SemanticContext getGatedPredicateContext() {
+			SemanticContext gatedLeft = left.getGatedPredicateContext();
+			SemanticContext gatedRight = right.getGatedPredicateContext();
+			if ( gatedLeft==null ) {
+				return gatedRight;
+			}
+			if ( gatedRight==null ) {
+				return gatedLeft;
+			}
+			return new AND(gatedLeft, gatedRight);
+		}
+		public boolean isSyntacticPredicate() {
+			return left.isSyntacticPredicate()||right.isSyntacticPredicate();
+		}
+		public String toString() {
+			return "("+left+"&&"+right+")";
+		}
+	}
+
+	public static class OR extends SemanticContext {
+		protected Set operands;
+		public OR(SemanticContext a, SemanticContext b) {
+			operands = new HashSet();
+			if ( a instanceof OR ) {
+				operands.addAll(((OR)a).operands);
+			}
+			else if ( a!=null ) {
+				operands.add(a);
+			}
+			if ( b instanceof OR ) {
+				operands.addAll(((OR)b).operands);
+			}
+			else if ( b!=null ) {
+				operands.add(b);
+			}
+		}
+		public SemanticContext getGatedPredicateContext() {
+			SemanticContext result = null;
+			for (Iterator it = operands.iterator(); it.hasNext();) {
+				SemanticContext semctx = (SemanticContext) it.next();
+				SemanticContext gatedPred = semctx.getGatedPredicateContext();
+				if ( gatedPred!=null ) {
+					result = or(result, gatedPred);
+					// result = new OR(result, gatedPred);
+				}
+			}
+			return result;
+		}
+		public boolean isSyntacticPredicate() {
+			for (Iterator it = operands.iterator(); it.hasNext();) {
+				SemanticContext semctx = (SemanticContext) it.next();
+				if ( semctx.isSyntacticPredicate() ) {
+					return true;
+				}
+			}
+			return false;
+		}
+		public String toString() {
+			StringBuffer buf = new StringBuffer();
+			buf.append("(");
+			int i = 0;
+			for (Iterator it = operands.iterator(); it.hasNext();) {
+				SemanticContext semctx = (SemanticContext) it.next();
+				if ( i>0 ) {
+					buf.append("||");
+				}
+				buf.append(semctx.toString());
+				i++;
+			}
+			buf.append(")");
+			return buf.toString();
+		}
+	}
+
+	public static class NOT extends SemanticContext {
+		protected SemanticContext ctx;
+		public NOT(SemanticContext ctx) {
+			this.ctx = ctx;
+		}
+		public SemanticContext getGatedPredicateContext() {
+			SemanticContext p = ctx.getGatedPredicateContext();
+			if ( p==null ) {
+				return null;
+			}
+			return new NOT(p);
+		}
+		public boolean isSyntacticPredicate() {
+			return ctx.isSyntacticPredicate();
+		}
+		public boolean equals(Object object) {
+			if ( !(object instanceof NOT) ) {
+				return false;
+			}
+			return this.ctx.equals(((NOT)object).ctx);
+		}
+
+		public String toString() {
+			return "!("+ctx+")";
+		}
+	}
+
+	public static SemanticContext and(SemanticContext a, SemanticContext b) {
+		//System.out.println("AND: "+a+"&&"+b);
+		if ( a==EMPTY_SEMANTIC_CONTEXT || a==null ) {
+			return b;
+		}
+		if ( b==EMPTY_SEMANTIC_CONTEXT || b==null ) {
+			return a;
+		}
+		if ( a.equals(b) ) {
+			return a; // if same, just return left one
+		}
+		//System.out.println("## have to AND");
+		return new AND(a,b);
+	}
+
+	public static SemanticContext or(SemanticContext a, SemanticContext b) {
+		//System.out.println("OR: "+a+"||"+b);
+		if ( a==EMPTY_SEMANTIC_CONTEXT || a==null ) {
+			return b;
+		}
+		if ( b==EMPTY_SEMANTIC_CONTEXT || b==null ) {
+			return a;
+		}
+		if ( a instanceof TruePredicate ) {
+			return a;
+		}
+		if ( b instanceof TruePredicate ) {
+			return b;
+		}
+		if ( a instanceof NOT && b instanceof Predicate ) {
+			NOT n = (NOT)a;
+			// check for !p||p
+			if ( n.ctx.equals(b) ) {
+				return new TruePredicate();
+			}
+		}
+		else if ( b instanceof NOT && a instanceof Predicate ) {
+			NOT n = (NOT)b;
+			// check for p||!p
+			if ( n.ctx.equals(a) ) {
+				return new TruePredicate();
+			}
+		}
+		else if ( a.equals(b) ) {
+			return a;
+		}
+		//System.out.println("## have to OR");
+		return new OR(a,b);
+	}
+
+	public static SemanticContext not(SemanticContext a) {
+		return new NOT(a);
+	}
+
+}

tool/src/org/antlr/v4/analysis/SetLabel.java

@@ -0,0 +1,37 @@
+package org.antlr.v4.analysis;
+
+import org.antlr.v4.misc.IntSet;
+import org.antlr.v4.misc.IntervalSet;
+
+/** A label containing a set of values */
+public class SetLabel extends Label {
+	/** A set of token types or character codes if label==SET */
+	// TODO: try IntervalSet for everything
+	protected IntSet label;
+
+	public SetLabel(IntSet label) {
+		if ( label==null ) {
+			this.label = IntervalSet.of(INVALID);
+			return;
+		}
+		this.label = label;
+	}
+
+	public boolean intersect(Label other) {
+		if ( other.getClass() == SetLabel.class ) {
+			return label.and(((SetLabel)other).label).isNil();
+		}
+		return label.member(((AtomLabel)other).label);
+	}
+	
+	public int hashCode() {	return label.hashCode(); }
+
+	public boolean equals(Object o) {
+		if ( o==null ) return false;
+		if ( this == o ) return true; // equals if same object
+		if ( o.getClass() == AtomLabel.class ) {
+			o = IntervalSet.of(((AtomLabel)o).label);
+		}
+		return this.label.equals(((SetLabel)o).label);
+	}
+}

tool/src/org/antlr/v4/analysis/Transition.java

@@ -0,0 +1,15 @@
+package org.antlr.v4.analysis;
+
+import org.antlr.analysis.State;
+
+/** An NFA transition between any two NFA states.  Subclasses define
+ *  atom, set, epsilon, action, predicate, rule transitions.
+ *
+ *  This is a one way link.  It emanates from a state (usually via a list of
+ *  transitions) and has a target state.
+ */
+public abstract class Transition implements Comparable {
+	/** The target of this transition */
+	public State target;
+	
+}

tool/src/org/antlr/v4/misc/BitSet.java

@@ -0,0 +1,574 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.v4.misc;
+
+import org.antlr.analysis.Label;
+import org.antlr.tool.Grammar;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**A BitSet to replace java.util.BitSet.
+ *
+ * Primary differences are that most set operators return new sets
+ * as opposed to oring and anding "in place".  Further, a number of
+ * operations were added.  I cannot contain a BitSet because there
+ * is no way to access the internal bits (which I need for speed)
+ * and, because it is final, I cannot subclass to add functionality.
+ * Consider defining set degree.  Without access to the bits, I must
+ * call a method n times to test the ith bit...ack!
+ *
+ * Also seems like or() from util is wrong when size of incoming set is bigger
+ * than this.bits.length.
+ *
+ * @author Terence Parr
+ */
+public class BitSet implements IntSet, Cloneable {
+    protected final static int BITS = 64;    // number of bits / long
+    protected final static int LOG_BITS = 6; // 2^6 == 64
+
+    /* We will often need to do a mod operator (i mod nbits).  Its
+     * turns out that, for powers of two, this mod operation is
+     * same as (i & (nbits-1)).  Since mod is slow, we use a
+     * precomputed mod mask to do the mod instead.
+     */
+    protected final static int MOD_MASK = BITS - 1;
+
+    /** The actual data bits */
+    protected long bits[];
+
+    /** Construct a bitset of size one word (64 bits) */
+    public BitSet() {
+        this(BITS);
+    }
+
+    /** Construction from a static array of longs */
+    public BitSet(long[] bits_) {
+        bits = bits_;
+    }
+
+    /** Construct a bitset given the size
+     * @param nbits The size of the bitset in bits
+     */
+    public BitSet(int nbits) {
+        bits = new long[((nbits - 1) >> LOG_BITS) + 1];
+    }
+
+    /** or this element into this set (grow as necessary to accommodate) */
+    public void add(int el) {
+        //System.out.println("add("+el+")");
+        int n = wordNumber(el);
+        //System.out.println("word number is "+n);
+        //System.out.println("bits.length "+bits.length);
+        if (n >= bits.length) {
+            growToInclude(el);
+        }
+        bits[n] |= bitMask(el);
+    }
+
+    public void addAll(IntSet set) {
+        if ( set instanceof BitSet ) {
+            this.orInPlace((BitSet)set);
+        }
+		else if ( set instanceof IntervalSet ) {
+			IntervalSet other = (IntervalSet)set;
+			// walk set and add each interval
+			for (Iterator iter = other.intervals.iterator(); iter.hasNext();) {
+				Interval I = (Interval) iter.next();
+				this.orInPlace(BitSet.range(I.a,I.b));
+			}
+		}
+		else {
+			throw new IllegalArgumentException("can't add "+
+											   set.getClass().getName()+
+											   " to BitSet");
+		}
+    }
+
+	public void addAll(int[] elements) {
+		if ( elements==null ) {
+			return;
+		}
+		for (int i = 0; i < elements.length; i++) {
+			int e = elements[i];
+			add(e);
+		}
+	}
+
+	public void addAll(Iterable elements) {
+		if ( elements==null ) {
+			return;
+		}
+		Iterator it = elements.iterator();
+		while (it.hasNext()) {
+			Object o = (Object) it.next();
+			if ( !(o instanceof Integer) ) {
+				throw new IllegalArgumentException();
+			}
+			Integer eI = (Integer)o;
+			add(eI.intValue());
+		}
+		/*
+		int n = elements.size();
+		for (int i = 0; i < n; i++) {
+			Object o = elements.get(i);
+			if ( !(o instanceof Integer) ) {
+				throw new IllegalArgumentException();
+			}
+			Integer eI = (Integer)o;
+			add(eI.intValue());
+		}
+		 */
+	}
+
+    public IntSet and(IntSet a) {
+        BitSet s = (BitSet)this.clone();
+        s.andInPlace((BitSet)a);
+        return s;
+    }
+
+    public void andInPlace(BitSet a) {
+        int min = Math.min(bits.length, a.bits.length);
+        for (int i = min - 1; i >= 0; i--) {
+            bits[i] &= a.bits[i];
+        }
+        // clear all bits in this not present in a (if this bigger than a).
+        for (int i = min; i < bits.length; i++) {
+            bits[i] = 0;
+        }
+    }
+
+    private final static long bitMask(int bitNumber) {
+        int bitPosition = bitNumber & MOD_MASK; // bitNumber mod BITS
+        return 1L << bitPosition;
+    }
+
+    public void clear() {
+        for (int i = bits.length - 1; i >= 0; i--) {
+            bits[i] = 0;
+        }
+    }
+
+    public void clear(int el) {
+        int n = wordNumber(el);
+        if (n >= bits.length) {	// grow as necessary to accommodate
+            growToInclude(el);
+        }
+        bits[n] &= ~bitMask(el);
+    }
+
+    public Object clone() {
+        BitSet s;
+        try {
+            s = (BitSet)super.clone();
+            s.bits = new long[bits.length];
+            System.arraycopy(bits, 0, s.bits, 0, bits.length);
+        }
+        catch (CloneNotSupportedException e) {
+            throw new InternalError();
+        }
+        return s;
+    }
+
+    public int size() {
+        int deg = 0;
+        for (int i = bits.length - 1; i >= 0; i--) {
+            long word = bits[i];
+            if (word != 0L) {
+                for (int bit = BITS - 1; bit >= 0; bit--) {
+                    if ((word & (1L << bit)) != 0) {
+                        deg++;
+                    }
+                }
+            }
+        }
+        return deg;
+    }
+
+    public boolean equals(Object other) {
+        if ( other == null || !(other instanceof BitSet) ) {
+            return false;
+        }
+
+        BitSet otherSet = (BitSet)other;
+
+        int n = Math.min(this.bits.length, otherSet.bits.length);
+
+        // for any bits in common, compare
+        for (int i=0; i<n; i++) {
+            if (this.bits[i] != otherSet.bits[i]) {
+                return false;
+            }
+        }
+
+        // make sure any extra bits are off
+
+        if (this.bits.length > n) {
+            for (int i = n+1; i<this.bits.length; i++) {
+                if (this.bits[i] != 0) {
+                    return false;
+                }
+            }
+        }
+        else if (otherSet.bits.length > n) {
+            for (int i = n+1; i<otherSet.bits.length; i++) {
+                if (otherSet.bits[i] != 0) {
+                    return false;
+                }
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * Grows the set to a larger number of bits.
+     * @param bit element that must fit in set
+     */
+    public void growToInclude(int bit) {
+        int newSize = Math.max(bits.length << 1, numWordsToHold(bit));
+        long newbits[] = new long[newSize];
+        System.arraycopy(bits, 0, newbits, 0, bits.length);
+        bits = newbits;
+    }
+
+    public boolean member(int el) {
+        int n = wordNumber(el);
+        if (n >= bits.length) return false;
+        return (bits[n] & bitMask(el)) != 0;
+    }
+
+    /** Get the first element you find and return it.  Return Label.INVALID
+     *  otherwise.
+     */
+    public int getSingleElement() {
+        for (int i = 0; i < (bits.length << LOG_BITS); i++) {
+            if (member(i)) {
+                return i;
+            }
+        }
+        return Label.INVALID;
+    }
+
+    public boolean isNil() {
+        for (int i = bits.length - 1; i >= 0; i--) {
+            if (bits[i] != 0) return false;
+        }
+        return true;
+    }
+
+    public IntSet complement() {
+        BitSet s = (BitSet)this.clone();
+        s.notInPlace();
+        return s;
+    }
+
+    public IntSet complement(IntSet set) {
+		if ( set==null ) {
+			return this.complement();
+		}
+        return set.subtract(this);
+    }
+
+    public void notInPlace() {
+        for (int i = bits.length - 1; i >= 0; i--) {
+            bits[i] = ~bits[i];
+        }
+    }
+
+    /** complement bits in the range 0..maxBit. */
+    public void notInPlace(int maxBit) {
+        notInPlace(0, maxBit);
+    }
+
+    /** complement bits in the range minBit..maxBit.*/
+    public void notInPlace(int minBit, int maxBit) {
+        // make sure that we have room for maxBit
+        growToInclude(maxBit);
+        for (int i = minBit; i <= maxBit; i++) {
+            int n = wordNumber(i);
+            bits[n] ^= bitMask(i);
+        }
+    }
+
+    private final int numWordsToHold(int el) {
+        return (el >> LOG_BITS) + 1;
+    }
+
+    public static BitSet of(int el) {
+        BitSet s = new BitSet(el + 1);
+        s.add(el);
+        return s;
+    }
+
+    public static BitSet of(Collection elements) {
+        BitSet s = new BitSet();
+        Iterator iter = elements.iterator();
+        while (iter.hasNext()) {
+            Integer el = (Integer) iter.next();
+            s.add(el.intValue());
+        }
+        return s;
+    }
+
+	public static BitSet of(IntSet set) {
+		if ( set==null ) {
+			return null;
+		}
+
+		if ( set instanceof BitSet ) {
+			return (BitSet)set;
+		}
+		if ( set instanceof IntervalSet ) {
+			BitSet s = new BitSet();
+			s.addAll(set);
+			return s;
+		}
+		throw new IllegalArgumentException("can't create BitSet from "+set.getClass().getName());
+	}
+
+    public static BitSet of(Map elements) {
+        return BitSet.of(elements.keySet());
+    }
+
+	public static BitSet range(int a, int b) {
+		BitSet s = new BitSet(b + 1);
+		for (int i = a; i <= b; i++) {
+			int n = wordNumber(i);
+			s.bits[n] |= bitMask(i);
+		}
+		return s;
+	}
+
+    /** return this | a in a new set */
+    public IntSet or(IntSet a) {
+		if ( a==null ) {
+			return this;
+		}
+        BitSet s = (BitSet)this.clone();
+        s.orInPlace((BitSet)a);
+        return s;
+    }
+
+    public void orInPlace(BitSet a) {
+		if ( a==null ) {
+			return;
+		}
+        // If this is smaller than a, grow this first
+        if (a.bits.length > bits.length) {
+            setSize(a.bits.length);
+        }
+        int min = Math.min(bits.length, a.bits.length);
+        for (int i = min - 1; i >= 0; i--) {
+            bits[i] |= a.bits[i];
+        }
+    }
+
+    // remove this element from this set
+    public void remove(int el) {
+        int n = wordNumber(el);
+        if (n >= bits.length) {
+            growToInclude(el);
+        }
+        bits[n] &= ~bitMask(el);
+    }
+
+    /**
+     * Sets the size of a set.
+     * @param nwords how many words the new set should be
+     */
+    private void setSize(int nwords) {
+        long newbits[] = new long[nwords];
+        int n = Math.min(nwords, bits.length);
+        System.arraycopy(bits, 0, newbits, 0, n);
+        bits = newbits;
+    }
+
+    public int numBits() {
+        return bits.length << LOG_BITS; // num words * bits per word
+    }
+
+    /** return how much space is being used by the bits array not
+     *  how many actually have member bits on.
+     */
+    public int lengthInLongWords() {
+        return bits.length;
+    }
+
+    /**Is this contained within a? */
+    public boolean subset(BitSet a) {
+        if (a == null) return false;
+        return this.and(a).equals(this);
+    }
+
+    /**Subtract the elements of 'a' from 'this' in-place.
+     * Basically, just turn off all bits of 'this' that are in 'a'.
+     */
+    public void subtractInPlace(BitSet a) {
+        if (a == null) return;
+        // for all words of 'a', turn off corresponding bits of 'this'
+        for (int i = 0; i < bits.length && i < a.bits.length; i++) {
+            bits[i] &= ~a.bits[i];
+        }
+    }
+
+    public IntSet subtract(IntSet a) {
+        if (a == null || !(a instanceof BitSet)) return null;
+
+        BitSet s = (BitSet)this.clone();
+        s.subtractInPlace((BitSet)a);
+        return s;
+    }
+
+	public List toList() {
+		throw new NoSuchMethodError("BitSet.toList() unimplemented");
+	}
+
+    public int[] toArray() {
+        int[] elems = new int[size()];
+        int en = 0;
+        for (int i = 0; i < (bits.length << LOG_BITS); i++) {
+            if (member(i)) {
+                elems[en++] = i;
+            }
+        }
+        return elems;
+    }
+
+    public long[] toPackedArray() {
+        return bits;
+    }
+
+    public String toString() {
+        return toString(null);
+    }
+
+    /** Transform a bit set into a string by formatting each element as an integer
+     * separator The string to put in between elements
+     * @return A commma-separated list of values
+     */
+    public String toString(Grammar g) {
+        StringBuffer buf = new StringBuffer();
+        String separator = ",";
+		boolean havePrintedAnElement = false;
+		buf.append('{');
+
+        for (int i = 0; i < (bits.length << LOG_BITS); i++) {
+            if (member(i)) {
+                if (i > 0 && havePrintedAnElement ) {
+                    buf.append(separator);
+                }
+                if ( g!=null ) {
+                    buf.append(g.getTokenDisplayName(i));
+                }
+                else {
+                    buf.append(i);
+                }
+				havePrintedAnElement = true;
+            }
+        }
+		buf.append('}');
+        return buf.toString();
+    }
+
+    /**Create a string representation where instead of integer elements, the
+     * ith element of vocabulary is displayed instead.  Vocabulary is a Vector
+     * of Strings.
+     * separator The string to put in between elements
+     * @return A commma-separated list of character constants.
+     */
+    public String toString(String separator, List vocabulary) {
+        if (vocabulary == null) {
+            return toString(null);
+        }
+        String str = "";
+        for (int i = 0; i < (bits.length << LOG_BITS); i++) {
+            if (member(i)) {
+                if (str.length() > 0) {
+                    str += separator;
+                }
+                if (i >= vocabulary.size()) {
+                    str += "'" + (char)i + "'";
+                }
+                else if (vocabulary.get(i) == null) {
+                    str += "'" + (char)i + "'";
+                }
+                else {
+                    str += (String)vocabulary.get(i);
+                }
+            }
+        }
+        return str;
+    }
+
+    /**
+     * Dump a comma-separated list of the words making up the bit set.
+     * Split each 64 bit number into two more manageable 32 bit numbers.
+     * This generates a comma-separated list of C++-like unsigned long constants.
+     */
+    public String toStringOfHalfWords() {
+        StringBuffer s = new StringBuffer();
+        for (int i = 0; i < bits.length; i++) {
+            if (i != 0) s.append(", ");
+            long tmp = bits[i];
+            tmp &= 0xFFFFFFFFL;
+            s.append(tmp);
+			s.append("UL");
+            s.append(", ");
+            tmp = bits[i] >>> 32;
+            tmp &= 0xFFFFFFFFL;
+			s.append(tmp);
+			s.append("UL");
+        }
+		return s.toString();
+    }
+
+    /**
+     * Dump a comma-separated list of the words making up the bit set.
+     * This generates a comma-separated list of Java-like long int constants.
+     */
+    public String toStringOfWords() {
+		StringBuffer s = new StringBuffer();
+        for (int i = 0; i < bits.length; i++) {
+            if (i != 0) s.append(", ");
+            s.append(bits[i]);
+			s.append("L");
+        }
+        return s.toString();
+    }
+
+    public String toStringWithRanges() {
+        return toString();
+    }
+
+    private final static int wordNumber(int bit) {
+        return bit >> LOG_BITS; // bit / BITS
+    }
+}

tool/src/org/antlr/v4/misc/IntSet.java

@@ -0,0 +1,84 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.v4.misc;
+
+import org.antlr.tool.Grammar;
+
+import java.util.List;
+
+/** A generic set of ints that has an efficient implementation, BitSet,
+ *  which is a compressed bitset and is useful for ints that
+ *  are small, for example less than 500 or so, and w/o many ranges.  For
+ *  ranges with large values like unicode char sets, this is not very efficient.
+ *  Consider using IntervalSet.  Not all methods in IntervalSet are implemented.
+ *
+ *  @see org.antlr.misc.BitSet
+ *  @see org.antlr.misc.IntervalSet
+ */
+public interface IntSet {
+    /** Add an element to the set */
+    void add(int el);
+
+    /** Add all elements from incoming set to this set.  Can limit
+     *  to set of its own type.
+     */
+    void addAll(IntSet set);
+
+    /** Return the intersection of this set with the argument, creating
+     *  a new set.
+     */
+    IntSet and(IntSet a);
+
+    IntSet complement(IntSet elements);
+
+    IntSet or(IntSet a);
+
+    IntSet subtract(IntSet a);
+
+    /** Return the size of this set (not the underlying implementation's
+     *  allocated memory size, for example).
+     */
+    int size();
+
+    boolean isNil();
+
+    boolean equals(Object obj);
+
+    int getSingleElement();
+
+    boolean member(int el);
+
+    /** remove this element from this set */
+    void remove(int el);
+
+    List toList();
+
+    String toString();
+
+    String toString(Grammar g);
+}

tool/src/org/antlr/v4/misc/Interval.java

@@ -0,0 +1,142 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.v4.misc;
+
+/** An immutable inclusive interval a..b */
+public class Interval {
+	public static final int INTERVAL_POOL_MAX_VALUE = 1000;
+
+	static Interval[] cache = new Interval[INTERVAL_POOL_MAX_VALUE+1];
+
+	public int a;
+	public int b;
+
+	public static int creates = 0;
+	public static int misses = 0;
+	public static int hits = 0;
+	public static int outOfRange = 0;
+
+	public Interval(int a, int b) { this.a=a; this.b=b; }
+
+	/** Interval objects are used readonly so share all with the
+	 *  same single value a==b up to some max size.  Use an array as a perfect hash.
+	 *  Return shared object for 0..INTERVAL_POOL_MAX_VALUE or a new
+	 *  Interval object with a..a in it.  On Java.g, 218623 IntervalSets
+	 *  have a..a (set with 1 element).
+	 */
+	public static Interval create(int a, int b) {
+		//return new Interval(a,b);
+		// cache just a..a
+		if ( a!=b || a<0 || a>INTERVAL_POOL_MAX_VALUE ) {
+			return new Interval(a,b);
+		}
+		if ( cache[a]==null ) {
+			cache[a] = new Interval(a,a);
+		}
+		return cache[a];
+	}
+
+	public boolean equals(Object o) {
+		if ( o==null ) {
+			return false;
+		}
+		Interval other = (Interval)o;
+		return this.a==other.a && this.b==other.b;
+	}
+
+	/** Does this start completely before other? Disjoint */
+	public boolean startsBeforeDisjoint(Interval other) {
+		return this.a<other.a && this.b<other.a;
+	}
+
+	/** Does this start at or before other? Nondisjoint */
+	public boolean startsBeforeNonDisjoint(Interval other) {
+		return this.a<=other.a && this.b>=other.a;
+	}
+
+	/** Does this.a start after other.b? May or may not be disjoint */
+	public boolean startsAfter(Interval other) { return this.a>other.a; }
+
+	/** Does this start completely after other? Disjoint */
+	public boolean startsAfterDisjoint(Interval other) {
+		return this.a>other.b;
+	}
+
+	/** Does this start after other? NonDisjoint */
+	public boolean startsAfterNonDisjoint(Interval other) {
+		return this.a>other.a && this.a<=other.b; // this.b>=other.b implied
+	}
+
+	/** Are both ranges disjoint? I.e., no overlap? */
+	public boolean disjoint(Interval other) {
+		return startsBeforeDisjoint(other) || startsAfterDisjoint(other);
+	}
+
+	/** Are two intervals adjacent such as 0..41 and 42..42? */
+	public boolean adjacent(Interval other) {
+		return this.a == other.b+1 || this.b == other.a-1;
+	}
+
+	public boolean properlyContains(Interval other) {
+		return other.a >= this.a && other.b <= this.b;
+	}
+
+	/** Return the interval computed from combining this and other */
+	public Interval union(Interval other) {
+		return Interval.create(Math.min(a,other.a), Math.max(b,other.b));
+	}
+
+	/** Return the interval in common between this and o */
+	public Interval intersection(Interval other) {
+		return Interval.create(Math.max(a,other.a), Math.min(b,other.b));
+	}
+
+	/** Return the interval with elements from this not in other;
+	 *  other must not be totally enclosed (properly contained)
+	 *  within this, which would result in two disjoint intervals
+	 *  instead of the single one returned by this method.
+	 */
+	public Interval differenceNotProperlyContained(Interval other) {
+		Interval diff = null;
+		// other.a to left of this.a (or same)
+		if ( other.startsBeforeNonDisjoint(this) ) {
+			diff = Interval.create(Math.max(this.a,other.b+1),
+								   this.b);
+		}
+
+		// other.a to right of this.a
+		else if ( other.startsAfterNonDisjoint(this) ) {
+			diff = Interval.create(this.a, other.a-1);
+		}
+		return diff;
+	}
+
+	public String toString() {
+		return a+".."+b;
+	}
+}

tool/src/org/antlr/v4/misc/IntervalSet.java

@@ -0,0 +1,692 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.v4.misc;
+
+import org.antlr.analysis.Label;
+import org.antlr.tool.Grammar;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A set of integers that relies on ranges being common to do
+ *  "run-length-encoded" like compression (if you view an IntSet like
+ *  a BitSet with runs of 0s and 1s).  Only ranges are recorded so that
+ *  a few ints up near value 1000 don't cause massive bitsets, just two
+ *  integer intervals.
+ *
+ *  element values may be negative.  Useful for sets of EPSILON and EOF.
+ *
+ *  0..9 char range is index pair ['\u0030','\u0039'].
+ *  Multiple ranges are encoded with multiple index pairs.  Isolated
+ *  elements are encoded with an index pair where both intervals are the same.
+ *
+ *  The ranges are ordered and disjoint so that 2..6 appears before 101..103.
+ */
+public class IntervalSet implements IntSet {
+	public static final IntervalSet COMPLETE_SET = IntervalSet.of(0,Label.MAX_CHAR_VALUE);
+
+	/** The list of sorted, disjoint intervals. */
+    protected List<Interval> intervals;
+
+	/** Create a set with no elements */
+    public IntervalSet() {
+        intervals = new ArrayList<Interval>(2); // most sets are 1 or 2 elements
+    }
+
+	public IntervalSet(List<Interval> intervals) {
+		this.intervals = intervals;
+	}
+
+	/** Create a set with a single element, el. */
+    public static IntervalSet of(int a) {
+		IntervalSet s = new IntervalSet();
+        s.add(a);
+        return s;
+    }
+
+    /** Create a set with all ints within range [a..b] (inclusive) */
+    public static IntervalSet of(int a, int b) {
+        IntervalSet s = new IntervalSet();
+        s.add(a,b);
+        return s;
+    }
+
+    /** Add a single element to the set.  An isolated element is stored
+     *  as a range el..el.
+     */
+    public void add(int el) {
+        add(el,el);
+    }
+
+    /** Add interval; i.e., add all integers from a to b to set.
+     *  If b<a, do nothing.
+     *  Keep list in sorted order (by left range value).
+     *  If overlap, combine ranges.  For example,
+     *  If this is {1..5, 10..20}, adding 6..7 yields
+     *  {1..5, 6..7, 10..20}.  Adding 4..8 yields {1..8, 10..20}.
+     */
+    public void add(int a, int b) {
+        add(Interval.create(a,b));
+    }
+
+	// copy on write so we can cache a..a intervals and sets of that
+	protected void add(Interval addition) {
+		//System.out.println("add "+addition+" to "+intervals.toString());
+		if ( addition.b<addition.a ) {
+			return;
+		}
+		// find position in list
+		// Use iterators as we modify list in place
+		for (ListIterator iter = intervals.listIterator(); iter.hasNext();) {
+			Interval r = (Interval) iter.next();
+			if ( addition.equals(r) ) {
+				return;
+			}
+			if ( addition.adjacent(r) || !addition.disjoint(r) ) {
+				// next to each other, make a single larger interval
+				Interval bigger = addition.union(r);
+				iter.set(bigger);
+				// make sure we didn't just create an interval that
+				// should be merged with next interval in list
+				if ( iter.hasNext() ) {
+					Interval next = (Interval) iter.next();
+					if ( bigger.adjacent(next)||!bigger.disjoint(next) ) {
+						// if we bump up against or overlap next, merge
+						iter.remove();   // remove this one
+						iter.previous(); // move backwards to what we just set
+						iter.set(bigger.union(next)); // set to 3 merged ones
+					}
+				}
+				return;
+			}
+			if ( addition.startsBeforeDisjoint(r) ) {
+				// insert before r
+				iter.previous();
+				iter.add(addition);
+				return;
+			}
+			// if disjoint and after r, a future iteration will handle it
+		}
+		// ok, must be after last interval (and disjoint from last interval)
+		// just add it
+		intervals.add(addition);
+	}
+
+	/*
+	protected void add(Interval addition) {
+        //System.out.println("add "+addition+" to "+intervals.toString());
+        if ( addition.b<addition.a ) {
+            return;
+        }
+        // find position in list
+        //for (ListIterator iter = intervals.listIterator(); iter.hasNext();) {
+		int n = intervals.size();
+		for (int i=0; i<n; i++) {
+			Interval r = (Interval)intervals.get(i);
+            if ( addition.equals(r) ) {
+                return;
+            }
+            if ( addition.adjacent(r) || !addition.disjoint(r) ) {
+                // next to each other, make a single larger interval
+                Interval bigger = addition.union(r);
+				intervals.set(i, bigger);
+                // make sure we didn't just create an interval that
+                // should be merged with next interval in list
+				if ( (i+1)<n ) {
+					i++;
+					Interval next = (Interval)intervals.get(i);
+                    if ( bigger.adjacent(next)||!bigger.disjoint(next) ) {
+                        // if we bump up against or overlap next, merge
+						intervals.remove(i); // remove next one
+						i--;
+						intervals.set(i, bigger.union(next)); // set to 3 merged ones
+                    }
+                }
+                return;
+            }
+            if ( addition.startsBeforeDisjoint(r) ) {
+                // insert before r
+				intervals.add(i, addition);
+                return;
+            }
+            // if disjoint and after r, a future iteration will handle it
+        }
+        // ok, must be after last interval (and disjoint from last interval)
+        // just add it
+        intervals.add(addition);
+    }
+*/
+
+	public void addAll(IntSet set) {
+		if ( set==null ) {
+			return;
+		}
+        if ( !(set instanceof IntervalSet) ) {
+            throw new IllegalArgumentException("can't add non IntSet ("+
+											   set.getClass().getName()+
+											   ") to IntervalSet");
+        }
+        IntervalSet other = (IntervalSet)set;
+        // walk set and add each interval
+		int n = other.intervals.size();
+		for (int i = 0; i < n; i++) {
+			Interval I = (Interval) other.intervals.get(i);
+			this.add(I.a,I.b);
+		}
+    }
+
+    public IntSet complement(int minElement, int maxElement) {
+        return this.complement(IntervalSet.of(minElement,maxElement));
+    }
+
+    /** Given the set of possible values (rather than, say UNICODE or MAXINT),
+     *  return a new set containing all elements in vocabulary, but not in
+     *  this.  The computation is (vocabulary - this).
+     *
+     *  'this' is assumed to be either a subset or equal to vocabulary.
+     */
+    public IntSet complement(IntSet vocabulary) {
+        if ( vocabulary==null ) {
+            return null; // nothing in common with null set
+        }
+		if ( !(vocabulary instanceof IntervalSet ) ) {
+			throw new IllegalArgumentException("can't complement with non IntervalSet ("+
+											   vocabulary.getClass().getName()+")");
+		}
+		IntervalSet vocabularyIS = ((IntervalSet)vocabulary);
+		int maxElement = vocabularyIS.getMaxElement();
+
+		IntervalSet compl = new IntervalSet();
+		int n = intervals.size();
+		if ( n ==0 ) {
+			return compl;
+		}
+		Interval first = (Interval)intervals.get(0);
+		// add a range from 0 to first.a constrained to vocab
+		if ( first.a > 0 ) {
+			IntervalSet s = IntervalSet.of(0, first.a-1);
+			IntervalSet a = (IntervalSet)s.and(vocabularyIS);
+			compl.addAll(a);
+		}
+		for (int i=1; i<n; i++) { // from 2nd interval .. nth
+			Interval previous = (Interval)intervals.get(i-1);
+			Interval current = (Interval)intervals.get(i);
+			IntervalSet s = IntervalSet.of(previous.b+1, current.a-1);
+			IntervalSet a = (IntervalSet)s.and(vocabularyIS);
+			compl.addAll(a);
+		}
+		Interval last = (Interval)intervals.get(n -1);
+		// add a range from last.b to maxElement constrained to vocab
+		if ( last.b < maxElement ) {
+			IntervalSet s = IntervalSet.of(last.b+1, maxElement);
+			IntervalSet a = (IntervalSet)s.and(vocabularyIS);
+			compl.addAll(a);
+		}
+		return compl;
+    }
+
+	/** Compute this-other via this&~other.
+	 *  Return a new set containing all elements in this but not in other.
+	 *  other is assumed to be a subset of this;
+     *  anything that is in other but not in this will be ignored.
+	 */
+	public IntSet subtract(IntSet other) {
+		// assume the whole unicode range here for the complement
+		// because it doesn't matter.  Anything beyond the max of this' set
+		// will be ignored since we are doing this & ~other.  The intersection
+		// will be empty.  The only problem would be when this' set max value
+		// goes beyond MAX_CHAR_VALUE, but hopefully the constant MAX_CHAR_VALUE
+		// will prevent this.
+		return this.and(((IntervalSet)other).complement(COMPLETE_SET));
+	}
+
+	/** return a new set containing all elements in this but not in other.
+     *  Intervals may have to be broken up when ranges in this overlap
+     *  with ranges in other.  other is assumed to be a subset of this;
+     *  anything that is in other but not in this will be ignored.
+	 *
+	 *  Keep around, but 10-20-2005, I decided to make complement work w/o
+	 *  subtract and so then subtract can simply be a&~b
+	 *
+    public IntSet subtract(IntSet other) {
+        if ( other==null || !(other instanceof IntervalSet) ) {
+            return null; // nothing in common with null set
+        }
+
+        IntervalSet diff = new IntervalSet();
+
+        // iterate down both interval lists
+        ListIterator thisIter = this.intervals.listIterator();
+        ListIterator otherIter = ((IntervalSet)other).intervals.listIterator();
+        Interval mine=null;
+        Interval theirs=null;
+        if ( thisIter.hasNext() ) {
+            mine = (Interval)thisIter.next();
+        }
+        if ( otherIter.hasNext() ) {
+            theirs = (Interval)otherIter.next();
+        }
+        while ( mine!=null ) {
+            //System.out.println("mine="+mine+", theirs="+theirs);
+            // CASE 1: nothing in theirs removes a chunk from mine
+            if ( theirs==null || mine.disjoint(theirs) ) {
+                // SUBCASE 1a: finished traversing theirs; keep adding mine now
+                if ( theirs==null ) {
+                    // add everything in mine to difference since theirs done
+                    diff.add(mine);
+                    mine = null;
+                    if ( thisIter.hasNext() ) {
+                        mine = (Interval)thisIter.next();
+                    }
+                }
+                else {
+                    // SUBCASE 1b: mine is completely to the left of theirs
+                    // so we can add to difference; move mine, but not theirs
+                    if ( mine.startsBeforeDisjoint(theirs) ) {
+                        diff.add(mine);
+                        mine = null;
+                        if ( thisIter.hasNext() ) {
+                            mine = (Interval)thisIter.next();
+                        }
+                    }
+                    // SUBCASE 1c: theirs is completely to the left of mine
+                    else {
+                        // keep looking in theirs
+                        theirs = null;
+                        if ( otherIter.hasNext() ) {
+                            theirs = (Interval)otherIter.next();
+                        }
+                    }
+                }
+            }
+            else {
+                // CASE 2: theirs breaks mine into two chunks
+                if ( mine.properlyContains(theirs) ) {
+                    // must add two intervals: stuff to left and stuff to right
+                    diff.add(mine.a, theirs.a-1);
+                    // don't actually add stuff to right yet as next 'theirs'
+                    // might overlap with it
+                    // The stuff to the right might overlap with next "theirs".
+                    // so it is considered next
+                    Interval right = new Interval(theirs.b+1, mine.b);
+                    mine = right;
+                    // move theirs forward
+                    theirs = null;
+                    if ( otherIter.hasNext() ) {
+                        theirs = (Interval)otherIter.next();
+                    }
+                }
+
+                // CASE 3: theirs covers mine; nothing to add to diff
+                else if ( theirs.properlyContains(mine) ) {
+                    // nothing to add, theirs forces removal totally of mine
+                    // just move mine looking for an overlapping interval
+                    mine = null;
+                    if ( thisIter.hasNext() ) {
+                        mine = (Interval)thisIter.next();
+                    }
+                }
+
+                // CASE 4: non proper overlap
+                else {
+                    // overlap, but not properly contained
+                    diff.add(mine.differenceNotProperlyContained(theirs));
+                    // update iterators
+                    boolean moveTheirs = true;
+                    if ( mine.startsBeforeNonDisjoint(theirs) ||
+                         theirs.b > mine.b )
+                    {
+                        // uh oh, right of theirs extends past right of mine
+                        // therefore could overlap with next of mine so don't
+                        // move theirs iterator yet
+                        moveTheirs = false;
+                    }
+                    // always move mine
+                    mine = null;
+                    if ( thisIter.hasNext() ) {
+                        mine = (Interval)thisIter.next();
+                    }
+                    if ( moveTheirs ) {
+                        theirs = null;
+                        if ( otherIter.hasNext() ) {
+                            theirs = (Interval)otherIter.next();
+                        }
+                    }
+                }
+            }
+        }
+        return diff;
+    }
+	 */
+
+    /** TODO: implement this! */
+	public IntSet or(IntSet a) {
+		IntervalSet o = new IntervalSet();
+		o.addAll(this);
+		o.addAll(a);
+		//throw new NoSuchMethodError();
+		return o;
+	}
+
+    /** Return a new set with the intersection of this set with other.  Because
+     *  the intervals are sorted, we can use an iterator for each list and
+     *  just walk them together.  This is roughly O(min(n,m)) for interval
+     *  list lengths n and m.
+     */
+	public IntSet and(IntSet other) {
+		if ( other==null ) { //|| !(other instanceof IntervalSet) ) {
+			return null; // nothing in common with null set
+		}
+
+		ArrayList myIntervals = (ArrayList)this.intervals;
+		ArrayList theirIntervals = (ArrayList)((IntervalSet)other).intervals;
+		IntervalSet intersection = null;
+		int mySize = myIntervals.size();
+		int theirSize = theirIntervals.size();
+		int i = 0;
+		int j = 0;
+		// iterate down both interval lists looking for nondisjoint intervals
+		while ( i<mySize && j<theirSize ) {
+			Interval mine = (Interval)myIntervals.get(i);
+			Interval theirs = (Interval)theirIntervals.get(j);
+			//System.out.println("mine="+mine+" and theirs="+theirs);
+			if ( mine.startsBeforeDisjoint(theirs) ) {
+				// move this iterator looking for interval that might overlap
+				i++;
+			}
+			else if ( theirs.startsBeforeDisjoint(mine) ) {
+				// move other iterator looking for interval that might overlap
+				j++;
+			}
+			else if ( mine.properlyContains(theirs) ) {
+				// overlap, add intersection, get next theirs
+				if ( intersection==null ) {
+					intersection = new IntervalSet();
+				}
+				intersection.add(mine.intersection(theirs));
+				j++;
+			}
+			else if ( theirs.properlyContains(mine) ) {
+				// overlap, add intersection, get next mine
+				if ( intersection==null ) {
+					intersection = new IntervalSet();
+				}
+				intersection.add(mine.intersection(theirs));
+				i++;
+			}
+			else if ( !mine.disjoint(theirs) ) {
+				// overlap, add intersection
+				if ( intersection==null ) {
+					intersection = new IntervalSet();
+				}
+				intersection.add(mine.intersection(theirs));
+				// Move the iterator of lower range [a..b], but not
+				// the upper range as it may contain elements that will collide
+				// with the next iterator. So, if mine=[0..115] and
+				// theirs=[115..200], then intersection is 115 and move mine
+				// but not theirs as theirs may collide with the next range
+				// in thisIter.
+				// move both iterators to next ranges
+				if ( mine.startsAfterNonDisjoint(theirs) ) {
+					j++;
+				}
+				else if ( theirs.startsAfterNonDisjoint(mine) ) {
+					i++;
+				}
+			}
+		}
+		if ( intersection==null ) {
+			return new IntervalSet();
+		}
+		return intersection;
+	}
+
+    /** Is el in any range of this set? */
+    public boolean member(int el) {
+		int n = intervals.size();
+		for (int i = 0; i < n; i++) {
+			Interval I = (Interval) intervals.get(i);
+			int a = I.a;
+			int b = I.b;
+			if ( el<a ) {
+				break; // list is sorted and el is before this interval; not here
+			}
+			if ( el>=a && el<=b ) {
+				return true; // found in this interval
+			}
+		}
+		return false;
+/*
+		for (ListIterator iter = intervals.listIterator(); iter.hasNext();) {
+            Interval I = (Interval) iter.next();
+            if ( el<I.a ) {
+                break; // list is sorted and el is before this interval; not here
+            }
+            if ( el>=I.a && el<=I.b ) {
+                return true; // found in this interval
+            }
+        }
+        return false;
+        */
+    }
+
+    /** return true if this set has no members */
+    public boolean isNil() {
+        return intervals==null || intervals.size()==0;
+    }
+
+    /** If this set is a single integer, return it otherwise Label.INVALID */
+    public int getSingleElement() {
+        if ( intervals!=null && intervals.size()==1 ) {
+            Interval I = (Interval)intervals.get(0);
+            if ( I.a == I.b ) {
+                return I.a;
+            }
+        }
+        return Label.INVALID;
+    }
+
+	public int getMaxElement() {
+		if ( isNil() ) {
+			return Label.INVALID;
+		}
+		Interval last = (Interval)intervals.get(intervals.size()-1);
+		return last.b;
+	}
+
+	/** Return minimum element >= 0 */
+	public int getMinElement() {
+		if ( isNil() ) {
+			return Label.INVALID;
+		}
+		int n = intervals.size();
+		for (int i = 0; i < n; i++) {
+			Interval I = (Interval) intervals.get(i);
+			int a = I.a;
+			int b = I.b;
+			for (int v=a; v<=b; v++) {
+				if ( v>=0 ) return v;
+			}
+		}
+		return Label.INVALID;
+	}
+
+    /** Return a list of Interval objects. */
+    public List<Interval> getIntervals() {
+        return intervals;
+    }
+
+    /** Are two IntervalSets equal?  Because all intervals are sorted
+     *  and disjoint, equals is a simple linear walk over both lists
+     *  to make sure they are the same.  Interval.equals() is used
+     *  by the List.equals() method to check the ranges.
+     */
+    public boolean equals(Object obj) {
+        if ( obj==null || !(obj instanceof IntervalSet) ) {
+            return false;
+        }
+        IntervalSet other = (IntervalSet)obj;
+        return this.intervals.equals(other.intervals);
+    }
+
+    public String toString() {
+        return toString(null);
+    }
+
+    public String toString(Grammar g) {
+        StringBuffer buf = new StringBuffer();
+		if ( this.intervals==null || this.intervals.size()==0 ) {
+			return "{}";
+		}
+        if ( this.intervals.size()>1 ) {
+            buf.append("{");
+        }
+        Iterator iter = this.intervals.iterator();
+        while (iter.hasNext()) {
+            Interval I = (Interval) iter.next();
+            int a = I.a;
+            int b = I.b;
+            if ( a==b ) {
+                if ( g!=null ) {
+                    buf.append(g.getTokenDisplayName(a));
+                }
+                else {
+                    buf.append(a);
+                }
+            }
+            else {
+                if ( g!=null ) {
+                    buf.append(g.getTokenDisplayName(a)+".."+g.getTokenDisplayName(b));
+                }
+                else {
+                    buf.append(a+".."+b);
+                }
+            }
+            if ( iter.hasNext() ) {
+                buf.append(", ");
+            }
+        }
+        if ( this.intervals.size()>1 ) {
+            buf.append("}");
+        }
+        return buf.toString();
+    }
+
+    public int size() {
+		int n = 0;
+		int numIntervals = intervals.size();
+		if ( numIntervals==1 ) {
+			Interval firstInterval = this.intervals.get(0);
+			return firstInterval.b-firstInterval.a+1;
+		}
+		for (int i = 0; i < numIntervals; i++) {
+			Interval I = (Interval) intervals.get(i);
+			n += (I.b-I.a+1);
+		}
+		return n;
+    }
+
+    public List toList() {
+		List values = new ArrayList();
+		int n = intervals.size();
+		for (int i = 0; i < n; i++) {
+			Interval I = (Interval) intervals.get(i);
+			int a = I.a;
+			int b = I.b;
+			for (int v=a; v<=b; v++) {
+				values.add(Utils.integer(v));
+			}
+		}
+		return values;
+    }
+
+	/** Get the ith element of ordered set.  Used only by RandomPhrase so
+	 *  don't bother to implement if you're not doing that for a new
+	 *  ANTLR code gen target.
+	 */
+	public int get(int i) {
+		int n = intervals.size();
+		int index = 0;
+		for (int j = 0; j < n; j++) {
+			Interval I = (Interval) intervals.get(j);
+			int a = I.a;
+			int b = I.b;
+			for (int v=a; v<=b; v++) {
+				if ( index==i ) {
+					return v;
+				}
+				index++;
+			}
+		}
+		return -1;
+	}
+
+	public int[] toArray() {
+		int[] values = new int[size()];
+		int n = intervals.size();
+		int j = 0;
+		for (int i = 0; i < n; i++) {
+			Interval I = (Interval) intervals.get(i);
+			int a = I.a;
+			int b = I.b;
+			for (int v=a; v<=b; v++) {
+				values[j] = v;
+				j++;
+			}
+		}
+		return values;
+	}
+
+	public org.antlr.runtime.BitSet toRuntimeBitSet() {
+		org.antlr.runtime.BitSet s =
+			new org.antlr.runtime.BitSet(getMaxElement()+1);
+		int n = intervals.size();
+		for (int i = 0; i < n; i++) {
+			Interval I = (Interval) intervals.get(i);
+			int a = I.a;
+			int b = I.b;
+			for (int v=a; v<=b; v++) {
+				s.add(v);
+			}
+		}
+		return s;
+	}
+
+	public void remove(int el) {
+        throw new NoSuchMethodError("IntervalSet.remove() unimplemented");
+    }
+
+	/*
+	protected void finalize() throws Throwable {
+		super.finalize();
+		System.out.println("size "+intervals.size()+" "+size());
+	}
+	*/
+}

tool/src/org/antlr/v4/misc/Utils.java

@@ -4,6 +4,24 @@ import java.util.Iterator;
 
 /** */
 public class Utils {
+	public static final int INTEGER_POOL_MAX_VALUE = 1000;
+	static Integer[] ints = new Integer[INTEGER_POOL_MAX_VALUE+1];
+
+	/** Integer objects are immutable so share all Integers with the
+	 *  same value up to some max size.  Use an array as a perfect hash.
+	 *  Return shared object for 0..INTEGER_POOL_MAX_VALUE or a new
+	 *  Integer object with x in it.  Java's autoboxing only caches up to 127.
+	 */
+	public static Integer integer(int x) {
+		if ( x<0 || x>INTEGER_POOL_MAX_VALUE ) {
+			return new Integer(x);
+		}
+		if ( ints[x]==null ) {
+			ints[x] = new Integer(x);
+		}
+		return ints[x];
+	}
+	
     public static String stripFileExtension(String name) {
         if ( name==null ) return null;
         int lastDot = name.lastIndexOf('.');

tool/src/org/antlr/v4/tool/Grammar.java

@@ -374,16 +374,6 @@ public class Grammar implements AttributeResolver {
 
 	public boolean resolvesToListLabel(String x, ActionAST node) { return false; }
 	
-	//	/** $x in grammar action can only be scope name */
-//    public boolean resolves(String x, ActionAST node) {
-//        return scopes.get(x)!=null;
-//    }
-//
-//    /** $x.y makes no sense in grammar action; Rule.resolves()
-//     *  shouldn't call this.
-//     */
-//    public boolean resolves(String x, String y, ActionAST node) { return false; }
-
     /** Given a grammar type, what should be the default action scope?
      *  If I say @members in a COMBINED grammar, for example, the
      *  default scope should be "parser".