pull in Sam's ATN alt collapsing optimizations with optimizeStates off as it causes a class cast exception.

runtime/Java/src/org/antlr/v4/runtime/atn/ATNState.java

@@ -31,7 +31,12 @@ package org.antlr.v4.runtime.atn;
 
 import org.antlr.v4.runtime.misc.IntervalSet;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 public class ATNState {
 	public static final int INITIAL_NUM_TRANSITIONS = 4;
@@ -137,6 +142,10 @@ public class ATNState {
 		transitions.set(i, e);
 	}
 
+	public Transition removeTransition(int index) {
+		return transitions.remove(index);
+	}
+
 	public int getStateType() {
 		return serializationTypes.get(this.getClass());
 	}

runtime/Java/src/org/antlr/v4/runtime/atn/ActionTransition.java

@@ -31,7 +31,7 @@ package org.antlr.v4.runtime.atn;
 
 import org.antlr.v4.runtime.misc.NotNull;
 
-public class ActionTransition extends Transition {
+public final class ActionTransition extends Transition {
 	public final int ruleIndex;
 	public final int actionIndex;
 	public final boolean isCtxDependent; // e.g., $i ref in action
@@ -47,6 +47,11 @@ public class ActionTransition extends Transition {
 		this.isCtxDependent = isCtxDependent;
 	}
 
+	@Override
+	public int getSerializationType() {
+		return ACTION;
+	}
+
 	@Override
 	public boolean isEpsilon() {
 		return true; // we are to be ignored by analysis 'cept for predicates

runtime/Java/src/org/antlr/v4/runtime/atn/AtomTransition.java

@@ -29,11 +29,11 @@
 
 package org.antlr.v4.runtime.atn;
 
-import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.misc.IntervalSet;
+import org.antlr.v4.runtime.misc.NotNull;
 
 /** TODO: make all transitions sets? no, should remove set edges */
-public class AtomTransition extends Transition {
+public final class AtomTransition extends Transition {
 	/** The token type or character value; or, signifies special label. */
 	public final int label;
 
@@ -42,6 +42,11 @@ public class AtomTransition extends Transition {
 		this.label = label;
 	}
 
+	@Override
+	public int getSerializationType() {
+		return ATOM;
+	}
+
 	@Override
 	@NotNull
 	public IntervalSet label() { return IntervalSet.of(label); }

runtime/Java/src/org/antlr/v4/runtime/atn/EpsilonTransition.java

@@ -31,9 +31,14 @@ package org.antlr.v4.runtime.atn;
 
 import org.antlr.v4.runtime.misc.NotNull;
 
-public class EpsilonTransition extends Transition {
+public final class EpsilonTransition extends Transition {
 	public EpsilonTransition(@NotNull ATNState target) { super(target); }
 
+	@Override
+	public int getSerializationType() {
+		return EPSILON;
+	}
+
 	@Override
 	public boolean isEpsilon() { return true; }
 

runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java

@@ -462,7 +462,8 @@ public class LexerATNSimulator extends ATNSimulator {
 
 	@Nullable
 	public ATNState getReachableTarget(Transition trans, int t) {
-		if ( trans instanceof AtomTransition ) {
+		switch (trans.getSerializationType()) {
+		case Transition.ATOM:
 			AtomTransition at = (AtomTransition)trans;
 			if ( at.label == t ) {
 				if ( debug ) {
@@ -471,8 +472,10 @@ public class LexerATNSimulator extends ATNSimulator {
 
 				return at.target;
 			}
-		}
+
-		else if ( trans.getClass() == RangeTransition.class ) {
+			return null;
+
+		case Transition.RANGE:
 			RangeTransition rt = (RangeTransition)trans;
 			if ( t>=rt.from && t<=rt.to ) {
 				if ( debug ) {
@@ -481,24 +484,44 @@ public class LexerATNSimulator extends ATNSimulator {
 
 				return rt.target;
 			}
-		}
+
-		else if ( trans instanceof SetTransition ) {
+			return null;
+
+		case Transition.SET:
 			SetTransition st = (SetTransition)trans;
-			boolean not = trans instanceof NotSetTransition;
+			if ( st.set.contains(t) ) {
-			if ( (!not && st.set.contains(t)) ||
-				 (not && !st.set.contains(t) && t!=CharStream.EOF) ) // ~set doesn't not match EOF
-			{
 				if ( debug ) {
-					System.out.format("match %sset %s\n", not ? "~" : "", st.set.toString(true));
+					System.out.format("match set %s\n", st.set.toString(true));
 				}
 
 				return st.target;
 			}
+
+			return null;
+
+		case Transition.NOT_SET:
+			NotSetTransition nst = (NotSetTransition)trans;
+			if (!nst.set.contains(t) && t!=CharStream.EOF) // ~set doesn't not match EOF
+			{
+				if ( debug ) {
+					System.out.format("match ~set %s\n", nst.set.toString(true));
 				}
-		else if ( trans instanceof WildcardTransition && t!=CharStream.EOF ) {
+
+				return nst.target;
+			}
+
+			return null;
+
+		case Transition.WILDCARD:
+			if (t != CharStream.EOF) {
 				return trans.target;
 			}
+
 			return null;
+
+		default:
+			return null;
+		}
 	}
 
 	/** Delete configs for alt following ci. Closure is unmodified; copy returned. */
@@ -591,16 +614,18 @@ public class LexerATNSimulator extends ATNSimulator {
 									  @NotNull ATNConfigSet configs)
 	{
 		ATNState p = config.state;
+
 		LexerATNConfig c = null;
-		if ( t.getClass() == RuleTransition.class ) {
+		switch (t.getSerializationType()) {
+			case Transition.RULE:
 				PredictionContext newContext =
 					new SingletonPredictionContext(config.context, p.stateNumber);
 				c = new LexerATNConfig(config, t.target, newContext);
-		}
+				break;
-		else if ( t.getClass() == PredicateTransition.class ) {
+			case Transition.PREDICATE:
-			if (recog == null) {
+//				if (recog == null) {
-				System.out.format("Predicates cannot be evaluated without a recognizer; assuming true.\n");
+//					System.out.format("Predicates cannot be evaluated without a recognizer; assuming true.\n");
-			}
+//				}
 
 				/*  Track traversing semantic predicates. If we traverse,
 				 we cannot add a DFA state for this "reach" computation
@@ -628,14 +653,16 @@ public class LexerATNSimulator extends ATNSimulator {
 				if ( recog == null || recog.sempred(null, pt.ruleIndex, pt.predIndex) ) {
 					c = new LexerATNConfig(config, t.target, pt.getPredicate());
 				}
-		}
+				break;
 			// ignore actions; just exec one per rule upon accept
-		else if ( t.getClass() == ActionTransition.class ) {
+			case Transition.ACTION:
 				c = new LexerATNConfig(config, t.target, ((ActionTransition)t).actionIndex);
-		}
+				break;
-		else if ( t.isEpsilon() ) {
+			case Transition.EPSILON:
 				c = new LexerATNConfig(config, t.target);
+				break;
 		}
+
 		return c;
 	}
 

runtime/Java/src/org/antlr/v4/runtime/atn/NotSetTransition.java

@@ -33,11 +33,16 @@ import org.antlr.v4.runtime.misc.IntervalSet;
 import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.misc.Nullable;
 
-public class NotSetTransition extends SetTransition {
+public final class NotSetTransition extends SetTransition {
 	public NotSetTransition(@NotNull ATNState target, @Nullable IntervalSet set) {
 		super(target, set);
 	}
 
+	@Override
+	public int getSerializationType() {
+		return NOT_SET;
+	}
+
 	@Override
 	public String toString() {
 		return '~'+super.toString();

runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java

@@ -911,27 +911,44 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 
 	@Nullable
 	public ATNState getReachableTarget(@NotNull Transition trans, int ttype) {
-		if ( trans instanceof AtomTransition ) {
+		switch (trans.getSerializationType()) {
+		case Transition.ATOM:
 			AtomTransition at = (AtomTransition)trans;
 			if ( at.label == ttype ) {
 				return at.target;
 			}
-		}
+			return null;
-		else if ( trans instanceof SetTransition ) {
+
+		case Transition.SET:
 			SetTransition st = (SetTransition)trans;
-			boolean not = trans instanceof NotSetTransition;
+			if ( st.set.contains(ttype) ) {
-			if ( !not && st.set.contains(ttype) || not && !st.set.contains(ttype) ) {
 				return st.target;
 			}
+			return null;
+
+		case Transition.NOT_SET:
+			NotSetTransition nst = (NotSetTransition)trans;
+			if ( !nst.set.contains(ttype) ) {
+				return nst.target;
 			}
-		else if ( trans instanceof RangeTransition ) { // TODO: can't happen in parser, right? remove
+			return null;
+
+		case Transition.RANGE:
 			RangeTransition rt = (RangeTransition)trans;
-			if ( ttype>=rt.from && ttype<=rt.to ) return rt.target;
+			if ( ttype>=rt.from && ttype<=rt.to ) {
+				return rt.target;
 			}
-		else if ( trans instanceof WildcardTransition && ttype!=Token.EOF ) {
+			return null;
+
+		case Transition.WILDCARD:
+			if (ttype != Token.EOF) {
 				return trans.target;
 			}
 			return null;
+
+		default:
+			return null;
+		}
 	}
 
 	public SemanticContext[] getPredsForAmbigAlts(@NotNull IntervalSet ambigAlts,
@@ -1214,23 +1231,26 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 									  boolean inContext,
 									  boolean fullCtx)
 	{
-		if ( t instanceof RuleTransition ) {
+		switch (t.getSerializationType()) {
+		case Transition.RULE:
 			return ruleTransition(config, t);
-		}
+
-		else if ( t instanceof PredicateTransition ) {
+		case Transition.PREDICATE:
 			return predTransition(config, (PredicateTransition)t,
 								  collectPredicates,
 								  inContext,
 								  fullCtx);
-		}
+
-		else if ( t instanceof ActionTransition ) {
+		case Transition.ACTION:
 			return actionTransition(config, (ActionTransition)t);
-		}
+
-		else if ( t.isEpsilon() ) {
+		case Transition.EPSILON:
 			return new ATNConfig(config, t.target);
-		}
+
+		default:
 			return null;
 		}
+	}
 
 	@NotNull
 	public ATNConfig actionTransition(@NotNull ATNConfig config, @NotNull ActionTransition t) {

runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java

@@ -37,7 +37,7 @@ import org.antlr.v4.runtime.misc.NotNull;
  *  may have to combine a bunch of them as it collects predicates from
  *  multiple ATN configurations into a single DFA state.
  */
-public class PredicateTransition extends Transition {
+public final class PredicateTransition extends Transition {
 	public final int ruleIndex;
 	public final int predIndex;
 	public final boolean isCtxDependent;  // e.g., $i ref in pred
@@ -49,6 +49,11 @@ public class PredicateTransition extends Transition {
 		this.isCtxDependent = isCtxDependent;
 	}
 
+	@Override
+	public int getSerializationType() {
+		return PREDICATE;
+	}
+
 	@Override
 	public boolean isEpsilon() { return true; }
 

runtime/Java/src/org/antlr/v4/runtime/atn/RangeTransition.java

@@ -29,10 +29,10 @@
 
 package org.antlr.v4.runtime.atn;
 
-import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.misc.IntervalSet;
+import org.antlr.v4.runtime.misc.NotNull;
 
-public class RangeTransition extends Transition {
+public final class RangeTransition extends Transition {
 	public final int from;
 	public final int to;
 
@@ -42,6 +42,11 @@ public class RangeTransition extends Transition {
 		this.to = to;
 	}
 
+	@Override
+	public int getSerializationType() {
+		return RANGE;
+	}
+
 	@Override
 	@NotNull
 	public IntervalSet label() { return IntervalSet.of(from, to); }

runtime/Java/src/org/antlr/v4/runtime/atn/RuleTransition.java

@@ -32,7 +32,7 @@ package org.antlr.v4.runtime.atn;
 import org.antlr.v4.runtime.misc.NotNull;
 
 /** */
-public class RuleTransition extends Transition {
+public final class RuleTransition extends Transition {
 	/** Ptr to the rule definition object for this rule ref */
 	public final int ruleIndex;     // no Rule object at runtime
 
@@ -49,6 +49,11 @@ public class RuleTransition extends Transition {
 		this.followState = followState;
 	}
 
+	@Override
+	public int getSerializationType() {
+		return RULE;
+	}
+
 	@Override
 	public boolean isEpsilon() { return true; }
 }

runtime/Java/src/org/antlr/v4/runtime/atn/SetTransition.java

@@ -29,10 +29,10 @@
 
 package org.antlr.v4.runtime.atn;
 
-import org.antlr.v4.runtime.misc.NotNull;
-import org.antlr.v4.runtime.misc.Nullable;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.misc.IntervalSet;
+import org.antlr.v4.runtime.misc.NotNull;
+import org.antlr.v4.runtime.misc.Nullable;
 
 /** A transition containing a set of values */
 public class SetTransition extends Transition {
@@ -46,6 +46,11 @@ public class SetTransition extends Transition {
 		this.set = set;
 	}
 
+	@Override
+	public int getSerializationType() {
+		return SET;
+	}
+
 	@Override
 	@NotNull
 	public IntervalSet label() { return set; }

runtime/Java/src/org/antlr/v4/runtime/atn/Transition.java

@@ -33,7 +33,11 @@ import org.antlr.v4.runtime.misc.IntervalSet;
 import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.misc.Nullable;
 
-import java.util.*;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 /** An ATN transition between any two ATN states.  Subclasses define
  *  atom, set, epsilon, action, predicate, rule transitions.
@@ -99,7 +103,7 @@ public abstract class Transition {
 		this.target = target;
 	}
 
-	public int getSerializationType() { return 0; }
+	public abstract int getSerializationType();
 
 	/** Are we epsilon, action, sempred? */
 	public boolean isEpsilon() { return false; }

runtime/Java/src/org/antlr/v4/runtime/atn/WildcardTransition.java

@@ -31,9 +31,14 @@ package org.antlr.v4.runtime.atn;
 
 import org.antlr.v4.runtime.misc.NotNull;
 
-public class WildcardTransition extends Transition {
+public final class WildcardTransition extends Transition {
 	public WildcardTransition(@NotNull ATNState target) { super(target); }
 
+	@Override
+	public int getSerializationType() {
+		return WILDCARD;
+	}
+
 	@Override
 	@NotNull
 	public String toString() {

tool/playground/A.g4

@@ -2,8 +2,8 @@ grammar A;
 
 s : e ;
 
-e : e '*' e 		-> Mult
+e : e '*' e 		# Mult
-  | INT        		-> primary
+  | INT        		# primary
   ;
 
 INT : [0-9]+ ;

tool/playground/LL1.g

@@ -1,14 +0,0 @@
-grammar LL1;
-
-b : B | C ;
-
-c1 : A? B ;
-
-c2 : (B|C)? D ;
-
-d1 : A* B ;
-
-d2 : ({true}? B | {true}? A)* D {System.out.println("works!");} ;
-
-e1 : A+ B ;
-e2 : (B|A)+ D ;

tool/src/org/antlr/v4/automata/ATNFactory.java

@@ -31,7 +31,11 @@ package org.antlr.v4.automata;
 
 import org.antlr.v4.runtime.atn.ATN;
 import org.antlr.v4.runtime.atn.ATNState;
-import org.antlr.v4.tool.ast.*;
+import org.antlr.v4.tool.ast.ActionAST;
+import org.antlr.v4.tool.ast.BlockAST;
+import org.antlr.v4.tool.ast.GrammarAST;
+import org.antlr.v4.tool.ast.PredAST;
+import org.antlr.v4.tool.ast.TerminalAST;
 
 import java.util.List;
 
@@ -74,9 +78,6 @@ public interface ATNFactory {
 
 	Handle charSetLiteral(GrammarAST charSetAST);
 
-
-	Handle tree(GrammarAST node, List<Handle> els);
-
 	Handle range(GrammarAST a, GrammarAST b);
 
 	/** For a non-lexer, just build a simple token reference atom.

tool/src/org/antlr/v4/automata/ATNOptimizer.java

@@ -0,0 +1,191 @@
+/*
+ [The "BSD license"]
+  Copyright (c) 2012 Terence Parr
+  Copyright (c) 2012 Sam Harwell
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+
+  1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+  2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+  3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.v4.automata;
+
+import org.antlr.v4.runtime.atn.ATN;
+import org.antlr.v4.runtime.atn.ATNState;
+import org.antlr.v4.runtime.atn.AtomTransition;
+import org.antlr.v4.runtime.atn.BlockEndState;
+import org.antlr.v4.runtime.atn.DecisionState;
+import org.antlr.v4.runtime.atn.EpsilonTransition;
+import org.antlr.v4.runtime.atn.NotSetTransition;
+import org.antlr.v4.runtime.atn.RangeTransition;
+import org.antlr.v4.runtime.atn.SetTransition;
+import org.antlr.v4.runtime.atn.Transition;
+import org.antlr.v4.runtime.misc.Interval;
+import org.antlr.v4.runtime.misc.IntervalSet;
+import org.antlr.v4.runtime.misc.NotNull;
+import org.antlr.v4.tool.Grammar;
+import org.antlr.v4.tool.Rule;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ * @author Sam Harwell
+ */
+public class ATNOptimizer {
+
+	public static void optimize(@NotNull Grammar g, @NotNull ATN atn) {
+		optimizeSets(g, atn);
+//		optimizeStates(atn); // TODO leads to class cast exception
+	}
+
+	private static void optimizeSets(Grammar g, ATN atn) {
+		if (g.isParser()) {
+			// parser codegen doesn't currently support SetTransition
+			return;
+		}
+
+		int removedStates = 0;
+		List<DecisionState> decisions = atn.decisionToState;
+		for (DecisionState decision : decisions) {
+			if (decision.ruleIndex >= 0) {
+				Rule rule = g.getRule(decision.ruleIndex);
+				if (Character.isLowerCase(rule.name.charAt(0))) {
+					// parser codegen doesn't currently support SetTransition
+					continue;
+				}
+			}
+
+			IntervalSet setTransitions = new IntervalSet();
+			for (int i = 0; i < decision.getNumberOfTransitions(); i++) {
+				Transition epsTransition = decision.transition(i);
+				if (!(epsTransition instanceof EpsilonTransition)) {
+					continue;
+				}
+
+				if (epsTransition.target.getNumberOfTransitions() != 1) {
+					continue;
+				}
+
+				Transition transition = epsTransition.target.transition(0);
+				if (!(transition.target instanceof BlockEndState)) {
+					continue;
+				}
+
+				if (transition instanceof NotSetTransition) {
+					// TODO: not yet implemented
+					continue;
+				}
+
+				if (transition instanceof AtomTransition
+					|| transition instanceof RangeTransition
+					|| transition instanceof SetTransition)
+				{
+					setTransitions.add(i);
+				}
+			}
+
+			// due to min alt resolution policies, can only collapse sequential alts
+			for (int i = setTransitions.getIntervals().size() - 1; i >= 0; i--) {
+				Interval interval = setTransitions.getIntervals().get(i);
+				if (interval.length() <= 1) {
+					continue;
+				}
+
+				ATNState blockEndState = decision.transition(interval.a).target.transition(0).target;
+				IntervalSet matchSet = new IntervalSet();
+				for (int j = interval.a; j <= interval.b; j++) {
+					Transition matchTransition = decision.transition(j).target.transition(0);
+					if (matchTransition instanceof NotSetTransition) {
+						throw new UnsupportedOperationException("Not yet implemented.");
+					} else {
+						matchSet.addAll(matchTransition.label());
+					}
+				}
+
+				Transition newTransition;
+				if (matchSet.getIntervals().size() == 1) {
+					if (matchSet.size() == 1) {
+						newTransition = new AtomTransition(blockEndState, matchSet.getMinElement());
+					} else {
+						Interval matchInterval = matchSet.getIntervals().get(0);
+						newTransition = new RangeTransition(blockEndState, matchInterval.a, matchInterval.b);
+					}
+				} else {
+					newTransition = new SetTransition(blockEndState, matchSet);
+				}
+
+				decision.transition(interval.a).target.setTransition(0, newTransition);
+				for (int j = interval.a + 1; j <= interval.b; j++) {
+					Transition removed = decision.removeTransition(interval.a + 1);
+					atn.removeState(removed.target);
+					removedStates++;
+				}
+			}
+		}
+
+//		System.out.println("ATN optimizer removed " + removedStates + " states by collapsing sets.");
+	}
+
+	private static void optimizeStates(ATN atn) {
+		System.out.println(atn.states);
+		List<ATNState> compressed = new ArrayList<ATNState>();
+		int i = 0; // new state number
+		for (ATNState s : atn.states) {
+			if ( s!=null ) {
+				compressed.add(s);
+				s.stateNumber = i; // reset state number as we shift to new position
+				i++;
+			}
+		}
+		System.out.println(compressed);
+//		System.out.println("ATN optimizer removed " + (atn.states.size() - compressed.size()) + " null states.");
+		atn.states.clear();
+		atn.states.addAll(compressed);
+
+//		List<ATNState> states = atn.states;
+//		int current = 0;
+//		for (int i = 0; i < states.size(); i++) {
+//			ATNState state = states.get(i);
+//			if (state == null) {
+//				continue;
+//			}
+//
+//			if (i != current) {
+//				state.stateNumber = current;
+//				states.set(current, state);
+//				states.set(i, null);
+//			}
+//
+//			current++;
+//		}
+//
+//		System.out.println("ATN optimizer removed " + (states.size() - current) + " null states.");
+//		states.subList(current, states.size()).clear();
+	}
+
+	private ATNOptimizer() {
+	}
+
+}

tool/src/org/antlr/v4/automata/ATNVisitor.java

@@ -57,5 +57,5 @@ public class ATNVisitor {
 		}
 	}
 
-	public void visitState(ATNState s) { }
+	public void visitState(@NotNull ATNState s) { }
 }

tool/src/org/antlr/v4/automata/LexerATNFactory.java

@@ -104,6 +104,7 @@ public class LexerATNFactory extends ParserATNFactory {
 			}
 		}
 
+		ATNOptimizer.optimize(g, atn);
 		return atn;
 	}
 

tool/src/org/antlr/v4/automata/ParserATNFactory.java

@@ -83,41 +83,6 @@ import java.util.List;
  *  No side-effects. It builds an ATN object and returns it.
  */
 public class ParserATNFactory implements ATNFactory {
-    class TailEpsilonRemover extends ATNVisitor {
-        @Override
-        public void visitState(ATNState p) {
-            if ( p.getClass() == ATNState.class && p.getNumberOfTransitions()==1 ) {
-                ATNState q = p.transition(0).target;
-				if ( p.transition(0) instanceof RuleTransition ) {
-					q = ((RuleTransition)p.transition(0)).followState;
-				}
-				if ( q.getClass() == ATNState.class ) {
-					// we have p-x->q for x in {rule, action, pred, token, ...}
-					// if edge out of q is single epsilon to block end
-					// we can strip epsilon p-x->q-eps->r
-					Transition trans = q.transition(0);
-					if ( q.getNumberOfTransitions()==1 && trans.isEpsilon() &&
-					     !(trans instanceof ActionTransition) ) {
-						ATNState r = trans.target;
-						if ( r instanceof BlockEndState ||
-							r instanceof PlusLoopbackState ||
-							r instanceof StarLoopbackState )
-						{
-							// skip over q
-							if ( p.transition(0) instanceof RuleTransition ) {
-								((RuleTransition)p.transition(0)).followState = r;
-							}
-							else {
-								p.transition(0).target = r;
-							}
-							atn.removeState(q);
-						}
-					}
-				}
-			}
-		}
-	}
-
 	@NotNull
 	public final Grammar g;
 
@@ -128,7 +93,14 @@ public class ParserATNFactory implements ATNFactory {
 
 	public int currentOuterAlt;
 
-	public ParserATNFactory(@NotNull Grammar g) { this.g = g; atn = new ATN(); }
+	public ParserATNFactory(@NotNull Grammar g) {
+		if (g == null) {
+			throw new NullPointerException("g");
+		}
+
+		this.g = g;
+		this.atn = new ATN();
+	}
 
 	@Override
 	public ATN createATN() {
@@ -136,6 +108,7 @@ public class ParserATNFactory implements ATNFactory {
 		atn.maxTokenType = g.getMaxTokenType();
         addRuleFollowLinks();
 		addEOFTransitionToStartRules();
+		ATNOptimizer.optimize(g, atn);
 		return atn;
 	}
 
@@ -218,11 +191,6 @@ public class ParserATNFactory implements ATNFactory {
 		return new Handle(left, right);
 	}
 
-	@Override
-	public Handle tree(GrammarAST node, List<Handle> els) {
-		throw new UnsupportedOperationException("^(...) not allowed in non-tree grammar");
-	}
-
 	/** Not valid for non-lexers */
 	@Override
 	public Handle range(GrammarAST a, GrammarAST b) {
@@ -316,7 +284,7 @@ public class ParserATNFactory implements ATNFactory {
 
 	/** Build what amounts to an epsilon transition with an action.
 	 *  The action goes into ATN though it is ignored during prediction
-	 *  if actionIndex < 0.  Only forced are executed during prediction.
+	 *  if actionIndex &lt; 0.  Only forced are executed during prediction.
 	 */
 	@Override
 	public Handle action(ActionAST action) {
@@ -351,7 +319,7 @@ public class ParserATNFactory implements ATNFactory {
 	 *  begin/end.
 	 *
 	 *  Special case: if just a list of tokens/chars/sets, then collapse
-	 *  to a single edge'd o-set->o graph.
+	 *  to a single edged o-set->o graph.
 	 *
 	 *  TODO: Set alt number (1..n) in the states?
 	 */
@@ -396,7 +364,7 @@ public class ParserATNFactory implements ATNFactory {
 			epsilon(alt.right, end);
 			// no back link in ATN so must walk entire alt to see if we can
 			// strip out the epsilon to 'end' state
-			TailEpsilonRemover opt = new TailEpsilonRemover();
+			TailEpsilonRemover opt = new TailEpsilonRemover(atn);
 			opt.visit(alt.left);
 		}
 		Handle h = new Handle(start, end);

tool/src/org/antlr/v4/automata/TailEpsilonRemover.java

@@ -0,0 +1,81 @@
+/*
+ [The "BSD license"]
+  Copyright (c) 2012 Terence Parr
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+
+  1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+  2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+  3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.v4.automata;
+
+import org.antlr.v4.runtime.atn.ATN;
+import org.antlr.v4.runtime.atn.ATNState;
+import org.antlr.v4.runtime.atn.ActionTransition;
+import org.antlr.v4.runtime.atn.BlockEndState;
+import org.antlr.v4.runtime.atn.PlusLoopbackState;
+import org.antlr.v4.runtime.atn.RuleTransition;
+import org.antlr.v4.runtime.atn.StarLoopbackState;
+import org.antlr.v4.runtime.atn.Transition;
+import org.antlr.v4.runtime.misc.NotNull;
+
+/**
+ *
+ * @author Terence Parr
+ */
+public class TailEpsilonRemover extends ATNVisitor {
+	@NotNull
+	private final ATN _atn;
+
+	public TailEpsilonRemover(@NotNull ATN atn) {
+		this._atn = atn;
+	}
+
+	@Override
+	public void visitState(@NotNull ATNState p) {
+		if (p.getClass() == ATNState.class && p.getNumberOfTransitions() == 1) {
+			ATNState q = p.transition(0).target;
+			if (p.transition(0) instanceof RuleTransition) {
+				q = ((RuleTransition) p.transition(0)).followState;
+			}
+			if (q.getClass() == ATNState.class) {
+				// we have p-x->q for x in {rule, action, pred, token, ...}
+				// if edge out of q is single epsilon to block end
+				// we can strip epsilon p-x->q-eps->r
+				Transition trans = q.transition(0);
+				if (q.getNumberOfTransitions() == 1 && trans.isEpsilon() && !(trans instanceof ActionTransition)) {
+					ATNState r = trans.target;
+					if (r instanceof BlockEndState || r instanceof PlusLoopbackState || r instanceof StarLoopbackState) {
+						// skip over q
+						if (p.transition(0) instanceof RuleTransition) {
+							((RuleTransition) p.transition(0)).followState = r;
+						} else {
+							p.transition(0).target = r;
+						}
+						_atn.removeState(q);
+					}
+				}
+			}
+		}
+	}
+}