reorged a bit; improved msg about recur ovf.

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 6768]

tool/resources/org/antlr/v4/tool/templates/messages/languages/en.stg

@@ -276,8 +276,8 @@ ANALYSIS_TIMEOUT(arg) ::= <<
 ANTLR could not analyze this decision in rule <arg.enclosingRule>; often this is because of recursive rule references visible from the left edge of alternatives.  ANTLR will re-analyze the decision with a fixed lookahead of k=1.  Consider using "options {k=1;}" for that decision and possibly adding a syntactic predicate 
 >>
 
-RECURSION_OVERFLOW(alt,input,targetRules,callSiteStates) ::= <<
+RECURSION_OVERFLOW(arg) ::= <<
-Alternative <alt>: after matching input such as <input> decision cannot predict what comes next due to recursion overflow <targetRules,callSiteStates:{t,c|to <t> from <c:{s|<s.enclosingRule.name>};separator=", ">}; separator=" and ">
+Recursion overflow to <arg.targetRule.name> from alternative <arg.alt> of <arg.sourceRule.name> after matching input such as <arg.input>
 >>
 
 LEFT_RECURSION_CYCLES(arg) ::= <<

tool/src/org/antlr/v4/Tool.java

@@ -21,26 +21,26 @@ public class Tool {
     public String VERSION = "!Unknown version!";
     //public static final String VERSION = "${project.version}";
     public static final String UNINITIALIZED_DIR = "<unset-dir>";
-    private List<String> grammarFileNames = new ArrayList<String>();
+    public List<String> grammarFileNames = new ArrayList<String>();
-    private boolean generate_NFA_dot = false;
+    public boolean generate_NFA_dot = false;
-    private boolean generate_DFA_dot = false;
+    public boolean generate_DFA_dot = false;
-    private String outputDirectory = ".";
+    public String outputDirectory = ".";
-    private boolean haveOutputDir = false;
+    public boolean haveOutputDir = false;
-    private String inputDirectory = null;
+    public String inputDirectory = null;
-    private String parentGrammarDirectory;
+    public String parentGrammarDirectory;
-    private String grammarOutputDirectory;
+    public String grammarOutputDirectory;
-    private boolean haveInputDir = false;
+    public boolean haveInputDir = false;
-    private String libDirectory = ".";
+    public String libDirectory = ".";
-    private boolean debug = false;
+    public boolean debug = false;
-    private boolean trace = false;
+    public boolean trace = false;
-    private boolean profile = false;
+    public boolean profile = false;
-    private boolean report = false;
+    public boolean report = false;
-    private boolean printGrammar = false;
+    public boolean printGrammar = false;
-    private boolean depend = false;
+    public boolean depend = false;
-    private boolean forceAllFilesToOutputDir = false;
+    public boolean forceAllFilesToOutputDir = false;
-    private boolean forceRelativeOutput = false;
+    public boolean forceRelativeOutput = false;
-    protected boolean deleteTempLexer = true;
+    public boolean deleteTempLexer = true;
-    private boolean verbose = false;
+    public boolean verbose = false;
     /** Don't process grammar file if generated files are newer than grammar */
     /**
      * Indicate whether the tool should analyze the dependencies of the provided grammar
@@ -57,11 +57,11 @@ public class Tool {
      *
      * @param make
      */
-     private boolean make = false;
+     public boolean make = false;
-    private boolean showBanner = true;
+    public boolean showBanner = true;
 
     /** Exit after showing version or whatever */ 
-    private static boolean exitNow = false;
+    public static boolean exitNow = false;
 
     // The internal options are for my use on the command line during dev
     public static boolean internalOption_PrintGrammarTree = false;
@@ -507,13 +507,20 @@ public class Tool {
 
 	public void generateDFAs(Grammar g) {
 		for (DFA dfa : g.decisionDFAs.values()) {
+			generateDFA(g, dfa);
+		}
+	}
+
+	public void generateDFA(Grammar g, DFA dfa) {
 		DOTGenerator dotGenerator = new DOTGenerator(g);
 		String dot = dotGenerator.getDOT(dfa.startState);
-			String dotFileName = g.name + "." + "dec-" + dfa.decision;
+		String dec = "dec-";
+		//if ( dfa.minimized ) dec += "min-";
+		String dotFileName = g.name + "." + dec + dfa.decision;
 		if (g.implicitLexer!=null) {
 			dotFileName = g.name +
 						  Grammar.getGrammarTypeToFileNameSuffix(g.getType()) +
-							  "." + "dec-" + dfa.decision;
+						  "." + dec + dfa.decision;
 		}
 		try {
 			writeDOTFile(g, dotFileName, dot);
@@ -522,7 +529,6 @@ public class Tool {
 			ErrorManager.toolError(ErrorType.CANNOT_WRITE_FILE, dotFileName, ioe);
 		}
 	}
-	}
 
 	protected void writeDOTFile(Grammar g, Rule r, String dot) throws IOException {
 		writeDOTFile(g, r.g.name + "." + r.name, dot);
@@ -649,11 +655,11 @@ public class Tool {
 		return outputDir;
 	}	
 	
-    private static void version() {
+    public static void version() {
         ErrorManager.info("ANTLR Parser Generator  Version " + new Tool().VERSION);
     }
 
-    private static void help() {
+    public static void help() {
         ErrorManager.info("ANTLR Parser Generator  Version " + new Tool().VERSION);
         System.err.println("usage: java org.antlr.Tool [args] file.g [file2.g file3.g ...]");
         System.err.println("  -o outputDir          specify output directory where all output is generated");
@@ -673,7 +679,7 @@ public class Tool {
         System.err.println("  -X                    display extended argument list");
     }
 
-    private static void Xhelp() {
+    public static void Xhelp() {
         ErrorManager.info("ANTLR Parser Generator  Version " + new Tool().VERSION);
         System.err.println("  -Xgrtree                print the grammar AST");
         System.err.println("  -Xdfa                   print DFA as text ");

tool/src/org/antlr/v4/analysis/AnalysisPipeline.java

@@ -2,7 +2,6 @@ package org.antlr.v4.analysis;
 
 import org.antlr.v4.automata.DFA;
 import org.antlr.v4.automata.DecisionState;
-import org.antlr.v4.tool.ErrorManager;
 import org.antlr.v4.tool.Grammar;
 
 public class AnalysisPipeline {
@@ -27,7 +26,7 @@ public class AnalysisPipeline {
 	}
 
 	public DFA createDFA(DecisionState s) {
-		// TRY APPROXIMATE LL(*) ANALYSIS
+		// TRY APPROXIMATE (STACK LIMITED) LL(*) ANALYSIS
 		StackLimitedNFAToDFAConverter conv = new StackLimitedNFAToDFAConverter(g, s);
 		DFA dfa = conv.createDFA();
 		System.out.print("DFA="+dfa);
@@ -41,7 +40,7 @@ public class AnalysisPipeline {
 
 			System.out.println("MINIMIZE");
 			DFAMinimizer dmin = new DFAMinimizer(dfa);
-			dmin.minimize();
+			dfa.minimized = dmin.minimize();
 
 			return dfa;
 		}
@@ -50,21 +49,10 @@ public class AnalysisPipeline {
 		// limited version.  Ambiguities are ok because if the approx version
 		// gets an ambiguity it's defin
 		
-		// REAL LL(*) ANALYSIS IF THAT FAILS
+		// RECURSION LIMITED LL(*) ANALYSIS IF THAT FAILS
 		conv = new RecursionLimitedNFAToDFAConverter(g, s);
-		try {
 		dfa = conv.createDFA();
 		System.out.print("DFA="+dfa);
-		}
-		catch (RecursionOverflowSignal ros) {
-			ErrorManager.recursionOverflow(g.fileName, dfa, ros.state, ros.altNum, ros.depth);
-		}
-		catch (MultipleRecursiveAltsSignal mras) {
-			ErrorManager.multipleRecursiveAlts(g.fileName, dfa, mras.recursiveAltSet);
-		}
-		catch (AnalysisTimeoutSignal at) {// TODO: nobody throws yet
-			ErrorManager.analysisTimeout();
-		}
 
 		conv.issueAmbiguityWarnings(); // ambig / unreachable errors
 		//conv.issueRecursionWarnings();
@@ -72,13 +60,16 @@ public class AnalysisPipeline {
 			System.out.println("non-LL(*)");
 			System.out.println("recursion limited NOT valid");
 		}
-		else System.out.println("recursion limited valid");
+		else {
-
+			System.out.println("recursion limited valid");
+			// gen DOT now to see non-minimized
+			System.out.println("DFA #states="+dfa.stateSet.size());
+			//if ( g.tool.generate_NFA_dot ) g.tool.generateDFA(g, dfa);
 			System.out.println("MINIMIZE");
 			DFAMinimizer dmin = new DFAMinimizer(dfa);
-		dmin.minimize();
+			dfa.minimized = dmin.minimize();
-		
+			if ( dfa.minimized ) System.out.println("DFA minimized to #states="+dfa.stateSet.size());			
+		}
 		return dfa;
 	}
-
 }

tool/src/org/antlr/v4/analysis/DFAMinimizer.java

@@ -23,7 +23,7 @@ public class DFAMinimizer {
 		this.dfa = dfa;
 	}
 
-	public void minimize() {
+	public boolean minimize() {
 		int n = dfa.states.size();
 		boolean[][] distinct = new boolean[n][n];
 
@@ -156,7 +156,7 @@ public class DFAMinimizer {
 		System.out.println("uniq sets = "+uniq);
 		if ( uniq.size()==dfa.states.size() ) {
 			System.out.println("was already minimal");
-			return;
+			return false;
 		}
 
 		// minimize the DFA (combine equiv sets)
@@ -207,6 +207,7 @@ public class DFAMinimizer {
 				}
 			}
 		}
+		return true;
 	}
 
 	void print(boolean[][] distinct) {

tool/src/org/antlr/v4/analysis/DFAVerifier.java

@@ -1,165 +0,0 @@
-package org.antlr.v4.analysis;
-
-import org.antlr.v4.automata.DFA;
-import org.antlr.v4.automata.DFAState;
-import org.antlr.v4.automata.Edge;
-import org.antlr.v4.misc.Utils;
-import org.stringtemplate.v4.misc.MultiMap;
-
-import java.util.*;
-
-/** Detect imperfect DFA:
- *
- *  1. nonreduced DFA (dangling states)
- *  2. unreachable stop states
- *  3. nondeterministic states
- *
- *  TODO: unneeded?
- */
-public class DFAVerifier {
-	public static enum ReachableStatus {
-		UNKNOWN,
-		BUSY, // in process of computing
-		NO,
-		YES;
-	}
-	
-	Map<DFAState, ReachableStatus> status = new HashMap<DFAState, ReachableStatus>();
-	
-	DFA dfa;
-
-	StackLimitedNFAToDFAConverter converter;
-
-	// create 2D matrix showing incident edges; inverse of adjacency matrix
-	// incidentEdges.get(s) is list of edges pointing at state s
-	MultiMap<DFAState, DFAState> incidentStates = new MultiMap<DFAState, DFAState>();
-	
-	public DFAVerifier(DFA dfa, StackLimitedNFAToDFAConverter converter) {
-		this.dfa = dfa;
-		this.converter = converter;
-		for (DFAState d : dfa.stateSet.values()) {
-			for (Edge e : d.edges) incidentStates.map(e.target, d);
-		}
-	}
-
-	public Set<Integer> getUnreachableAlts() {
-		Set<Integer> unreachable = new HashSet<Integer>();
-		for (int alt=0; alt<=dfa.nAlts; alt++) {
-			if ( dfa.altToAcceptStates[alt]==null ) unreachable.add(alt);
-		}
-		return unreachable; 
-	}
-
-	public List<DFAState> getIncidentEdgeStates(DFAState d) {
-		return incidentStates.get(d);
-	}
-	
-	public Set<DFAState> getDeadStates() {
-		Set<DFAState> dead = new HashSet<DFAState>(dfa.stateSet.size());
-		dead.addAll(dfa.stateSet.values());
-//		for (DFAState a : dfa.altToAcceptState) {
-//			if ( a!=null ) dead.remove(a);
-//		}
-
-		// obviously accept states reach accept states
-		//reaches.addAll(Arrays.asList(dfa.altToAcceptState));
-
-		boolean changed = true;
-		while ( changed ) {
-			changed = false;
-			for (DFAState d : dfa.stateSet.values()) {
-				if ( !dead.contains(d) ) {
-					// if d isn't dead, it reaches accept state.
-					dead.remove(d);
-					// and, so do all states pointing at it
-					List<DFAState> incoming = incidentStates.get(d);
-					if ( incoming!=null ) dead.removeAll(incoming);
-					changed = true;
-				}
-			}
-		}
-
-//		boolean changed = true;
-//		while ( changed ) {
-//			changed = false;
-//			for (DFAState d : dfa.uniqueStates.values()) {
-//				if ( reaches.contains(d) ) {
-//					dead.remove(d);
-//					// if d reaches, so do all states pointing at it
-//					for (DFAState i : incidentStates.get(d)) {
-//						if ( !reaches.contains(i) ) {
-//							reaches.add(i);
-//							changed = true;
-//						}
-//					}
-//				}
-//			}
-//		}
-
-		System.out.println("dead="+dead);
-		return dead;
-	}
-
-	/** figure out if this state eventually reaches an accept state and
-	 *  modify the instance variable 'reduced' to indicate if we find
-	 *  at least one state that cannot reach an accept state.  This implies
-	 *  that the overall DFA is not reduced.  This algorithm should be
-	 *  linear in the number of DFA states.
-	 *
-	 *  The algorithm also tracks which alternatives have no accept state,
-	 *  indicating a nondeterminism.
-	 *
-	 *  Also computes whether the DFA is cyclic.
-	 *
-	 *  TODO: I call getUniquelyPredicatedAlt too much; cache predicted alt
-	 *  TODO: convert to nonrecursive version.
-	 */
-	 boolean _verify(DFAState d) { // TODO: rename to verify?
-		if ( d.isAcceptState ) {
-			// accept states have no edges emanating from them so we can return
-			status.put(d, ReachableStatus.YES);
-			// this alt is uniquely predicted, remove from nondeterministic list
-			int predicts = d.getUniquelyPredictedAlt();
-			converter.unreachableAlts.remove(Utils.integer(predicts));
-			return true;
-		}
-
-		// avoid infinite loops
-		status.put(d, ReachableStatus.BUSY);
-
-		boolean anEdgeReachesAcceptState = false;
-		// Visit every transition, track if at least one edge reaches stop state
-		// Cannot terminate when we know this state reaches stop state since
-		// all transitions must be traversed to set status of each DFA state.
-		for (int i=0; i<d.getNumberOfEdges(); i++) {
-			Edge t = d.edge(i);
-			DFAState edgeTarget = (DFAState)t.target;
-			ReachableStatus targetStatus = status.get(edgeTarget);
-			if ( targetStatus==ReachableStatus.BUSY ) { // avoid cycles; they say nothing
-				converter.cyclic = true;
-				continue;
-			}
-			if ( targetStatus==ReachableStatus.YES ) { // avoid unnecessary work
-				anEdgeReachesAcceptState = true;
-				continue;
-			}
-			if ( targetStatus==ReachableStatus.NO ) {  // avoid unnecessary work
-				continue;
-			}
-			// target must be ReachableStatus.UNKNOWN (i.e., unvisited)
-			if ( _verify(edgeTarget) ) {
-				anEdgeReachesAcceptState = true;
-				// have to keep looking so don't break loop
-				// must cover all states even if we find a path for this state
-			}
-		}
-		if ( anEdgeReachesAcceptState ) {
-			status.put(d, ReachableStatus.YES);
-		}
-		else {
-			status.put(d, ReachableStatus.NO);
-			converter.reduced = false;
-		}
-		return anEdgeReachesAcceptState;
-	}
-}

tool/src/org/antlr/v4/analysis/MachineProbe.java

@@ -49,6 +49,7 @@ public class MachineProbe {
 	public List<IntSet> getEdgeLabels(DFAState targetState) {
 		List<DFAState> dfaStates = getAnyDFAPathToTarget(targetState);
 		List<IntSet> labels = new ArrayList<IntSet>();
+		if ( dfaStates==null ) return labels;
 		for (int i=0; i<dfaStates.size()-1; i++) {
 			DFAState d = dfaStates.get(i);
 			DFAState nextState = dfaStates.get(i + 1);

tool/src/org/antlr/v4/analysis/RecursionLimitedNFAToDFAConverter.java

@@ -1,6 +1,9 @@
 package org.antlr.v4.analysis;
 
-import org.antlr.v4.automata.*;
+import org.antlr.v4.automata.DecisionState;
+import org.antlr.v4.automata.NFAState;
+import org.antlr.v4.automata.RuleTransition;
+import org.antlr.v4.automata.Transition;
 import org.antlr.v4.misc.BitSet;
 import org.antlr.v4.misc.IntSet;
 import org.antlr.v4.tool.Grammar;
@@ -146,11 +149,6 @@ public class RecursionLimitedNFAToDFAConverter extends StackLimitedNFAToDFAConve
 		super(g, nfaStartState);
 	}
 
-	@Override
-	void reach(DFAState d) {
-		super.reach(d);
-	}
-
 	/**
 	 *   1. Reach an NFA state associated with the end of a rule, r, in the
 	 *      grammar from which it was built.  We must add an implicit (i.e.,

tool/src/org/antlr/v4/analysis/StackLimitedNFAToDFAConverter.java

@@ -3,9 +3,9 @@ package org.antlr.v4.analysis;
 import org.antlr.v4.automata.*;
 import org.antlr.v4.misc.IntervalSet;
 import org.antlr.v4.misc.OrderedHashSet;
+import org.antlr.v4.tool.ErrorManager;
 import org.antlr.v4.tool.Grammar;
 import org.antlr.v4.tool.Rule;
-import org.stringtemplate.v4.misc.MultiMap;
 
 import java.util.*;
 
@@ -68,19 +68,14 @@ public class StackLimitedNFAToDFAConverter {
 
 	public boolean hasPredicateBlockedByAction = false;
 
-	/** Recursion is limited to a particular depth.  If that limit is exceeded
+	/** Recursion is limited to a particular depth. Which state tripped it? */
-	 *  the proposed new NFA configuration is recorded for the associated DFA state.
+	public DFAState recursionOverflowState;
-	 */
-	protected MultiMap<DFAState, NFAConfig> stateToRecursionOverflowConfigurationsMap =
-		new MultiMap<DFAState, NFAConfig>();
 
-	//public Set<DFAState> recursionOverflowStates = new HashSet<DFAState>();
+	/** Which state found multiple recursive alts? */
+	public DFAState abortedDueToMultipleRecursiveAltsAt;
 
-	/** Are there any loops in this DFA?  Computed by DFAVerifier */
+	/** Are there any loops in this DFA? */
-	public boolean cyclic = false;
+//	public boolean cyclic = false;
-
-	/** Is this DFA reduced?  I.e., can all states lead to an accept state? */
-	public boolean reduced = true;	
 
 	/** Used to prevent the closure operation from looping to itself and
      *  hence looping forever.  Sensitive to the NFA state, the alt, and
@@ -145,7 +140,20 @@ public class StackLimitedNFAToDFAConverter {
 //				closure(t);  // add any NFA states reachable via epsilon
 //			}
 
+			try {
 				closure(t);  // add any NFA states reachable via epsilon
+			}
+			catch (RecursionOverflowSignal ros) {
+				recursionOverflowState = d;
+				ErrorManager.recursionOverflow(g.fileName, d, ros.state, ros.altNum, ros.depth);
+			}
+			catch (MultipleRecursiveAltsSignal mras) {
+				abortedDueToMultipleRecursiveAltsAt = d;
+				ErrorManager.multipleRecursiveAlts(g.fileName, d, mras.recursiveAltSet);
+			}
+			catch (AnalysisTimeoutSignal at) {// TODO: nobody throws yet
+				ErrorManager.analysisTimeout();
+			}
 
 			addTransition(d, label, t); // make d-label->t transition
 		}

tool/src/org/antlr/v4/automata/DFA.java

@@ -44,6 +44,9 @@ public class DFA {
 	/** We only want one accept state per predicted alt; track here */
 	public List<DFAState>[] altToAcceptStates;
 
+	/** Did DFA minimization do anything? */
+	public boolean minimized;
+
 	/** Unique state numbers per DFA */
 	int stateCounter = 0;
 
@@ -91,7 +94,12 @@ public class DFA {
 		return converter.ambiguousStates.size()>0 && !resolvedWithPredicates;
 	}
 
-	public boolean valid() { return converter.danglingStates.size()==0; }
+	public boolean valid() {
+		return
+			converter.danglingStates.size()==0 &&
+			converter.abortedDueToMultipleRecursiveAltsAt ==null &&
+			converter.recursionOverflowState ==null;
+	}
 	
 	public String toString() {
 		if ( startState==null ) return "";

tool/src/org/antlr/v4/tool/ErrorManager.java

@@ -355,16 +355,16 @@ public class ErrorManager {
 	}
 
 	public static void recursionOverflow(String fileName,
-										 DFA dfa, NFAState s, int altNum, int depth) {
+										 DFAState d, NFAState s, int altNum, int depth) {
 		state.get().errors++;
-		Message msg = new RecursionOverflowMessage(fileName, dfa, s, altNum, depth);
+		Message msg = new RecursionOverflowMessage(fileName, d, s, altNum, depth);
 		state.get().listener.error(msg);
 	}
 
 	public static void multipleRecursiveAlts(String fileName,
-											 DFA dfa, IntSet recursiveAltSet) {
+											 DFAState d, IntSet recursiveAltSet) {
 		state.get().errors++;
-		Message msg = new MultipleRecursiveAltsMessage(fileName, dfa, recursiveAltSet);
+		Message msg = new MultipleRecursiveAltsMessage(fileName, d, recursiveAltSet);
 		state.get().listener.error(msg);
 	}
 

tool/src/org/antlr/v4/tool/MultipleRecursiveAltsMessage.java

@@ -1,18 +1,21 @@
 package org.antlr.v4.tool;
 
 import org.antlr.v4.automata.DFA;
+import org.antlr.v4.automata.DFAState;
 import org.antlr.v4.misc.IntSet;
 
 import java.util.HashMap;
 import java.util.Map;
 
 public class MultipleRecursiveAltsMessage extends Message {
+	public DFAState d;
 	public DFA dfa;
 	public IntSet recursiveAltSet;
 
-	public MultipleRecursiveAltsMessage(String fileName, DFA dfa, IntSet recursiveAltSet) {
+	public MultipleRecursiveAltsMessage(String fileName, DFAState d, IntSet recursiveAltSet) {
 		super(ErrorType.MULTIPLE_RECURSIVE_ALTS);
-		this.dfa = dfa;
+		this.d = d;
+		this.dfa = d.dfa;
 		this.recursiveAltSet = recursiveAltSet;
 
 		this.line = dfa.decisionNFAStartState.ast.getLine();

tool/src/org/antlr/v4/tool/RecursionOverflowMessage.java

@@ -1,19 +1,25 @@
 package org.antlr.v4.tool;
 
+import org.antlr.v4.analysis.MachineProbe;
 import org.antlr.v4.automata.DFA;
+import org.antlr.v4.automata.DFAState;
 import org.antlr.v4.automata.NFAState;
+import org.antlr.v4.misc.IntSet;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 public class RecursionOverflowMessage extends Message {
 	DFA dfa;
+	DFAState d;
 	NFAState s;
 	int altNum;
 	int depth;
-	public RecursionOverflowMessage(String fileName, DFA dfa, NFAState s, int altNum, int depth) {
+	public RecursionOverflowMessage(String fileName, DFAState d, NFAState s, int altNum, int depth) {
 		super(ErrorType.RECURSION_OVERFLOW);
-		this.dfa = dfa;
+		this.d = d;
+		this.dfa = d.dfa;
 		this.s = s;
 		this.altNum = altNum;
 		this.depth = depth;
@@ -22,10 +28,16 @@ public class RecursionOverflowMessage extends Message {
 		this.charPosition = dfa.decisionNFAStartState.ast.getCharPositionInLine();
 		this.fileName = fileName;
 
+		MachineProbe probe = new MachineProbe(dfa);
+		List<IntSet> labels = probe.getEdgeLabels(d);		
+		String input = probe.getInputSequenceDisplay(dfa.g, labels);
+
 		Map<String, Object> info = new HashMap<String, Object>();
 		info.put("dfa", dfa);
+		info.put("dfaState", d);
 		info.put("alt", altNum);
 		info.put("depth", depth);
+		info.put("input", input);
 		info.put("nfaState", s);
 		info.put("sourceRule", s.rule);
 		info.put("targetRule", s.transition(0).target.rule);