handles recursive lexer rules now

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 6776]

tool/src/org/antlr/v4/analysis/AnalysisPipeline.java

@@ -22,9 +22,19 @@ public class AnalysisPipeline {
 			LexerNFAToDFAConverter conv = new LexerNFAToDFAConverter(g);
 			DFA dfa = conv.createDFA();
 			g.setLookaheadDFA(0, dfa); // only one decision			
+
+			System.out.println("MINIMIZE");
+			int before = dfa.stateSet.size();
+			DFAMinimizer dmin = new DFAMinimizer(dfa);
+			dfa.minimized = dmin.minimize();
+			int after = dfa.stateSet.size();
+			if ( after < before ) {
+				System.out.println("DFA minimized from "+before+" to "+after+" states");
+			}
 			return;
 		}
-		// BUILD DFA FOR EACH DECISION
+
+		// BUILD DFA FOR EACH DECISION IN NONLEXER
 		for (DecisionState s : g.nfa.decisionToNFAState) {
 			System.out.println("\nDECISION "+s.decision);
 			// TRY LINEAR APPROX FIXED LOOKAHEAD FIRST

tool/src/org/antlr/v4/analysis/DFAMinimizer.java

@@ -59,10 +59,12 @@ public class DFAMinimizer {
 		}
 
 		// Nobody can merge with a state resolved with predicates to be safe
-		for (DFAState d : dfa.converter.resolvedWithSemanticPredicates) {
-			for (int i=1; i<n; i++) {
-				distinct[d.stateNumber][i] = true;
-				distinct[i][d.stateNumber] = true;
+		if ( dfa.converter!=null ) {
+			for (DFAState d : dfa.converter.resolvedWithSemanticPredicates) {
+				for (int i=1; i<n; i++) {
+					distinct[d.stateNumber][i] = true;
+					distinct[i][d.stateNumber] = true;
+				}
 			}
 		}
 

tool/src/org/antlr/v4/analysis/LexerNFAToDFAConverter.java

@@ -50,7 +50,7 @@ public class LexerNFAToDFAConverter {
 			for (NFAConfig c : d.nfaConfigs) {
 				NFAState s = c.state;
 				if ( s instanceof RuleStopState && !s.rule.isFragment() ) {
-					dfa.defineAcceptState(s.rule.index, d);
+					dfa.defineAcceptState(c.alt, d);
 					d.matchesRules.add(s.rule);
 				}
 			}
@@ -187,9 +187,14 @@ public class LexerNFAToDFAConverter {
 			for (int i=0; i<n; i++) {
 				Transition t = s.transition(i);
 				if ( t instanceof RuleTransition ) {
-					NFAContext newContext =
-						new NFAContext(context, ((RuleTransition)t).followState);
-					closure(d, t.target, ruleIndex, newContext);
+					// simulate an r=0 recursion limited conversion by avoiding
+					// any recursive call. It approximates recursive lexer
+					// rules with loops.  Later we can try rule for real.
+					if ( !context.contains(((RuleTransition)t).followState) ) {
+						NFAContext newContext =
+							new NFAContext(context, ((RuleTransition)t).followState);
+						closure(d, t.target, ruleIndex, newContext);
+					}
 				}
 				else if ( t.isEpsilon() ) {
 					closure(d, t.target, ruleIndex, context);

tool/src/org/antlr/v4/analysis/NFAContext.java

@@ -77,6 +77,16 @@ public class NFAContext {
         }
     }
 
+	/** Is s anywhere in the context? */
+	public boolean contains(NFAState s) {
+		NFAContext sp = this;
+		while ( sp!=null ) {
+			if ( sp.returnState == s ) return true;
+			sp = sp.parent;
+		}
+		return false;
+	}
+
 	/** Two contexts are equals() if both have
 	 *  same call stack; walk upwards to the root.
 	 *  Recall that the root sentinel node has no invokingStates and no parent.