From 8650c2962769ae8aaa58d22e774c403b01c09ba2 Mon Sep 17 00:00:00 2001 From: parrt Date: Tue, 23 Mar 2010 12:07:29 -0800 Subject: [PATCH] reorged a bit; improved msg about recur ovf. [git-p4: depot-paths = "//depot/code/antlr4/main/": change = 6768] --- .../tool/templates/messages/languages/en.stg | 4 +- tool/src/org/antlr/v4/Tool.java | 86 ++++----- .../antlr/v4/analysis/AnalysisPipeline.java | 39 ++--- .../org/antlr/v4/analysis/DFAMinimizer.java | 5 +- .../org/antlr/v4/analysis/DFAVerifier.java | 165 ------------------ .../org/antlr/v4/analysis/MachineProbe.java | 1 + .../RecursionLimitedNFAToDFAConverter.java | 10 +- .../StackLimitedNFAToDFAConverter.java | 46 +++-- tool/src/org/antlr/v4/automata/DFA.java | 10 +- tool/src/org/antlr/v4/tool/ErrorManager.java | 8 +- .../v4/tool/MultipleRecursiveAltsMessage.java | 7 +- .../v4/tool/RecursionOverflowMessage.java | 16 +- 12 files changed, 130 insertions(+), 267 deletions(-) delete mode 100644 tool/src/org/antlr/v4/analysis/DFAVerifier.java diff --git a/tool/resources/org/antlr/v4/tool/templates/messages/languages/en.stg b/tool/resources/org/antlr/v4/tool/templates/messages/languages/en.stg index a6f509b9f..c33bf3a0a 100644 --- a/tool/resources/org/antlr/v4/tool/templates/messages/languages/en.stg +++ b/tool/resources/org/antlr/v4/tool/templates/messages/languages/en.stg @@ -276,8 +276,8 @@ ANALYSIS_TIMEOUT(arg) ::= << ANTLR could not analyze this decision in rule ; often this is because of recursive rule references visible from the left edge of alternatives. ANTLR will re-analyze the decision with a fixed lookahead of k=1. Consider using "options {k=1;}" for that decision and possibly adding a syntactic predicate >> -RECURSION_OVERFLOW(alt,input,targetRules,callSiteStates) ::= << -Alternative : after matching input such as decision cannot predict what comes next due to recursion overflow from };separator=", ">}; separator=" and "> +RECURSION_OVERFLOW(arg) ::= << +Recursion overflow to from alternative of after matching input such as >> LEFT_RECURSION_CYCLES(arg) ::= << diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java index 83ce14de1..c7a396e0f 100644 --- a/tool/src/org/antlr/v4/Tool.java +++ b/tool/src/org/antlr/v4/Tool.java @@ -21,26 +21,26 @@ public class Tool { public String VERSION = "!Unknown version!"; //public static final String VERSION = "${project.version}"; public static final String UNINITIALIZED_DIR = ""; - private List grammarFileNames = new ArrayList(); - private boolean generate_NFA_dot = false; - private boolean generate_DFA_dot = false; - private String outputDirectory = "."; - private boolean haveOutputDir = false; - private String inputDirectory = null; - private String parentGrammarDirectory; - private String grammarOutputDirectory; - private boolean haveInputDir = false; - private String libDirectory = "."; - private boolean debug = false; - private boolean trace = false; - private boolean profile = false; - private boolean report = false; - private boolean printGrammar = false; - private boolean depend = false; - private boolean forceAllFilesToOutputDir = false; - private boolean forceRelativeOutput = false; - protected boolean deleteTempLexer = true; - private boolean verbose = false; + public List grammarFileNames = new ArrayList(); + public boolean generate_NFA_dot = false; + public boolean generate_DFA_dot = false; + public String outputDirectory = "."; + public boolean haveOutputDir = false; + public String inputDirectory = null; + public String parentGrammarDirectory; + public String grammarOutputDirectory; + public boolean haveInputDir = false; + public String libDirectory = "."; + public boolean debug = false; + public boolean trace = false; + public boolean profile = false; + public boolean report = false; + public boolean printGrammar = false; + public boolean depend = false; + public boolean forceAllFilesToOutputDir = false; + public boolean forceRelativeOutput = false; + public boolean deleteTempLexer = true; + public boolean verbose = false; /** Don't process grammar file if generated files are newer than grammar */ /** * Indicate whether the tool should analyze the dependencies of the provided grammar @@ -57,11 +57,11 @@ public class Tool { * * @param make */ - private boolean make = false; - private boolean showBanner = true; + public boolean make = false; + public boolean showBanner = true; /** Exit after showing version or whatever */ - private static boolean exitNow = false; + public static boolean exitNow = false; // The internal options are for my use on the command line during dev public static boolean internalOption_PrintGrammarTree = false; @@ -507,20 +507,26 @@ public class Tool { public void generateDFAs(Grammar g) { for (DFA dfa : g.decisionDFAs.values()) { - DOTGenerator dotGenerator = new DOTGenerator(g); - String dot = dotGenerator.getDOT(dfa.startState); - String dotFileName = g.name + "." + "dec-" + dfa.decision; - if (g.implicitLexer!=null) { - dotFileName = g.name + - Grammar.getGrammarTypeToFileNameSuffix(g.getType()) + - "." + "dec-" + dfa.decision; - } - try { - writeDOTFile(g, dotFileName, dot); - } - catch (IOException ioe) { - ErrorManager.toolError(ErrorType.CANNOT_WRITE_FILE, dotFileName, ioe); - } + generateDFA(g, dfa); + } + } + + public void generateDFA(Grammar g, DFA dfa) { + DOTGenerator dotGenerator = new DOTGenerator(g); + String dot = dotGenerator.getDOT(dfa.startState); + String dec = "dec-"; + //if ( dfa.minimized ) dec += "min-"; + String dotFileName = g.name + "." + dec + dfa.decision; + if (g.implicitLexer!=null) { + dotFileName = g.name + + Grammar.getGrammarTypeToFileNameSuffix(g.getType()) + + "." + dec + dfa.decision; + } + try { + writeDOTFile(g, dotFileName, dot); + } + catch (IOException ioe) { + ErrorManager.toolError(ErrorType.CANNOT_WRITE_FILE, dotFileName, ioe); } } @@ -649,11 +655,11 @@ public class Tool { return outputDir; } - private static void version() { + public static void version() { ErrorManager.info("ANTLR Parser Generator Version " + new Tool().VERSION); } - private static void help() { + public static void help() { ErrorManager.info("ANTLR Parser Generator Version " + new Tool().VERSION); System.err.println("usage: java org.antlr.Tool [args] file.g [file2.g file3.g ...]"); System.err.println(" -o outputDir specify output directory where all output is generated"); @@ -673,7 +679,7 @@ public class Tool { System.err.println(" -X display extended argument list"); } - private static void Xhelp() { + public static void Xhelp() { ErrorManager.info("ANTLR Parser Generator Version " + new Tool().VERSION); System.err.println(" -Xgrtree print the grammar AST"); System.err.println(" -Xdfa print DFA as text "); diff --git a/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java b/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java index 7a1f38073..d1bb858e0 100644 --- a/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java +++ b/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java @@ -2,7 +2,6 @@ package org.antlr.v4.analysis; import org.antlr.v4.automata.DFA; import org.antlr.v4.automata.DecisionState; -import org.antlr.v4.tool.ErrorManager; import org.antlr.v4.tool.Grammar; public class AnalysisPipeline { @@ -27,7 +26,7 @@ public class AnalysisPipeline { } public DFA createDFA(DecisionState s) { - // TRY APPROXIMATE LL(*) ANALYSIS + // TRY APPROXIMATE (STACK LIMITED) LL(*) ANALYSIS StackLimitedNFAToDFAConverter conv = new StackLimitedNFAToDFAConverter(g, s); DFA dfa = conv.createDFA(); System.out.print("DFA="+dfa); @@ -41,7 +40,7 @@ public class AnalysisPipeline { System.out.println("MINIMIZE"); DFAMinimizer dmin = new DFAMinimizer(dfa); - dmin.minimize(); + dfa.minimized = dmin.minimize(); return dfa; } @@ -50,21 +49,10 @@ public class AnalysisPipeline { // limited version. Ambiguities are ok because if the approx version // gets an ambiguity it's defin - // REAL LL(*) ANALYSIS IF THAT FAILS + // RECURSION LIMITED LL(*) ANALYSIS IF THAT FAILS conv = new RecursionLimitedNFAToDFAConverter(g, s); - try { - dfa = conv.createDFA(); - System.out.print("DFA="+dfa); - } - catch (RecursionOverflowSignal ros) { - ErrorManager.recursionOverflow(g.fileName, dfa, ros.state, ros.altNum, ros.depth); - } - catch (MultipleRecursiveAltsSignal mras) { - ErrorManager.multipleRecursiveAlts(g.fileName, dfa, mras.recursiveAltSet); - } - catch (AnalysisTimeoutSignal at) {// TODO: nobody throws yet - ErrorManager.analysisTimeout(); - } + dfa = conv.createDFA(); + System.out.print("DFA="+dfa); conv.issueAmbiguityWarnings(); // ambig / unreachable errors //conv.issueRecursionWarnings(); @@ -72,13 +60,16 @@ public class AnalysisPipeline { System.out.println("non-LL(*)"); System.out.println("recursion limited NOT valid"); } - else System.out.println("recursion limited valid"); - - System.out.println("MINIMIZE"); - DFAMinimizer dmin = new DFAMinimizer(dfa); - dmin.minimize(); - + else { + System.out.println("recursion limited valid"); + // gen DOT now to see non-minimized + System.out.println("DFA #states="+dfa.stateSet.size()); + //if ( g.tool.generate_NFA_dot ) g.tool.generateDFA(g, dfa); + System.out.println("MINIMIZE"); + DFAMinimizer dmin = new DFAMinimizer(dfa); + dfa.minimized = dmin.minimize(); + if ( dfa.minimized ) System.out.println("DFA minimized to #states="+dfa.stateSet.size()); + } return dfa; } - } diff --git a/tool/src/org/antlr/v4/analysis/DFAMinimizer.java b/tool/src/org/antlr/v4/analysis/DFAMinimizer.java index e22e2a9f2..776349bae 100644 --- a/tool/src/org/antlr/v4/analysis/DFAMinimizer.java +++ b/tool/src/org/antlr/v4/analysis/DFAMinimizer.java @@ -23,7 +23,7 @@ public class DFAMinimizer { this.dfa = dfa; } - public void minimize() { + public boolean minimize() { int n = dfa.states.size(); boolean[][] distinct = new boolean[n][n]; @@ -156,7 +156,7 @@ public class DFAMinimizer { System.out.println("uniq sets = "+uniq); if ( uniq.size()==dfa.states.size() ) { System.out.println("was already minimal"); - return; + return false; } // minimize the DFA (combine equiv sets) @@ -207,6 +207,7 @@ public class DFAMinimizer { } } } + return true; } void print(boolean[][] distinct) { diff --git a/tool/src/org/antlr/v4/analysis/DFAVerifier.java b/tool/src/org/antlr/v4/analysis/DFAVerifier.java deleted file mode 100644 index 5f608ebe9..000000000 --- a/tool/src/org/antlr/v4/analysis/DFAVerifier.java +++ /dev/null @@ -1,165 +0,0 @@ -package org.antlr.v4.analysis; - -import org.antlr.v4.automata.DFA; -import org.antlr.v4.automata.DFAState; -import org.antlr.v4.automata.Edge; -import org.antlr.v4.misc.Utils; -import org.stringtemplate.v4.misc.MultiMap; - -import java.util.*; - -/** Detect imperfect DFA: - * - * 1. nonreduced DFA (dangling states) - * 2. unreachable stop states - * 3. nondeterministic states - * - * TODO: unneeded? - */ -public class DFAVerifier { - public static enum ReachableStatus { - UNKNOWN, - BUSY, // in process of computing - NO, - YES; - } - - Map status = new HashMap(); - - DFA dfa; - - StackLimitedNFAToDFAConverter converter; - - // create 2D matrix showing incident edges; inverse of adjacency matrix - // incidentEdges.get(s) is list of edges pointing at state s - MultiMap incidentStates = new MultiMap(); - - public DFAVerifier(DFA dfa, StackLimitedNFAToDFAConverter converter) { - this.dfa = dfa; - this.converter = converter; - for (DFAState d : dfa.stateSet.values()) { - for (Edge e : d.edges) incidentStates.map(e.target, d); - } - } - - public Set getUnreachableAlts() { - Set unreachable = new HashSet(); - for (int alt=0; alt<=dfa.nAlts; alt++) { - if ( dfa.altToAcceptStates[alt]==null ) unreachable.add(alt); - } - return unreachable; - } - - public List getIncidentEdgeStates(DFAState d) { - return incidentStates.get(d); - } - - public Set getDeadStates() { - Set dead = new HashSet(dfa.stateSet.size()); - dead.addAll(dfa.stateSet.values()); -// for (DFAState a : dfa.altToAcceptState) { -// if ( a!=null ) dead.remove(a); -// } - - // obviously accept states reach accept states - //reaches.addAll(Arrays.asList(dfa.altToAcceptState)); - - boolean changed = true; - while ( changed ) { - changed = false; - for (DFAState d : dfa.stateSet.values()) { - if ( !dead.contains(d) ) { - // if d isn't dead, it reaches accept state. - dead.remove(d); - // and, so do all states pointing at it - List incoming = incidentStates.get(d); - if ( incoming!=null ) dead.removeAll(incoming); - changed = true; - } - } - } - -// boolean changed = true; -// while ( changed ) { -// changed = false; -// for (DFAState d : dfa.uniqueStates.values()) { -// if ( reaches.contains(d) ) { -// dead.remove(d); -// // if d reaches, so do all states pointing at it -// for (DFAState i : incidentStates.get(d)) { -// if ( !reaches.contains(i) ) { -// reaches.add(i); -// changed = true; -// } -// } -// } -// } -// } - - System.out.println("dead="+dead); - return dead; - } - - /** figure out if this state eventually reaches an accept state and - * modify the instance variable 'reduced' to indicate if we find - * at least one state that cannot reach an accept state. This implies - * that the overall DFA is not reduced. This algorithm should be - * linear in the number of DFA states. - * - * The algorithm also tracks which alternatives have no accept state, - * indicating a nondeterminism. - * - * Also computes whether the DFA is cyclic. - * - * TODO: I call getUniquelyPredicatedAlt too much; cache predicted alt - * TODO: convert to nonrecursive version. - */ - boolean _verify(DFAState d) { // TODO: rename to verify? - if ( d.isAcceptState ) { - // accept states have no edges emanating from them so we can return - status.put(d, ReachableStatus.YES); - // this alt is uniquely predicted, remove from nondeterministic list - int predicts = d.getUniquelyPredictedAlt(); - converter.unreachableAlts.remove(Utils.integer(predicts)); - return true; - } - - // avoid infinite loops - status.put(d, ReachableStatus.BUSY); - - boolean anEdgeReachesAcceptState = false; - // Visit every transition, track if at least one edge reaches stop state - // Cannot terminate when we know this state reaches stop state since - // all transitions must be traversed to set status of each DFA state. - for (int i=0; i getEdgeLabels(DFAState targetState) { List dfaStates = getAnyDFAPathToTarget(targetState); List labels = new ArrayList(); + if ( dfaStates==null ) return labels; for (int i=0; i> statesWithIncompletelyCoveredAlts = new HashMap>(); - public boolean hasPredicateBlockedByAction = false; + public boolean hasPredicateBlockedByAction = false; - /** Recursion is limited to a particular depth. If that limit is exceeded - * the proposed new NFA configuration is recorded for the associated DFA state. - */ - protected MultiMap stateToRecursionOverflowConfigurationsMap = - new MultiMap(); + /** Recursion is limited to a particular depth. Which state tripped it? */ + public DFAState recursionOverflowState; - //public Set recursionOverflowStates = new HashSet(); + /** Which state found multiple recursive alts? */ + public DFAState abortedDueToMultipleRecursiveAltsAt; - /** Are there any loops in this DFA? Computed by DFAVerifier */ - public boolean cyclic = false; - - /** Is this DFA reduced? I.e., can all states lead to an accept state? */ - public boolean reduced = true; + /** Are there any loops in this DFA? */ +// public boolean cyclic = false; /** Used to prevent the closure operation from looping to itself and * hence looping forever. Sensitive to the NFA state, the alt, and @@ -145,7 +140,20 @@ public class StackLimitedNFAToDFAConverter { // closure(t); // add any NFA states reachable via epsilon // } - closure(t); // add any NFA states reachable via epsilon + try { + closure(t); // add any NFA states reachable via epsilon + } + catch (RecursionOverflowSignal ros) { + recursionOverflowState = d; + ErrorManager.recursionOverflow(g.fileName, d, ros.state, ros.altNum, ros.depth); + } + catch (MultipleRecursiveAltsSignal mras) { + abortedDueToMultipleRecursiveAltsAt = d; + ErrorManager.multipleRecursiveAlts(g.fileName, d, mras.recursiveAltSet); + } + catch (AnalysisTimeoutSignal at) {// TODO: nobody throws yet + ErrorManager.analysisTimeout(); + } addTransition(d, label, t); // make d-label->t transition } @@ -218,7 +226,7 @@ public class StackLimitedNFAToDFAConverter { // if we couldn't find any non-resolved edges to add, return nothing if ( labelTarget.nfaConfigs.size()==0 ) return null; - + return labelTarget; } @@ -259,9 +267,9 @@ public class StackLimitedNFAToDFAConverter { // off maybe by actions later hence we need a parameter to carry // it forward boolean collectPredicates = (d == dfa.startState); - + closureBusy = new HashSet(); - + List configs = new ArrayList(); for (NFAConfig c : d.nfaConfigs) { closure(c.state, c.alt, c.context, c.semanticContext, collectPredicates, configs); @@ -316,7 +324,7 @@ public class StackLimitedNFAToDFAConverter { } // if we have context info and we're at rule stop state, do - // local follow for invokingRule and global follow for other links + // local follow for invokingRule and global follow for other links void ruleStopStateClosure(NFAState s, int altNum, NFAContext context, SemanticContext semanticContext, boolean collectPredicates, @@ -407,7 +415,7 @@ public class StackLimitedNFAToDFAConverter { } } } - + public OrderedHashSet getReachableLabels(DFAState d) { OrderedHashSet reachableLabels = new OrderedHashSet(); for (NFAState s : d.getUniqueNFAStates()) { // for each state diff --git a/tool/src/org/antlr/v4/automata/DFA.java b/tool/src/org/antlr/v4/automata/DFA.java index d4a19f5c3..107c08a84 100644 --- a/tool/src/org/antlr/v4/automata/DFA.java +++ b/tool/src/org/antlr/v4/automata/DFA.java @@ -44,6 +44,9 @@ public class DFA { /** We only want one accept state per predicted alt; track here */ public List[] altToAcceptStates; + /** Did DFA minimization do anything? */ + public boolean minimized; + /** Unique state numbers per DFA */ int stateCounter = 0; @@ -91,7 +94,12 @@ public class DFA { return converter.ambiguousStates.size()>0 && !resolvedWithPredicates; } - public boolean valid() { return converter.danglingStates.size()==0; } + public boolean valid() { + return + converter.danglingStates.size()==0 && + converter.abortedDueToMultipleRecursiveAltsAt ==null && + converter.recursionOverflowState ==null; + } public String toString() { if ( startState==null ) return ""; diff --git a/tool/src/org/antlr/v4/tool/ErrorManager.java b/tool/src/org/antlr/v4/tool/ErrorManager.java index d5fa622be..922b068fc 100644 --- a/tool/src/org/antlr/v4/tool/ErrorManager.java +++ b/tool/src/org/antlr/v4/tool/ErrorManager.java @@ -355,16 +355,16 @@ public class ErrorManager { } public static void recursionOverflow(String fileName, - DFA dfa, NFAState s, int altNum, int depth) { + DFAState d, NFAState s, int altNum, int depth) { state.get().errors++; - Message msg = new RecursionOverflowMessage(fileName, dfa, s, altNum, depth); + Message msg = new RecursionOverflowMessage(fileName, d, s, altNum, depth); state.get().listener.error(msg); } public static void multipleRecursiveAlts(String fileName, - DFA dfa, IntSet recursiveAltSet) { + DFAState d, IntSet recursiveAltSet) { state.get().errors++; - Message msg = new MultipleRecursiveAltsMessage(fileName, dfa, recursiveAltSet); + Message msg = new MultipleRecursiveAltsMessage(fileName, d, recursiveAltSet); state.get().listener.error(msg); } diff --git a/tool/src/org/antlr/v4/tool/MultipleRecursiveAltsMessage.java b/tool/src/org/antlr/v4/tool/MultipleRecursiveAltsMessage.java index 02cb46bce..f2cdf84ed 100644 --- a/tool/src/org/antlr/v4/tool/MultipleRecursiveAltsMessage.java +++ b/tool/src/org/antlr/v4/tool/MultipleRecursiveAltsMessage.java @@ -1,18 +1,21 @@ package org.antlr.v4.tool; import org.antlr.v4.automata.DFA; +import org.antlr.v4.automata.DFAState; import org.antlr.v4.misc.IntSet; import java.util.HashMap; import java.util.Map; public class MultipleRecursiveAltsMessage extends Message { + public DFAState d; public DFA dfa; public IntSet recursiveAltSet; - public MultipleRecursiveAltsMessage(String fileName, DFA dfa, IntSet recursiveAltSet) { + public MultipleRecursiveAltsMessage(String fileName, DFAState d, IntSet recursiveAltSet) { super(ErrorType.MULTIPLE_RECURSIVE_ALTS); - this.dfa = dfa; + this.d = d; + this.dfa = d.dfa; this.recursiveAltSet = recursiveAltSet; this.line = dfa.decisionNFAStartState.ast.getLine(); diff --git a/tool/src/org/antlr/v4/tool/RecursionOverflowMessage.java b/tool/src/org/antlr/v4/tool/RecursionOverflowMessage.java index 68909d95a..cc341ce7a 100644 --- a/tool/src/org/antlr/v4/tool/RecursionOverflowMessage.java +++ b/tool/src/org/antlr/v4/tool/RecursionOverflowMessage.java @@ -1,19 +1,25 @@ package org.antlr.v4.tool; +import org.antlr.v4.analysis.MachineProbe; import org.antlr.v4.automata.DFA; +import org.antlr.v4.automata.DFAState; import org.antlr.v4.automata.NFAState; +import org.antlr.v4.misc.IntSet; import java.util.HashMap; +import java.util.List; import java.util.Map; public class RecursionOverflowMessage extends Message { DFA dfa; + DFAState d; NFAState s; int altNum; int depth; - public RecursionOverflowMessage(String fileName, DFA dfa, NFAState s, int altNum, int depth) { + public RecursionOverflowMessage(String fileName, DFAState d, NFAState s, int altNum, int depth) { super(ErrorType.RECURSION_OVERFLOW); - this.dfa = dfa; + this.d = d; + this.dfa = d.dfa; this.s = s; this.altNum = altNum; this.depth = depth; @@ -22,10 +28,16 @@ public class RecursionOverflowMessage extends Message { this.charPosition = dfa.decisionNFAStartState.ast.getCharPositionInLine(); this.fileName = fileName; + MachineProbe probe = new MachineProbe(dfa); + List labels = probe.getEdgeLabels(d); + String input = probe.getInputSequenceDisplay(dfa.g, labels); + Map info = new HashMap(); info.put("dfa", dfa); + info.put("dfaState", d); info.put("alt", altNum); info.put("depth", depth); + info.put("input", input); info.put("nfaState", s); info.put("sourceRule", s.rule); info.put("targetRule", s.transition(0).target.rule);