jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

no more resolved bit in config. use ATNConfigSet all over. final clean up of ATN sim. 

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9741]
This commit is contained in:
parrt 2011-12-25 12:01:24 -08:00
parent 79d1c40427
commit 6daa66f83f
12 changed files with 46 additions and 1527 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java
View File

@ -67,7 +67,7 @@ public class ATNConfig {
//public boolean traversedPredicate; // TODO: don't need
/** Ignore this config when examining config sets */
public boolean resolved;
// public boolean resolved;
/**
* We cannot execute predicates dependent upon local context unless

3
runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java
View File

@ -32,7 +32,6 @@ package org.antlr.v4.runtime.atn;
import org.antlr.v4.runtime.dfa.DFAState;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.OrderedHashSet;
import java.util.ArrayList;
import java.util.List;
@ -45,7 +44,7 @@ public abstract class ATNSimulator {
public final ATN atn;
static {
ERROR = new DFAState(new OrderedHashSet<ATNConfig>());
ERROR = new DFAState(new ATNConfigSet());
ERROR.stateNumber = Integer.MAX_VALUE;
}

37
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -34,7 +34,6 @@ import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.dfa.DFAState;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.misc.OrderedHashSet;
import java.io.IOException;
import java.io.OutputStream;
@ -184,7 +183,7 @@ public class LexerATNSimulator extends ATNSimulator {
System.out.format("mode %d start: %s\n", mode, startState);
}
OrderedHashSet<ATNConfig> s0_closure = computeStartState(input, startState);
ATNConfigSet s0_closure = computeStartState(input, startState);
int old_mode = mode;
dfa[mode].s0 = addDFAState(s0_closure);
int predict = exec(input, s0_closure);
@ -257,7 +256,7 @@ public class LexerATNSimulator extends ATNSimulator {
return Token.EOF;
}
if ( atnException!=null ) throw atnException;
throw new LexerNoViableAltException(recog, input, startIndex, s.configs);
throw new LexerNoViableAltException(recog, input, startIndex, s.configset);
}
int ruleIndex = dfaPrevAccept.state.ruleIndex;
@ -266,17 +265,17 @@ public class LexerATNSimulator extends ATNSimulator {
return dfaPrevAccept.state.prediction;
}
protected int exec(@NotNull CharStream input, @NotNull OrderedHashSet<ATNConfig> s0) {
protected int exec(@NotNull CharStream input, @NotNull ATNConfigSet s0) {
//System.out.println("enter exec index "+input.index()+" from "+s0);
@NotNull
OrderedHashSet<ATNConfig> closure = new OrderedHashSet<ATNConfig>();
ATNConfigSet closure = new ATNConfigSet();
closure.addAll(s0);
if ( debug ) {
System.out.format("start state closure=%s\n", closure);
}
@NotNull
OrderedHashSet<ATNConfig> reach = new OrderedHashSet<ATNConfig>();
ATNConfigSet reach = new ATNConfigSet();
atnPrevAccept.reset();
traceLookahead1();
@ -327,7 +326,7 @@ public class LexerATNSimulator extends ATNSimulator {
// swap to avoid reallocating space
// TODO: faster to reallocate?
@NotNull
OrderedHashSet<ATNConfig> tmp = reach;
ATNConfigSet tmp = reach;
reach = closure;
closure = tmp;
reach.clear();
@ -347,7 +346,7 @@ public class LexerATNSimulator extends ATNSimulator {
return atn.ruleToTokenType[ruleIndex];
}
protected void processAcceptStates(@NotNull CharStream input, @NotNull OrderedHashSet<ATNConfig> reach) {
protected void processAcceptStates(@NotNull CharStream input, @NotNull ATNConfigSet reach) {
for (int ci=0; ci<reach.size(); ci++) {
ATNConfig c = reach.get(ci);
if ( c.state instanceof RuleStopState) {
@ -435,7 +434,7 @@ public class LexerATNSimulator extends ATNSimulator {
return null;
}
public void deleteWildcardConfigsForAlt(@NotNull OrderedHashSet<ATNConfig> closure, int ci, int alt) {
public void deleteWildcardConfigsForAlt(@NotNull ATNConfigSet closure, int ci, int alt) {
int j=ci+1;
while ( j<closure.size() ) {
ATNConfig c = closure.get(j);
@ -453,11 +452,11 @@ public class LexerATNSimulator extends ATNSimulator {
}
@NotNull
protected OrderedHashSet<ATNConfig> computeStartState(@NotNull IntStream input,
protected ATNConfigSet computeStartState(@NotNull IntStream input,
@NotNull ATNState p)
{
RuleContext initialContext = EMPTY_LEXER_RULE_CONTEXT;
OrderedHashSet<ATNConfig> configs = new OrderedHashSet<ATNConfig>();
ATNConfigSet configs = new ATNConfigSet();
for (int i=0; i<p.getNumberOfTransitions(); i++) {
ATNState target = p.transition(i).target;
ATNConfig c = new ATNConfig(target, i+1, initialContext);
@ -466,7 +465,7 @@ public class LexerATNSimulator extends ATNSimulator {
return configs;
}
protected void closure(@NotNull ATNConfig config, @NotNull OrderedHashSet<ATNConfig> configs) {
protected void closure(@NotNull ATNConfig config, @NotNull ATNConfigSet configs) {
if ( debug ) {
System.out.println("closure("+config.toString(recog, true)+")");
}
@ -545,10 +544,10 @@ public class LexerATNSimulator extends ATNSimulator {
if ( dfa_debug ) {
System.out.format("no edge for %s\n", getTokenName(input.LA(1)));
System.out.format("ATN exec upon %s at DFA state %d = %s\n",
input.substring(startIndex, input.index()), s.stateNumber, s.configs);
input.substring(startIndex, input.index()), s.stateNumber, s.configset);
}
int ttype = exec(input, s.configs);
int ttype = exec(input, s.configset);
if ( dfa_debug ) {
System.out.format("back from DFA update, ttype=%d, dfa[mode %d]=\n%s\n",
@ -567,9 +566,9 @@ public class LexerATNSimulator extends ATNSimulator {
state.charPos = charPositionInLine;
}
protected void addDFAEdge(@NotNull OrderedHashSet<ATNConfig> p,
protected void addDFAEdge(@NotNull ATNConfigSet p,
int t,
@NotNull OrderedHashSet<ATNConfig> q)
@NotNull ATNConfigSet q)
{
// even if we can add the states, we can't add an edge for labels out of range
if (t < 0 || t > NUM_EDGES) {
@ -626,7 +625,7 @@ public class LexerATNSimulator extends ATNSimulator {
test them, we cannot cash the DFA state target of ID.
*/
@Nullable
protected DFAState addDFAState(@NotNull OrderedHashSet<ATNConfig> configs) {
protected DFAState addDFAState(@NotNull ATNConfigSet configs) {
DFAState proposed = new DFAState(configs);
DFAState existing = dfa[mode].states.get(proposed);
if ( existing!=null ) return existing;
@ -653,8 +652,8 @@ public class LexerATNSimulator extends ATNSimulator {
if ( traversedPredicate ) return null; // cannot cache
newState.stateNumber = dfa[mode].states.size();
newState.configs = new OrderedHashSet<ATNConfig>();
newState.configs.addAll(configs);
newState.configset = new ATNConfigSet();
newState.configset.addAll(configs);
dfa[mode].states.put(newState, newState);
return newState;
}

1447
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

File diff suppressed because it is too large Load Diff

32
runtime/Java/src/org/antlr/v4/runtime/atn/v2ParserATNSimulator.java
View File

@ -568,7 +568,7 @@ public class v2ParserATNSimulator<Symbol> extends ATNSimulator {
int nalts = decState.getNumberOfTransitions();
List<DFAState.PredPrediction> predPredictions =
predicateDFAState(D, D.configset, outerContext, nalts);
if ( tooFewPredicates(D, outerContext, nalts) ) {
if ( D.predicates.size() < nalts ) {
IntervalSet conflictingAlts = getConflictingAltsFromConfigSet(D.configset);
reportInsufficientPredicates(dfa, startIndex, input.index(),
conflictingAlts,
@ -1135,26 +1135,6 @@ public class v2ParserATNSimulator<Symbol> extends ATNSimulator {
return ambigAlts;
}
public boolean tooFewPredicates(DFAState D, RuleContext outerContext, int nalts) {
List<DFAState.PredPrediction> pairs;
if ( D.isCtxSensitive ) {
pairs = D.ctxToPredicates.get(outerContext); // TODO: rm
}
else {
pairs = D.predicates;
}
return pairs==null || pairs.size() < nalts;
// IntervalSet conflictingAlts = getConflictingAltsFromConfigSet(configs);
// SemanticContext[] altToPred = getPredsForAmbigAlts(conflictingAlts, configs, nalts);
// // We need at least n-1 predicates for n ambiguous alts
// int unpredicated = 0;
// for (int i = 1; i < altToPred.length; i++) {
// if ( altToPred[i]==SemanticContext.NONE ) unpredicated++;
// }
// return unpredicated > 1;
}
protected IntervalSet getConflictingAltsFromConfigSet(ATNConfigSet configs) {
IntervalSet conflictingAlts;
if ( configs.uniqueAlt!= ATN.INVALID_ALT_NUMBER ) {
@ -1168,7 +1148,7 @@ public class v2ParserATNSimulator<Symbol> extends ATNSimulator {
protected int resolveToMinAlt(@NotNull DFAState D, IntervalSet conflictingAlts) {
// kill dead alts so we don't chase them ever
killAlts(conflictingAlts, D.configset);
// killAlts(conflictingAlts, D.configset);
D.prediction = conflictingAlts.getMinElement();
if ( debug ) System.out.println("RESOLVED TO "+D.prediction+" for "+D);
return D.prediction;
@ -1182,17 +1162,11 @@ public class v2ParserATNSimulator<Symbol> extends ATNSimulator {
int exitAlt = 2;
conflictingAlts.remove(exitAlt);
// kill dead alts so we don't chase them ever
killAlts(conflictingAlts, reach);
// killAlts(conflictingAlts, reach);
if ( debug ) System.out.println("RESOLVED TO "+reach);
return exitAlt;
}
public static void killAlts(@NotNull IntervalSet alts, @NotNull ATNConfigSet configs) {
for (ATNConfig c : configs) {
if ( alts.contains(c.alt) ) c.resolved = true;
}
}
@NotNull
public String getTokenName(int t) {
if ( t==-1 ) return "EOF";

21
runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java
View File

@ -29,16 +29,13 @@
package org.antlr.v4.runtime.dfa;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.atn.ATNConfig;
import org.antlr.v4.runtime.atn.ATNConfigSet;
import org.antlr.v4.runtime.atn.SemanticContext;
import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.runtime.misc.OrderedHashSet;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/** A DFA state represents a set of possible ATN configurations.
@ -69,8 +66,8 @@ public class DFAState {
public int stateNumber = -1;
/** The set of ATN configurations (state,alt,context) for this DFA state */
@Nullable
public OrderedHashSet<ATNConfig> configs = new OrderedHashSet<ATNConfig>();
// @Nullable
// public OrderedHashSet<ATNConfig> configs = new OrderedHashSet<ATNConfig>();
// TODO: rename to configs after flipping to new ATN sim
public ATNConfigSet configset = new ATNConfigSet();
@ -89,12 +86,6 @@ public class DFAState {
public boolean complete; // all alts predict "prediction"
public boolean isCtxSensitive;
// TODO: refactor to subclass?
@Nullable
public Map<RuleContext, Integer> ctxToPrediction; // used for ctx sensitive parsing
public Map<RuleContext, List<PredPrediction>> ctxToPredicates; // used for ctx sensitive parsing
/** DFA accept states use predicates in two situations:
* disambiguating and validating predicates. If an accept state
* predicts more than one alternative, It's ambiguous and we
@ -147,7 +138,7 @@ public class DFAState {
public DFAState(int stateNumber) { this.stateNumber = stateNumber; }
public DFAState(OrderedHashSet<ATNConfig> configs) { this.configs = configs; }
public DFAState(ATNConfigSet configs) { this.configset = configs; }
/** Get the set of all alts mentioned by all ATN configurations in this
* DFA state.
@ -155,7 +146,7 @@ public class DFAState {
public Set<Integer> getAltSet() {
// TODO (sam): what to do when configs==null?
Set<Integer> alts = new HashSet<Integer>();
for (ATNConfig c : configs) {
for (ATNConfig c : configset) {
alts.add(c.alt);
}
if ( alts.size()==0 ) return null;
@ -177,7 +168,7 @@ public class DFAState {
public int hashCode() {
// TODO (sam): what to do when configs==null?
int h = 0;
for (ATNConfig c : configs) {
for (ATNConfig c : configset) {
h += c.alt;
}
return h;
@ -200,7 +191,7 @@ public class DFAState {
if ( this==o ) return true;
DFAState other = (DFAState)o;
// TODO (sam): what to do when configs==null?
boolean sameSet = this.configs.equals(other.configs);
boolean sameSet = this.configset.equals(other.configset);
// System.out.println("DFAState.equals: "+configs+(sameSet?"==":"!=")+other.configs);
return sameSet;
}

10
tool/playground/TestYang.java
View File

@ -1,6 +1,8 @@
import org.antlr.runtime.debug.BlankDebugEventListener;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.ANTLRFileStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.atn.v2ParserATNSimulator;
import java.io.File;
@ -32,8 +34,8 @@ class TestYang {
System.out.println("finished parsing OK");
System.out.println(LexerATNSimulator.ATN_failover+" lexer failovers");
System.out.println(LexerATNSimulator.match_calls+" lexer match calls");
System.out.println(ParserATNSimulator.ATN_failover+" parser failovers");
System.out.println(ParserATNSimulator.predict_calls +" parser predict calls");
System.out.println(v2ParserATNSimulator.ATN_failover+" parser failovers");
System.out.println(v2ParserATNSimulator.predict_calls +" parser predict calls");
if ( profile ) {
System.out.println("num decisions "+profiler.numDecisions);
}

2
tool/src/org/antlr/v4/tool/DOTGenerator.java
View File

@ -123,7 +123,7 @@ public class DOTGenerator {
List<Integer> altList = new ArrayList<Integer>();
altList.addAll(alts);
Collections.sort(altList);
Set<ATNConfig> configurations = s.configs;
Set<ATNConfig> configurations = s.configset;
for (int altIndex = 0; altIndex < altList.size(); altIndex++) {
int alt = altList.get(altIndex);
if ( altIndex>0 ) {

4
tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java
View File

@ -32,7 +32,6 @@ package org.antlr.v4.tool.interp;
import org.antlr.v4.Tool;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.ParserATNSimulator;
import org.antlr.v4.runtime.atn.v2ParserATNSimulator;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.NotNull;
@ -88,7 +87,8 @@ public class ParserInterpreter {
public int matchATN(@NotNull TokenStream input,
@NotNull ATNState startState)
{
return new ParserATNSimulator<Token>(new DummyParser(g, input), g.atn).matchATN(input, startState);
// return new v2ParserATNSimulator<Token>(new DummyParser(g, input), g.atn).matchATN(input, startState);
return 0;
}
public v2ParserATNSimulator<Token> getATNSimulator() {

8
tool/test/org/antlr/v4/test/TestATNDeserialization.java
View File

@ -2,8 +2,10 @@ package org.antlr.v4.test;
import org.antlr.v4.automata.ATNSerializer;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.tool.*;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.v2ParserATNSimulator;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar;
import org.junit.Test;
public class TestATNDeserialization extends BaseTest {
@ -145,7 +147,7 @@ public class TestATNDeserialization extends BaseTest {
ATN atn = createATN(g);
char[] data = Utils.toCharArray(ATNSerializer.getSerialized(g, atn));
String atnData = ATNSerializer.getDecoded(g, atn);
ATN atn2 = ParserATNSimulator.deserialize(data);
ATN atn2 = v2ParserATNSimulator.deserialize(data);
String atn2Data = ATNSerializer.getDecoded(g, atn2);
assertEquals(atnData, atn2Data);

6
tool/test/org/antlr/v4/test/TestATNParserPrediction.java
View File

@ -37,7 +37,7 @@ import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.atn.ParserATNSimulator;
import org.antlr.v4.runtime.atn.v2ParserATNSimulator;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.tool.DOTGenerator;
import org.antlr.v4.tool.Grammar;
@ -544,7 +544,7 @@ public class TestATNParserPrediction extends BaseTest {
// System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("b"))));
// System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e"))));
ParserATNSimulator interp = new ParserATNSimulator(atn);
v2ParserATNSimulator interp = new v2ParserATNSimulator(atn);
List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
System.out.println(types);
TokenStream input = new IntTokenStream(types);
@ -553,7 +553,7 @@ public class TestATNParserPrediction extends BaseTest {
DFA dfa = new DFA(startState);
// Rule r = g.getRule(ruleName);
//ATNState startState = atn.ruleToStartState.get(r);
interp.predictATN(dfa, input, ctx, false);
interp.predictATN(dfa, input, ctx);
}
catch (NoViableAltException nvae) {
nvae.printStackTrace(System.err);

1
tool/test/org/antlr/v4/test/TestPerformance.java
View File

@ -7,7 +7,6 @@ import org.antlr.v4.runtime.tree.ParseTreeListener;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
import org.junit.Assert;
import org.junit.Test;
import org.antlr.v4.runtime.atn.ParserATNSimulator;
import java.io.*;
import java.lang.reflect.Constructor;