forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

The lexer and parser ATN simulators' adaptivePredict now synchronize on the specific DFA of the decision to be simulated. This should prevent a lot of contention that would occur if we synchronize the entire adaptivePredict method. When the individual DFA are created, we also synchronize on the shared DFA[] table quickly to create a DFA and insert it into the array. Code generation modified to have _decisionToDFA generated at the top of both the parser and the lexer. Simulators created now with the recognizer, ATN, DFA[]. Not sure the LexerInterp/ParserInterp work but pushing ahead anyway for the moment.

This commit is contained in:
Terence Parr 2012-07-29 09:49:35 -07:00
parent 8c8752a7b2
commit e3e739dfc7
12 changed files with 105 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
runtime/Java/src/org/antlr/v4/runtime/Parser.java
View File

@ -532,27 +532,31 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
}
/** For debugging and other purposes */
public List<String> getDFAStrings() {
List<String> s = new ArrayList<String>();
for (int d = 0; d < _interp.decisionToDFA.length; d++) {
DFA dfa = _interp.decisionToDFA[d];
s.add( dfa.toString(getTokenNames()) );
}
return s;
public List<String> getDFAStrings() {
synchronized (_interp.decisionToDFA) {
List<String> s = new ArrayList<String>();
for (int d = 0; d < _interp.decisionToDFA.length; d++) {
DFA dfa = _interp.decisionToDFA[d];
s.add( dfa.toString(getTokenNames()) );
}
return s;
}
}
/** For debugging and other purposes */
public void dumpDFA() {
boolean seenOne = false;
for (int d = 0; d < _interp.decisionToDFA.length; d++) {
DFA dfa = _interp.decisionToDFA[d];
if ( dfa!=null ) {
if ( seenOne ) System.out.println();
System.out.println("Decision " + dfa.decision + ":");
System.out.print(dfa.toString(getTokenNames()));
seenOne = true;
}
}
/** For debugging and other purposes */
public void dumpDFA() {
synchronized (_interp.decisionToDFA) {
boolean seenOne = false;
for (int d = 0; d < _interp.decisionToDFA.length; d++) {
DFA dfa = _interp.decisionToDFA[d];
if ( dfa!=null ) {
if ( seenOne ) System.out.println();
System.out.println("Decision " + dfa.decision + ":");
System.out.print(dfa.toString(getTokenNames()));
seenOne = true;
}
}
}
}
public String getSourceName() {

8
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -111,11 +111,13 @@ public class LexerATNSimulator extends ATNSimulator {
public static int ATN_failover = 0;
public static int match_calls = 0;
public LexerATNSimulator(@NotNull ATN atn) {
this(null, atn);
public LexerATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
this(null, atn, decisionToDFA);
}
public LexerATNSimulator(@Nullable Lexer recog, @NotNull ATN atn) {
public LexerATNSimulator(@Nullable Lexer recog, @NotNull ATN atn,
@NotNull DFA[] decisionToDFA)
{
super(atn);
dfa = new DFA[atn.modeToStartState.size()];
for (int i=0; i<atn.modeToStartState.size(); i++) {

5
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNPathFinder.java
View File

@ -34,6 +34,7 @@ import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
@ -45,8 +46,8 @@ import java.util.List;
import java.util.Set;
public class ParserATNPathFinder extends ParserATNSimulator<Token> {
public ParserATNPathFinder(@Nullable Parser parser, @NotNull ATN atn) {
super(parser, atn);
public ParserATNPathFinder(@Nullable Parser parser, @NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
super(parser, atn, decisionToDFA);
}
/** Given an input sequence, as a subset of the input stream, trace the path through the

58
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -150,16 +150,14 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
protected ParserRuleContext<?> _outerContext;
/** Testing only! */
public ParserATNSimulator(@NotNull ATN atn) {
this(null, atn);
public ParserATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
this(null, atn, decisionToDFA);
}
public ParserATNSimulator(@Nullable Parser parser, @NotNull ATN atn) {
public ParserATNSimulator(@Nullable Parser parser, @NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
super(atn);
this.parser = parser;
// ctxToDFAs = new HashMap<RuleContext, DFA[]>();
// TODO (sam): why distinguish on parser != null?
decisionToDFA = new DFA[atn.getNumberOfDecisions() + (parser != null ? 1 : 0)];
this.decisionToDFA = decisionToDFA;
// DOTGenerator dot = new DOTGenerator(null);
// System.out.println(dot.getDOT(atn.rules.get(0), parser.getRuleNames()));
// System.out.println(dot.getDOT(atn.rules.get(1), parser.getRuleNames()));
@ -174,22 +172,37 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
{
if ( debug || debug_list_atn_decisions ) {
System.out.println("adaptivePredict decision "+decision+
" exec LA(1)=="+ getLookaheadName(input)+
" line "+input.LT(1).getLine()+":"+input.LT(1).getCharPositionInLine());
" exec LA(1)=="+ getLookaheadName(input)+
" line "+input.LT(1).getLine()+":"+input.LT(1).getCharPositionInLine());
}
_input = input;
_startIndex = input.index();
_outerContext = outerContext;
predict_calls++;
DFA dfa = decisionToDFA[decision];
if ( dfa==null || dfa.s0==null ) {
DecisionState startState = atn.decisionToState.get(decision);
decisionToDFA[decision] = dfa = new DFA(startState, decision);
return predictATN(dfa, input, outerContext);
// First, synchronize on the array of DFA for this parser
// so that we can get the DFA for a decision or create and set one
if ( dfa==null || dfa.s0==null ) { // only create one if not there
synchronized (decisionToDFA) {
dfa = decisionToDFA[decision];
if ( dfa==null || dfa.s0==null ) { // the usual double-check
DecisionState startState = atn.decisionToState.get(decision);
decisionToDFA[decision] = dfa = new DFA(startState, decision);
}
}
// Now we are certain to have a specific decision's DFA
// Synchronize on the DFA so that nobody can read or write
// to it while we updated during ATN simulation
synchronized (decisionToDFA[decision]) {
return predictATN(dfa, input, outerContext);
}
}
else {
//dump(dfa);
// start with the DFA
// We can start with an existing DFA
synchronized (decisionToDFA[decision]) {
// Only enter the DFA simulation if nobody else is playing with it.
// This blocks multiple readonly simulations of the same DFA but that's
// unlikely to happen a lot
int m = input.mark();
int index = input.index();
try {
@ -206,7 +219,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
public int predictATN(@NotNull DFA dfa, @NotNull TokenStream input,
@Nullable ParserRuleContext<?> outerContext)
{
//contextCache = new PredictionContextCache("predict ctx cache");
// caller must ensure current thread is sync'd on dfa
if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY;
if ( debug || debug_list_atn_decisions ) {
System.out.println("predictATN decision "+dfa.decision+
@ -246,6 +259,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
@NotNull TokenStream input, int startIndex,
@Nullable ParserRuleContext<?> outerContext)
{
// caller must ensure current thread is sync'd on dfa
if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY;
if ( dfa_debug ) {
System.out.println("execDFA decision "+dfa.decision+
@ -555,6 +569,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
int SLL_min_alt, // todo: is this in D as min ambig alts?
boolean greedy)
{
// caller must ensure current thread is sync'd on dfa
retry_with_context++;
reportAttemptingFullContext(dfa, s0, startIndex, input.index());
@ -1364,7 +1379,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
{
DFAState from = addDFAState(dfa, p);
DFAState to = addDFAState(dfa, q);
if ( debug ) System.out.println("EDGE "+from+" -> "+to+" upon "+getTokenName(t));
if ( debug ) System.out.println("EDGE "+from+" -> "+to+" upon "+getTokenName(t));
addDFAEdge(from, t, to);
if ( debug ) System.out.println("DFA=\n"+dfa.toString(parser!=null?parser.getTokenNames():null));
return to;
@ -1386,19 +1401,12 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
if ( existing!=null ) return existing;
DFAState newState = proposed;
newState.stateNumber = dfa.states.size();
// System.out.println("Before opt, cache size = "+ sharedContextCache.size());
configs.optimizeConfigs(this);
// System.out.println("After opt, cache size = " + sharedContextCache.size());
// System.out.println(configs.size());
// if ( configs.hasSemanticContext ) System.out.println(configs.getPredicates().size());
// System.out.println(configs.getPredicates().size());
configs.setReadonly(true);
newState.configs = configs;
dfa.states.put(newState, newState);
if ( debug ) System.out.println("adding new DFA state: "+newState);
if ( debug ) System.out.println("adding new DFA state: "+newState);
return newState;
}

14
tool/playground/T.g
View File

@ -1,10 +1,10 @@
lexer grammar T;
grammar T;
RBRACE : '}' ;
s : x B ;
x : A B | A | A ;
s2 : x B ;
mode Action;
END_ACTION
: '}' -> popMode
;
A : 'a';
B : 'b' ;
WS : [ \t\n\r]+ -> channel(HIDDEN) ;

1
tool/playground/TestT.java
View File

@ -27,6 +27,7 @@ public class TestT {
parser.addErrorListener(new DiagnosticErrorListener());
ParserRuleContext tree = parser.s();
System.out.println(tree.toStringTree(parser));
// tree.save(parser, "/tmp/t.ps");
}
}

7
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -118,6 +118,7 @@ Parser(parser, funcs, atn, sempredFuncs, superclass) ::= <<
Parser_(parser, funcs, atn, sempredFuncs, ctor, extras, superclass) ::= <<
@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
public <if(parser.abstractRecognizer)>abstract <endif>class <parser.name> extends <superclass> {
protected static final DFA[] _decisionToDFA;
<if(parser.tokens)>
public static final int
<parser.tokens:{k | <k>=<parser.tokens.(k)>}; separator=", ", wrap, anchor>;
@ -190,7 +191,7 @@ case <f.ruleIndex> : return <f.name>_sempred((<f.ctxType>)_localctx, predIndex);
parser_ctor(p) ::= <<
public <p.name>(TokenStream input) {
super(input);
_interp = new ParserATNSimulator\<Token>(this,_ATN);
_interp = new ParserATNSimulator\<Token>(this,_ATN,_decisionToDFA);
}
>>
@ -714,6 +715,7 @@ import org.antlr.v4.runtime.misc.*;
Lexer(lexer, atn, actionFuncs, sempredFuncs) ::= <<
@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
public <if(lexer.abstractRecognizer)>abstract <endif>class <lexer.name> extends Lexer {
protected static final DFA[] _decisionToDFA;
public static final int
<lexer.tokens:{k | <k>=<lexer.tokens.(k)>}; separator=", ", wrap, anchor>;
<rest(lexer.modes):{m| public static final int <m> = <i>;}; separator="\n">
@ -733,7 +735,7 @@ public <if(lexer.abstractRecognizer)>abstract <endif>class <lexer.name> extends
public <lexer.name>(CharStream input) {
super(input);
_interp = new LexerATNSimulator(this,_ATN);
_interp = new LexerATNSimulator(this,_ATN,_decisionToDFA);
}
@Override
@ -765,6 +767,7 @@ public static final String _serializedATN =
public static final ATN _ATN =
ATNSimulator.deserialize(_serializedATN.toCharArray());
static {
_decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
//org.antlr.v4.tool.DOTGenerator dot = new org.antlr.v4.tool.DOTGenerator(null);
//System.out.println(dot.getDOT(_ATN.decisionToState.get(0), ruleNames, false));
//System.out.println(dot.getDOT(_ATN.ruleToStartState[2], ruleNames, false));

2
tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java
View File

@ -56,7 +56,7 @@ public class LexerInterpreter implements TokenSource {
public LexerInterpreter(LexerGrammar g) {
Tool antlr = new Tool();
antlr.process(g,false);
interp = new LexerATNSimulator(g.atn);
interp = new LexerATNSimulator(g.atn,null);
}
public void setInput(String inputString) {

17
tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java
View File

@ -44,11 +44,13 @@ import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.tool.Grammar;
public class ParserInterpreter {
class DummyParser extends Parser {
public static class DummyParser extends Parser {
public final DFA[] decisionToDFA; // not shared for interp
public Grammar g;
public DummyParser(Grammar g, TokenStream input) {
super(input);
this.g = g;
decisionToDFA = new DFA[100];
}
@Override
@ -70,9 +72,12 @@ public class ParserInterpreter {
public ATN getATN() {
return null;
}
static {
}
}
protected Grammar g;
public DummyParser parser;
protected ParserATNSimulator<Token> atnSimulator;
protected TokenStream input;
@ -83,13 +88,15 @@ public class ParserInterpreter {
public ParserInterpreter(@NotNull Grammar g, @NotNull TokenStream input) {
Tool antlr = new Tool();
antlr.process(g,false);
atnSimulator = new ParserATNSimulator<Token>(new DummyParser(g, input), g.atn);
parser = new DummyParser(g, input);
atnSimulator = new ParserATNSimulator<Token>(parser, g.atn, parser.decisionToDFA);
}
public int predictATN(@NotNull DFA dfa, @NotNull TokenStream input,
@Nullable ParserRuleContext outerContext,
boolean useContext)
public synchronized int predictATN(@NotNull DFA dfa, @NotNull TokenStream input,
@Nullable ParserRuleContext outerContext,
boolean useContext)
{
// sync to ensure this entry doesn't race for dfa access
return atnSimulator.predictATN(dfa, input, outerContext);
}

2
tool/test/org/antlr/v4/test/BaseTest.java
View File

@ -211,7 +211,7 @@ public abstract class BaseTest {
CharStream input,
boolean adaptive)
{
LexerATNSimulator interp = new LexerATNSimulator(atn);
LexerATNSimulator interp = new LexerATNSimulator(atn,null);
List<String> tokenTypes = new ArrayList<String>();
int ttype;
boolean hitEOF = false;

2
tool/test/org/antlr/v4/test/TestATNInterpreter.java
View File

@ -273,7 +273,7 @@ public class TestATNInterpreter extends BaseTest {
int expected)
{
ATN lexatn = createATN(lg);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
System.out.println(types);

21
tool/test/org/antlr/v4/test/TestATNParserPrediction.java
View File

@ -35,7 +35,10 @@ import org.antlr.v4.runtime.NoViableAltException;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.atn.ParserATNSimulator;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.tool.DOTGenerator;
import org.antlr.v4.tool.Grammar;
@ -476,7 +479,7 @@ public class TestATNParserPrediction extends BaseTest {
{
Tool.internalOption_ShowATNConfigsInDFA = true;
ATN lexatn = createATN(lg);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
System.out.println(types);
@ -522,12 +525,13 @@ public class TestATNParserPrediction extends BaseTest {
assertEquals(expectedAlt, alt);
}
public DFA getDFA(LexerGrammar lg, Grammar g, String ruleName,
String inputString, ParserRuleContext<?> ctx)
public synchronized DFA getDFA(LexerGrammar lg, Grammar g, String ruleName,
String inputString, ParserRuleContext<?> ctx)
{
// sync to ensure multiple tests don't race on dfa access
Tool.internalOption_ShowATNConfigsInDFA = true;
ATN lexatn = createATN(lg);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
semanticProcess(lg);
g.importVocab(lg);
@ -541,7 +545,8 @@ public class TestATNParserPrediction extends BaseTest {
// System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("b"))));
// System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e"))));
ParserATNSimulator<Token> interp = new ParserATNSimulator<Token>(atn);
ParserATNSimulator<Token> interp =
new ParserATNSimulator<Token>(atn, new DFA[atn.getNumberOfDecisions()+1]);
List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
System.out.println(types);
TokenStream input = new IntTokenStream(types);
@ -563,7 +568,7 @@ public class TestATNParserPrediction extends BaseTest {
{
// Tool.internalOption_ShowATNConfigsInDFA = true;
ATN lexatn = createATN(lg);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
semanticProcess(lg);
g.importVocab(lg);
@ -581,7 +586,7 @@ public class TestATNParserPrediction extends BaseTest {
catch (NoViableAltException nvae) {
nvae.printStackTrace(System.err);
}
DFA dfa = interp.getATNSimulator().decisionToDFA[decision];
DFA dfa = interp.parser.decisionToDFA[decision];
assertEquals(dfaString[i], dfa.toString(g.getTokenDisplayNames()));
}
}