diff --git a/runtime/Java/src/org/antlr/v4/runtime/Parser.java b/runtime/Java/src/org/antlr/v4/runtime/Parser.java index 4a5792d0f..761a88b7a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Parser.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Parser.java @@ -532,27 +532,31 @@ public abstract class Parser extends Recognizer } /** For debugging and other purposes */ - public List getDFAStrings() { - List s = new ArrayList(); - for (int d = 0; d < _interp.decisionToDFA.length; d++) { - DFA dfa = _interp.decisionToDFA[d]; - s.add( dfa.toString(getTokenNames()) ); - } - return s; + public List getDFAStrings() { + synchronized (_interp.decisionToDFA) { + List s = new ArrayList(); + for (int d = 0; d < _interp.decisionToDFA.length; d++) { + DFA dfa = _interp.decisionToDFA[d]; + s.add( dfa.toString(getTokenNames()) ); + } + return s; + } } - /** For debugging and other purposes */ - public void dumpDFA() { - boolean seenOne = false; - for (int d = 0; d < _interp.decisionToDFA.length; d++) { - DFA dfa = _interp.decisionToDFA[d]; - if ( dfa!=null ) { - if ( seenOne ) System.out.println(); - System.out.println("Decision " + dfa.decision + ":"); - System.out.print(dfa.toString(getTokenNames())); - seenOne = true; - } - } + /** For debugging and other purposes */ + public void dumpDFA() { + synchronized (_interp.decisionToDFA) { + boolean seenOne = false; + for (int d = 0; d < _interp.decisionToDFA.length; d++) { + DFA dfa = _interp.decisionToDFA[d]; + if ( dfa!=null ) { + if ( seenOne ) System.out.println(); + System.out.println("Decision " + dfa.decision + ":"); + System.out.print(dfa.toString(getTokenNames())); + seenOne = true; + } + } + } } public String getSourceName() { diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index e4afb552a..710615180 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -111,11 +111,13 @@ public class LexerATNSimulator extends ATNSimulator { public static int ATN_failover = 0; public static int match_calls = 0; - public LexerATNSimulator(@NotNull ATN atn) { - this(null, atn); + public LexerATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA) { + this(null, atn, decisionToDFA); } - public LexerATNSimulator(@Nullable Lexer recog, @NotNull ATN atn) { + public LexerATNSimulator(@Nullable Lexer recog, @NotNull ATN atn, + @NotNull DFA[] decisionToDFA) + { super(atn); dfa = new DFA[atn.modeToStartState.size()]; for (int i=0; i { - public ParserATNPathFinder(@Nullable Parser parser, @NotNull ATN atn) { - super(parser, atn); + public ParserATNPathFinder(@Nullable Parser parser, @NotNull ATN atn, @NotNull DFA[] decisionToDFA) { + super(parser, atn, decisionToDFA); } /** Given an input sequence, as a subset of the input stream, trace the path through the diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 99b547ec5..f35c54aa0 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -150,16 +150,14 @@ public class ParserATNSimulator extends ATNSimulator { protected ParserRuleContext _outerContext; /** Testing only! */ - public ParserATNSimulator(@NotNull ATN atn) { - this(null, atn); + public ParserATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA) { + this(null, atn, decisionToDFA); } - public ParserATNSimulator(@Nullable Parser parser, @NotNull ATN atn) { + public ParserATNSimulator(@Nullable Parser parser, @NotNull ATN atn, @NotNull DFA[] decisionToDFA) { super(atn); this.parser = parser; -// ctxToDFAs = new HashMap(); - // TODO (sam): why distinguish on parser != null? - decisionToDFA = new DFA[atn.getNumberOfDecisions() + (parser != null ? 1 : 0)]; + this.decisionToDFA = decisionToDFA; // DOTGenerator dot = new DOTGenerator(null); // System.out.println(dot.getDOT(atn.rules.get(0), parser.getRuleNames())); // System.out.println(dot.getDOT(atn.rules.get(1), parser.getRuleNames())); @@ -174,22 +172,37 @@ public class ParserATNSimulator extends ATNSimulator { { if ( debug || debug_list_atn_decisions ) { System.out.println("adaptivePredict decision "+decision+ - " exec LA(1)=="+ getLookaheadName(input)+ - " line "+input.LT(1).getLine()+":"+input.LT(1).getCharPositionInLine()); + " exec LA(1)=="+ getLookaheadName(input)+ + " line "+input.LT(1).getLine()+":"+input.LT(1).getCharPositionInLine()); } _input = input; _startIndex = input.index(); _outerContext = outerContext; predict_calls++; DFA dfa = decisionToDFA[decision]; - if ( dfa==null || dfa.s0==null ) { - DecisionState startState = atn.decisionToState.get(decision); - decisionToDFA[decision] = dfa = new DFA(startState, decision); - return predictATN(dfa, input, outerContext); + // First, synchronize on the array of DFA for this parser + // so that we can get the DFA for a decision or create and set one + if ( dfa==null || dfa.s0==null ) { // only create one if not there + synchronized (decisionToDFA) { + dfa = decisionToDFA[decision]; + if ( dfa==null || dfa.s0==null ) { // the usual double-check + DecisionState startState = atn.decisionToState.get(decision); + decisionToDFA[decision] = dfa = new DFA(startState, decision); + } + } + // Now we are certain to have a specific decision's DFA + // Synchronize on the DFA so that nobody can read or write + // to it while we updated during ATN simulation + synchronized (decisionToDFA[decision]) { + return predictATN(dfa, input, outerContext); + } } - else { - //dump(dfa); - // start with the DFA + + // We can start with an existing DFA + synchronized (decisionToDFA[decision]) { + // Only enter the DFA simulation if nobody else is playing with it. + // This blocks multiple readonly simulations of the same DFA but that's + // unlikely to happen a lot int m = input.mark(); int index = input.index(); try { @@ -206,7 +219,7 @@ public class ParserATNSimulator extends ATNSimulator { public int predictATN(@NotNull DFA dfa, @NotNull TokenStream input, @Nullable ParserRuleContext outerContext) { - //contextCache = new PredictionContextCache("predict ctx cache"); + // caller must ensure current thread is sync'd on dfa if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; if ( debug || debug_list_atn_decisions ) { System.out.println("predictATN decision "+dfa.decision+ @@ -246,6 +259,7 @@ public class ParserATNSimulator extends ATNSimulator { @NotNull TokenStream input, int startIndex, @Nullable ParserRuleContext outerContext) { + // caller must ensure current thread is sync'd on dfa if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY; if ( dfa_debug ) { System.out.println("execDFA decision "+dfa.decision+ @@ -555,6 +569,7 @@ public class ParserATNSimulator extends ATNSimulator { int SLL_min_alt, // todo: is this in D as min ambig alts? boolean greedy) { + // caller must ensure current thread is sync'd on dfa retry_with_context++; reportAttemptingFullContext(dfa, s0, startIndex, input.index()); @@ -1364,7 +1379,7 @@ public class ParserATNSimulator extends ATNSimulator { { DFAState from = addDFAState(dfa, p); DFAState to = addDFAState(dfa, q); - if ( debug ) System.out.println("EDGE "+from+" -> "+to+" upon "+getTokenName(t)); + if ( debug ) System.out.println("EDGE "+from+" -> "+to+" upon "+getTokenName(t)); addDFAEdge(from, t, to); if ( debug ) System.out.println("DFA=\n"+dfa.toString(parser!=null?parser.getTokenNames():null)); return to; @@ -1386,19 +1401,12 @@ public class ParserATNSimulator extends ATNSimulator { if ( existing!=null ) return existing; DFAState newState = proposed; - newState.stateNumber = dfa.states.size(); -// System.out.println("Before opt, cache size = "+ sharedContextCache.size()); - configs.optimizeConfigs(this); -// System.out.println("After opt, cache size = " + sharedContextCache.size()); -// System.out.println(configs.size()); -// if ( configs.hasSemanticContext ) System.out.println(configs.getPredicates().size()); -// System.out.println(configs.getPredicates().size()); configs.setReadonly(true); newState.configs = configs; dfa.states.put(newState, newState); - if ( debug ) System.out.println("adding new DFA state: "+newState); + if ( debug ) System.out.println("adding new DFA state: "+newState); return newState; } diff --git a/tool/playground/T.g b/tool/playground/T.g index b489287d6..8d75695e5 100644 --- a/tool/playground/T.g +++ b/tool/playground/T.g @@ -1,10 +1,10 @@ -lexer grammar T; +grammar T; -RBRACE : '}' ; +s : x B ; +x : A B | A | A ; +s2 : x B ; -mode Action; - -END_ACTION - : '}' -> popMode - ; +A : 'a'; +B : 'b' ; +WS : [ \t\n\r]+ -> channel(HIDDEN) ; diff --git a/tool/playground/TestT.java b/tool/playground/TestT.java index d57757d8f..8c7809471 100644 --- a/tool/playground/TestT.java +++ b/tool/playground/TestT.java @@ -27,6 +27,7 @@ public class TestT { parser.addErrorListener(new DiagnosticErrorListener()); ParserRuleContext tree = parser.s(); + System.out.println(tree.toStringTree(parser)); // tree.save(parser, "/tmp/t.ps"); } } diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index 28e3f69a0..fab5251c1 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -118,6 +118,7 @@ Parser(parser, funcs, atn, sempredFuncs, superclass) ::= << Parser_(parser, funcs, atn, sempredFuncs, ctor, extras, superclass) ::= << @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) public abstract class extends { + protected static final DFA[] _decisionToDFA; public static final int =}; separator=", ", wrap, anchor>; @@ -190,7 +191,7 @@ case : return _sempred(()_localctx, predIndex); parser_ctor(p) ::= << public (TokenStream input) { super(input); - _interp = new ParserATNSimulator\(this,_ATN); + _interp = new ParserATNSimulator\(this,_ATN,_decisionToDFA); } >> @@ -714,6 +715,7 @@ import org.antlr.v4.runtime.misc.*; Lexer(lexer, atn, actionFuncs, sempredFuncs) ::= << @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) public abstract class extends Lexer { + protected static final DFA[] _decisionToDFA; public static final int =}; separator=", ", wrap, anchor>; = ;}; separator="\n"> @@ -733,7 +735,7 @@ public abstract class extends public (CharStream input) { super(input); - _interp = new LexerATNSimulator(this,_ATN); + _interp = new LexerATNSimulator(this,_ATN,_decisionToDFA); } @Override @@ -765,6 +767,7 @@ public static final String _serializedATN = public static final ATN _ATN = ATNSimulator.deserialize(_serializedATN.toCharArray()); static { + _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()]; //org.antlr.v4.tool.DOTGenerator dot = new org.antlr.v4.tool.DOTGenerator(null); //System.out.println(dot.getDOT(_ATN.decisionToState.get(0), ruleNames, false)); //System.out.println(dot.getDOT(_ATN.ruleToStartState[2], ruleNames, false)); diff --git a/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java b/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java index 36b5a4801..9ae4544fa 100644 --- a/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java +++ b/tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java @@ -56,7 +56,7 @@ public class LexerInterpreter implements TokenSource { public LexerInterpreter(LexerGrammar g) { Tool antlr = new Tool(); antlr.process(g,false); - interp = new LexerATNSimulator(g.atn); + interp = new LexerATNSimulator(g.atn,null); } public void setInput(String inputString) { diff --git a/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java b/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java index 21318e151..796e4694b 100644 --- a/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java +++ b/tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java @@ -44,11 +44,13 @@ import org.antlr.v4.runtime.misc.Nullable; import org.antlr.v4.tool.Grammar; public class ParserInterpreter { - class DummyParser extends Parser { + public static class DummyParser extends Parser { + public final DFA[] decisionToDFA; // not shared for interp public Grammar g; public DummyParser(Grammar g, TokenStream input) { super(input); this.g = g; + decisionToDFA = new DFA[100]; } @Override @@ -70,9 +72,12 @@ public class ParserInterpreter { public ATN getATN() { return null; } + static { + } } protected Grammar g; + public DummyParser parser; protected ParserATNSimulator atnSimulator; protected TokenStream input; @@ -83,13 +88,15 @@ public class ParserInterpreter { public ParserInterpreter(@NotNull Grammar g, @NotNull TokenStream input) { Tool antlr = new Tool(); antlr.process(g,false); - atnSimulator = new ParserATNSimulator(new DummyParser(g, input), g.atn); + parser = new DummyParser(g, input); + atnSimulator = new ParserATNSimulator(parser, g.atn, parser.decisionToDFA); } - public int predictATN(@NotNull DFA dfa, @NotNull TokenStream input, - @Nullable ParserRuleContext outerContext, - boolean useContext) + public synchronized int predictATN(@NotNull DFA dfa, @NotNull TokenStream input, + @Nullable ParserRuleContext outerContext, + boolean useContext) { + // sync to ensure this entry doesn't race for dfa access return atnSimulator.predictATN(dfa, input, outerContext); } diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java index 2d95f28a4..19d3a7f39 100644 --- a/tool/test/org/antlr/v4/test/BaseTest.java +++ b/tool/test/org/antlr/v4/test/BaseTest.java @@ -211,7 +211,7 @@ public abstract class BaseTest { CharStream input, boolean adaptive) { - LexerATNSimulator interp = new LexerATNSimulator(atn); + LexerATNSimulator interp = new LexerATNSimulator(atn,null); List tokenTypes = new ArrayList(); int ttype; boolean hitEOF = false; diff --git a/tool/test/org/antlr/v4/test/TestATNInterpreter.java b/tool/test/org/antlr/v4/test/TestATNInterpreter.java index 5b10a6f52..cd429aac1 100644 --- a/tool/test/org/antlr/v4/test/TestATNInterpreter.java +++ b/tool/test/org/antlr/v4/test/TestATNInterpreter.java @@ -273,7 +273,7 @@ public class TestATNInterpreter extends BaseTest { int expected) { ATN lexatn = createATN(lg); - LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn); + LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null); List types = getTokenTypesViaATN(inputString, lexInterp); System.out.println(types); diff --git a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java index 4393a5a1b..5e5db1edc 100644 --- a/tool/test/org/antlr/v4/test/TestATNParserPrediction.java +++ b/tool/test/org/antlr/v4/test/TestATNParserPrediction.java @@ -35,7 +35,10 @@ import org.antlr.v4.runtime.NoViableAltException; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.DecisionState; +import org.antlr.v4.runtime.atn.LexerATNSimulator; +import org.antlr.v4.runtime.atn.ParserATNSimulator; import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.tool.DOTGenerator; import org.antlr.v4.tool.Grammar; @@ -476,7 +479,7 @@ public class TestATNParserPrediction extends BaseTest { { Tool.internalOption_ShowATNConfigsInDFA = true; ATN lexatn = createATN(lg); - LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn); + LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null); List types = getTokenTypesViaATN(inputString, lexInterp); System.out.println(types); @@ -522,12 +525,13 @@ public class TestATNParserPrediction extends BaseTest { assertEquals(expectedAlt, alt); } - public DFA getDFA(LexerGrammar lg, Grammar g, String ruleName, - String inputString, ParserRuleContext ctx) + public synchronized DFA getDFA(LexerGrammar lg, Grammar g, String ruleName, + String inputString, ParserRuleContext ctx) { + // sync to ensure multiple tests don't race on dfa access Tool.internalOption_ShowATNConfigsInDFA = true; ATN lexatn = createATN(lg); - LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn); + LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null); semanticProcess(lg); g.importVocab(lg); @@ -541,7 +545,8 @@ public class TestATNParserPrediction extends BaseTest { // System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("b")))); // System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e")))); - ParserATNSimulator interp = new ParserATNSimulator(atn); + ParserATNSimulator interp = + new ParserATNSimulator(atn, new DFA[atn.getNumberOfDecisions()+1]); List types = getTokenTypesViaATN(inputString, lexInterp); System.out.println(types); TokenStream input = new IntTokenStream(types); @@ -563,7 +568,7 @@ public class TestATNParserPrediction extends BaseTest { { // Tool.internalOption_ShowATNConfigsInDFA = true; ATN lexatn = createATN(lg); - LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn); + LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null); semanticProcess(lg); g.importVocab(lg); @@ -581,7 +586,7 @@ public class TestATNParserPrediction extends BaseTest { catch (NoViableAltException nvae) { nvae.printStackTrace(System.err); } - DFA dfa = interp.getATNSimulator().decisionToDFA[decision]; + DFA dfa = interp.parser.decisionToDFA[decision]; assertEquals(dfaString[i], dfa.toString(g.getTokenDisplayNames())); } }