The lexer and parser ATN simulators' adaptivePredict now synchronize on the specific DFA of the decision to be simulated. This should prevent a lot of contention that would occur if we synchronize the entire adaptivePredict method. When the individual DFA are created, we also synchronize on the shared DFA[] table quickly to create a DFA and insert it into the array. Code generation modified to have _decisionToDFA generated at the top of both the parser and the lexer. Simulators created now with the recognizer, ATN, DFA[]. Not sure the LexerInterp/ParserInterp work but pushing ahead anyway for the moment.

runtime/Java/src/org/antlr/v4/runtime/Parser.java

@@ -532,27 +532,31 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator<Token>
 	}
 
     /** For debugging and other purposes */
-    public List<String> getDFAStrings() {
+	public List<String> getDFAStrings() {
-        List<String> s = new ArrayList<String>();
+		synchronized (_interp.decisionToDFA) {
-        for (int d = 0; d < _interp.decisionToDFA.length; d++) {
+			List<String> s = new ArrayList<String>();
-            DFA dfa = _interp.decisionToDFA[d];
+			for (int d = 0; d < _interp.decisionToDFA.length; d++) {
-            s.add( dfa.toString(getTokenNames()) );
+				DFA dfa = _interp.decisionToDFA[d];
-        }
+				s.add( dfa.toString(getTokenNames()) );
-        return s;
+			}
+			return s;
+		}
     }
 
-    /** For debugging and other purposes */
+	/** For debugging and other purposes */
-    public void dumpDFA() {
+	public void dumpDFA() {
-        boolean seenOne = false;
+		synchronized (_interp.decisionToDFA) {
-        for (int d = 0; d < _interp.decisionToDFA.length; d++) {
+			boolean seenOne = false;
-            DFA dfa = _interp.decisionToDFA[d];
+			for (int d = 0; d < _interp.decisionToDFA.length; d++) {
-            if ( dfa!=null ) {
+				DFA dfa = _interp.decisionToDFA[d];
-                if ( seenOne ) System.out.println();
+				if ( dfa!=null ) {
-                System.out.println("Decision " + dfa.decision + ":");
+					if ( seenOne ) System.out.println();
-                System.out.print(dfa.toString(getTokenNames()));
+					System.out.println("Decision " + dfa.decision + ":");
-                seenOne = true;
+					System.out.print(dfa.toString(getTokenNames()));
-            }
+					seenOne = true;
-        }
+				}
+			}
+		}
     }
 
 	public String getSourceName() {

runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java

@@ -111,11 +111,13 @@ public class LexerATNSimulator extends ATNSimulator {
 	public static int ATN_failover = 0;
 	public static int match_calls = 0;
 
-	public LexerATNSimulator(@NotNull ATN atn) {
+	public LexerATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
-		this(null, atn);
+		this(null, atn, decisionToDFA);
 	}
 
-	public LexerATNSimulator(@Nullable Lexer recog, @NotNull ATN atn) {
+	public LexerATNSimulator(@Nullable Lexer recog, @NotNull ATN atn,
+							 @NotNull DFA[] decisionToDFA)
+	{
 		super(atn);
 		dfa = new DFA[atn.modeToStartState.size()];
 		for (int i=0; i<atn.modeToStartState.size(); i++) {

runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNPathFinder.java

@@ -34,6 +34,7 @@ import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.RuleContext;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.TokenStream;
+import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.runtime.misc.IntervalSet;
 import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.misc.Nullable;
@@ -45,8 +46,8 @@ import java.util.List;
 import java.util.Set;
 
 public class ParserATNPathFinder extends ParserATNSimulator<Token> {
-	public ParserATNPathFinder(@Nullable Parser parser, @NotNull ATN atn) {
+	public ParserATNPathFinder(@Nullable Parser parser, @NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
-		super(parser, atn);
+		super(parser, atn, decisionToDFA);
 	}
 
 	/** Given an input sequence, as a subset of the input stream, trace the path through the

runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java

@@ -150,16 +150,14 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 	protected ParserRuleContext<?> _outerContext;
 
 	/** Testing only! */
-	public ParserATNSimulator(@NotNull ATN atn) {
+	public ParserATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
-		this(null, atn);
+		this(null, atn, decisionToDFA);
 	}
 
-	public ParserATNSimulator(@Nullable Parser parser, @NotNull ATN atn) {
+	public ParserATNSimulator(@Nullable Parser parser, @NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
 		super(atn);
 		this.parser = parser;
-//		ctxToDFAs = new HashMap<RuleContext, DFA[]>();
+		this.decisionToDFA = decisionToDFA;
-		// TODO (sam): why distinguish on parser != null?
-		decisionToDFA = new DFA[atn.getNumberOfDecisions() + (parser != null ? 1 : 0)];
 		//		DOTGenerator dot = new DOTGenerator(null);
 		//		System.out.println(dot.getDOT(atn.rules.get(0), parser.getRuleNames()));
 		//		System.out.println(dot.getDOT(atn.rules.get(1), parser.getRuleNames()));
@@ -174,22 +172,37 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 	{
 		if ( debug || debug_list_atn_decisions )  {
 			System.out.println("adaptivePredict decision "+decision+
-							   " exec LA(1)=="+ getLookaheadName(input)+
+								   " exec LA(1)=="+ getLookaheadName(input)+
-							   " line "+input.LT(1).getLine()+":"+input.LT(1).getCharPositionInLine());
+								   " line "+input.LT(1).getLine()+":"+input.LT(1).getCharPositionInLine());
 		}
 		_input = input;
 		_startIndex = input.index();
 		_outerContext = outerContext;
 		predict_calls++;
 		DFA dfa = decisionToDFA[decision];
-		if ( dfa==null || dfa.s0==null ) {
+		// First, synchronize on the array of DFA for this parser
-			DecisionState startState = atn.decisionToState.get(decision);
+		// so that we can get the DFA for a decision or create and set one
-			decisionToDFA[decision] = dfa = new DFA(startState, decision);
+		if ( dfa==null || dfa.s0==null ) { // only create one if not there
-			return predictATN(dfa, input, outerContext);
+			synchronized (decisionToDFA) {
+				dfa = decisionToDFA[decision];
+				if ( dfa==null || dfa.s0==null ) { // the usual double-check
+					DecisionState startState = atn.decisionToState.get(decision);
+					decisionToDFA[decision] = dfa = new DFA(startState, decision);
+				}
+			}
+			// Now we are certain to have a specific decision's DFA
+			// Synchronize on the DFA so that nobody can read or write
+			// to it while we updated during ATN simulation
+			synchronized (decisionToDFA[decision]) {
+				return predictATN(dfa, input, outerContext);
+			}
 		}
-		else {
+
-			//dump(dfa);
+		// We can start with an existing DFA
-			// start with the DFA
+		synchronized (decisionToDFA[decision]) {
+			// Only enter the DFA simulation if nobody else is playing with it.
+			// This blocks multiple readonly simulations of the same DFA but that's
+			// unlikely to happen a lot
 			int m = input.mark();
 			int index = input.index();
 			try {
@@ -206,7 +219,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 	public int predictATN(@NotNull DFA dfa, @NotNull TokenStream input,
 						  @Nullable ParserRuleContext<?> outerContext)
 	{
-		//contextCache = new PredictionContextCache("predict ctx cache");
+		// caller must ensure current thread is sync'd on dfa
 		if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY;
 		if ( debug || debug_list_atn_decisions )  {
 			System.out.println("predictATN decision "+dfa.decision+
@@ -246,6 +259,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 					   @NotNull TokenStream input, int startIndex,
                        @Nullable ParserRuleContext<?> outerContext)
     {
+		// caller must ensure current thread is sync'd on dfa
 		if ( outerContext==null ) outerContext = ParserRuleContext.EMPTY;
 		if ( dfa_debug ) {
 			System.out.println("execDFA decision "+dfa.decision+
@@ -555,6 +569,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 											   int SLL_min_alt, // todo: is this in D as min ambig alts?
 											   boolean greedy)
 	{
+		// caller must ensure current thread is sync'd on dfa
 		retry_with_context++;
 		reportAttemptingFullContext(dfa, s0, startIndex, input.index());
 
@@ -1364,7 +1379,7 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 	{
 		DFAState from = addDFAState(dfa, p);
 		DFAState to = addDFAState(dfa, q);
-        if ( debug ) System.out.println("EDGE "+from+" -> "+to+" upon "+getTokenName(t));
+		if ( debug ) System.out.println("EDGE "+from+" -> "+to+" upon "+getTokenName(t));
 		addDFAEdge(from, t, to);
 		if ( debug ) System.out.println("DFA=\n"+dfa.toString(parser!=null?parser.getTokenNames():null));
 		return to;
@@ -1386,19 +1401,12 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
 		if ( existing!=null ) return existing;
 
 		DFAState newState = proposed;
-
 		newState.stateNumber = dfa.states.size();
-//		System.out.println("Before opt, cache size = "+ sharedContextCache.size());
-
 		configs.optimizeConfigs(this);
-//		System.out.println("After opt, cache size = " + sharedContextCache.size());
-//		System.out.println(configs.size());
-//		if ( configs.hasSemanticContext ) System.out.println(configs.getPredicates().size());
-//		System.out.println(configs.getPredicates().size());
 		configs.setReadonly(true);
 		newState.configs = configs;
 		dfa.states.put(newState, newState);
-        if ( debug ) System.out.println("adding new DFA state: "+newState);
+		if ( debug ) System.out.println("adding new DFA state: "+newState);
 		return newState;
 	}
 

tool/playground/T.g

@@ -1,10 +1,10 @@
-lexer grammar T;
+grammar T;
 
-RBRACE : '}' ;
+s : x B ;
+x : A B | A | A ;
+s2 : x B ;
 
-mode Action;
-
-END_ACTION
-    :   '}' -> popMode
-    ;
 
+A : 'a';
+B : 'b' ;
+WS : [ \t\n\r]+ -> channel(HIDDEN) ;

tool/playground/TestT.java

@@ -27,6 +27,7 @@ public class TestT {
 		parser.addErrorListener(new DiagnosticErrorListener());
 
 		ParserRuleContext tree = parser.s();
+		System.out.println(tree.toStringTree(parser));
 //		tree.save(parser, "/tmp/t.ps");
 	}
 }

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -118,6 +118,7 @@ Parser(parser, funcs, atn, sempredFuncs, superclass) ::= <<
 Parser_(parser, funcs, atn, sempredFuncs, ctor, extras, superclass) ::= <<
 @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
 public <if(parser.abstractRecognizer)>abstract <endif>class <parser.name> extends <superclass> {
+	protected static final DFA[] _decisionToDFA;
 	<if(parser.tokens)>
 	public static final int
 		<parser.tokens:{k | <k>=<parser.tokens.(k)>}; separator=", ", wrap, anchor>;
@@ -190,7 +191,7 @@ case <f.ruleIndex> : return <f.name>_sempred((<f.ctxType>)_localctx, predIndex);
 parser_ctor(p) ::= <<
 public <p.name>(TokenStream input) {
 	super(input);
-	_interp = new ParserATNSimulator\<Token>(this,_ATN);
+	_interp = new ParserATNSimulator\<Token>(this,_ATN,_decisionToDFA);
 }
 >>
 
@@ -714,6 +715,7 @@ import org.antlr.v4.runtime.misc.*;
 Lexer(lexer, atn, actionFuncs, sempredFuncs) ::= <<
 @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
 public <if(lexer.abstractRecognizer)>abstract <endif>class <lexer.name> extends Lexer {
+	protected static final DFA[] _decisionToDFA;
 	public static final int
 		<lexer.tokens:{k | <k>=<lexer.tokens.(k)>}; separator=", ", wrap, anchor>;
 	<rest(lexer.modes):{m| public static final int <m> = <i>;}; separator="\n">
@@ -733,7 +735,7 @@ public <if(lexer.abstractRecognizer)>abstract <endif>class <lexer.name> extends
 
 	public <lexer.name>(CharStream input) {
 		super(input);
-		_interp = new LexerATNSimulator(this,_ATN);
+		_interp = new LexerATNSimulator(this,_ATN,_decisionToDFA);
 	}
 
 	@Override
@@ -765,6 +767,7 @@ public static final String _serializedATN =
 public static final ATN _ATN =
 	ATNSimulator.deserialize(_serializedATN.toCharArray());
 static {
+    _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
 	//org.antlr.v4.tool.DOTGenerator dot = new org.antlr.v4.tool.DOTGenerator(null);
 	//System.out.println(dot.getDOT(_ATN.decisionToState.get(0), ruleNames, false));
 	//System.out.println(dot.getDOT(_ATN.ruleToStartState[2], ruleNames, false));

tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java

@@ -56,7 +56,7 @@ public class LexerInterpreter implements TokenSource {
 	public LexerInterpreter(LexerGrammar g) {
 		Tool antlr = new Tool();
 		antlr.process(g,false);
-		interp = new LexerATNSimulator(g.atn);
+		interp = new LexerATNSimulator(g.atn,null);
 	}
 
 	public void setInput(String inputString) {

tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java

@@ -44,11 +44,13 @@ import org.antlr.v4.runtime.misc.Nullable;
 import org.antlr.v4.tool.Grammar;
 
 public class ParserInterpreter {
-	class DummyParser extends Parser {
+	public static class DummyParser extends Parser {
+		public final DFA[] decisionToDFA; // not shared for interp
 		public Grammar g;
 		public DummyParser(Grammar g, TokenStream input) {
 			super(input);
 			this.g = g;
+			decisionToDFA = new DFA[100];
 		}
 
 		@Override
@@ -70,9 +72,12 @@ public class ParserInterpreter {
 		public ATN getATN() {
 			return null;
 		}
+		static {
+		}
 	}
 
 	protected Grammar g;
+	public DummyParser parser;
 	protected ParserATNSimulator<Token> atnSimulator;
 	protected TokenStream input;
 
@@ -83,13 +88,15 @@ public class ParserInterpreter {
 	public ParserInterpreter(@NotNull Grammar g, @NotNull TokenStream input) {
 		Tool antlr = new Tool();
 		antlr.process(g,false);
-		atnSimulator = new ParserATNSimulator<Token>(new DummyParser(g, input), g.atn);
+		parser = new DummyParser(g, input);
+		atnSimulator = new ParserATNSimulator<Token>(parser, g.atn, parser.decisionToDFA);
 	}
 
-	public int predictATN(@NotNull DFA dfa, @NotNull TokenStream input,
+	public synchronized int predictATN(@NotNull DFA dfa, @NotNull TokenStream input,
-						  @Nullable ParserRuleContext outerContext,
+									   @Nullable ParserRuleContext outerContext,
-						  boolean useContext)
+									   boolean useContext)
 	{
+		// sync to ensure this entry doesn't race for dfa access
 		return atnSimulator.predictATN(dfa, input, outerContext);
 	}
 

tool/test/org/antlr/v4/test/BaseTest.java

@@ -211,7 +211,7 @@ public abstract class BaseTest {
 									  CharStream input,
 									  boolean adaptive)
 	{
-		LexerATNSimulator interp = new LexerATNSimulator(atn);
+		LexerATNSimulator interp = new LexerATNSimulator(atn,null);
 		List<String> tokenTypes = new ArrayList<String>();
 		int ttype;
 		boolean hitEOF = false;

tool/test/org/antlr/v4/test/TestATNInterpreter.java

@@ -273,7 +273,7 @@ public class TestATNInterpreter extends BaseTest {
 								int expected)
 	{
 		ATN lexatn = createATN(lg);
-		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn);
+		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
 		List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
 		System.out.println(types);
 

tool/test/org/antlr/v4/test/TestATNParserPrediction.java

@@ -35,7 +35,10 @@ import org.antlr.v4.runtime.NoViableAltException;
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.TokenStream;
-import org.antlr.v4.runtime.atn.*;
+import org.antlr.v4.runtime.atn.ATN;
+import org.antlr.v4.runtime.atn.DecisionState;
+import org.antlr.v4.runtime.atn.LexerATNSimulator;
+import org.antlr.v4.runtime.atn.ParserATNSimulator;
 import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.tool.DOTGenerator;
 import org.antlr.v4.tool.Grammar;
@@ -476,7 +479,7 @@ public class TestATNParserPrediction extends BaseTest {
 	{
 		Tool.internalOption_ShowATNConfigsInDFA = true;
 		ATN lexatn = createATN(lg);
-		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn);
+		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
 		List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
 		System.out.println(types);
 
@@ -522,12 +525,13 @@ public class TestATNParserPrediction extends BaseTest {
 		assertEquals(expectedAlt, alt);
 	}
 
-	public DFA getDFA(LexerGrammar lg, Grammar g, String ruleName,
+	public synchronized DFA getDFA(LexerGrammar lg, Grammar g, String ruleName,
-					  String inputString, ParserRuleContext<?> ctx)
+								   String inputString, ParserRuleContext<?> ctx)
 	{
+		// sync to ensure multiple tests don't race on dfa access
 		Tool.internalOption_ShowATNConfigsInDFA = true;
 		ATN lexatn = createATN(lg);
-		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn);
+		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
 
 		semanticProcess(lg);
 		g.importVocab(lg);
@@ -541,7 +545,8 @@ public class TestATNParserPrediction extends BaseTest {
 //		System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("b"))));
 //		System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e"))));
 
-		ParserATNSimulator<Token> interp = new ParserATNSimulator<Token>(atn);
+		ParserATNSimulator<Token> interp =
+			new ParserATNSimulator<Token>(atn, new DFA[atn.getNumberOfDecisions()+1]);
 		List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
 		System.out.println(types);
 		TokenStream input = new IntTokenStream(types);
@@ -563,7 +568,7 @@ public class TestATNParserPrediction extends BaseTest {
 	{
 //		Tool.internalOption_ShowATNConfigsInDFA = true;
 		ATN lexatn = createATN(lg);
-		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn);
+		LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
 
 		semanticProcess(lg);
 		g.importVocab(lg);
@@ -581,7 +586,7 @@ public class TestATNParserPrediction extends BaseTest {
 			catch (NoViableAltException nvae) {
 				nvae.printStackTrace(System.err);
 			}
-			DFA dfa = interp.getATNSimulator().decisionToDFA[decision];
+			DFA dfa = interp.parser.decisionToDFA[decision];
 			assertEquals(dfaString[i], dfa.toString(g.getTokenDisplayNames()));
 		}
 	}