jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

synchronized lexer. just as fast now to build new parser/lexer each iter in TestJavaLR

This commit is contained in:
Terence Parr 2012-07-29 10:48:08 -07:00
parent ceab49e3ce
commit 79594a942e
14 changed files with 83 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java
View File

@ -71,6 +71,7 @@ public class ATN {
// runtime for lexer only
public int[] ruleToTokenType;
public int[] ruleToActionIndex;
@NotNull
public final List<TokensStartState> modeToStartState = new ArrayList<TokensStartState>();

8
runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java
View File

@ -64,16 +64,18 @@ public abstract class ATNSimulator {
* more time I think and doesn't save on the overall footprint
* so it's not worth the complexity.
*/
protected final PredictionContextCache sharedContextCache =
new PredictionContextCache("shared DFA state context cache");
protected final PredictionContextCache sharedContextCache;
static {
ERROR = new DFAState(new ATNConfigSet());
ERROR.stateNumber = Integer.MAX_VALUE;
}
public ATNSimulator(@NotNull ATN atn) {
public ATNSimulator(@NotNull ATN atn,
@NotNull PredictionContextCache sharedContextCache)
{
this.atn = atn;
this.sharedContextCache = sharedContextCache;
}
public abstract void reset();

58
runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java
View File

@ -101,7 +101,7 @@ public class LexerATNSimulator extends ATNSimulator {
protected int charPositionInLine = 0;
@NotNull
public final DFA[] dfa;
public final DFA[] decisionToDFA;
protected int mode = Lexer.DEFAULT_MODE;
/** Used during DFA/ATN exec to record the most recent accept configuration info */
@ -111,17 +111,26 @@ public class LexerATNSimulator extends ATNSimulator {
public static int ATN_failover = 0;
public static int match_calls = 0;
public LexerATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
this(null, atn, decisionToDFA);
public LexerATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA,
@NotNull PredictionContextCache sharedContextCache)
{
this(null, atn, decisionToDFA,sharedContextCache);
}
public LexerATNSimulator(@Nullable Lexer recog, @NotNull ATN atn,
@NotNull DFA[] decisionToDFA)
@NotNull DFA[] decisionToDFA,
@NotNull PredictionContextCache sharedContextCache)
{
super(atn);
dfa = new DFA[atn.modeToStartState.size()];
for (int i=0; i<atn.modeToStartState.size(); i++) {
dfa[i] = new DFA(atn.modeToStartState.get(i));
super(atn,sharedContextCache);
this.decisionToDFA = decisionToDFA;
if ( decisionToDFA[Lexer.DEFAULT_MODE]==null ) { // create all mode dfa
synchronized (this.decisionToDFA) {
if ( decisionToDFA[Lexer.DEFAULT_MODE]==null ) { // create all mode dfa
for (int i=0; i<atn.modeToStartState.size(); i++) {
this.decisionToDFA[i] = new DFA(atn.modeToStartState.get(i));
}
}
}
}
this.recog = recog;
}
@ -153,14 +162,19 @@ public class LexerATNSimulator extends ATNSimulator {
int mark = input.mark();
traceBeginMatch(input, mode);
try {
if ( dfa[mode].s0==null ) {
return matchATN(input);
}
else {
return execDFA(input, dfa[mode].s0);
synchronized (decisionToDFA[mode]) {
// Synchronizing on the mode DFA causes a lot of contention
// between shared lexers in multiple threads, but is the
// simplest safety measure. We can start with it.
if ( decisionToDFA[mode].s0==null ) {
return matchATN(input);
}
else {
return execDFA(input, decisionToDFA[mode].s0);
}
}
}
finally {
finally {
traceEndMatch();
input.release(mark);
}
@ -188,11 +202,11 @@ public class LexerATNSimulator extends ATNSimulator {
ATNConfigSet s0_closure = computeStartState(input, startState);
int old_mode = mode;
dfa[mode].s0 = addDFAState(s0_closure);
int predict = execATN(input, s0_closure, dfa[mode].s0);
decisionToDFA[mode].s0 = addDFAState(s0_closure);
int predict = execATN(input, s0_closure, decisionToDFA[mode].s0);
if ( debug ) {
System.out.format("DFA after matchATN: %s\n", dfa[old_mode].toLexerString());
System.out.format("DFA after matchATN: %s\n", decisionToDFA[old_mode].toLexerString());
}
tracePredict(predict);
@ -642,7 +656,7 @@ public class LexerATNSimulator extends ATNSimulator {
if ( dfa_debug ) {
System.out.format("back from DFA update, ttype=%d, dfa[mode %d]=\n%s\n",
ttype, mode, dfa[mode].toLexerString());
ttype, mode, decisionToDFA[mode].toLexerString());
}
// action already executed by ATN
@ -714,7 +728,7 @@ public class LexerATNSimulator extends ATNSimulator {
if ( configs.hasSemanticContext ) return null;
DFAState proposed = new DFAState(configs);
DFAState existing = dfa[mode].states.get(proposed);
DFAState existing = decisionToDFA[mode].states.get(proposed);
if ( existing!=null ) return existing;
DFAState newState = proposed;
@ -735,16 +749,16 @@ public class LexerATNSimulator extends ATNSimulator {
newState.prediction = atn.ruleToTokenType[newState.lexerRuleIndex];
}
newState.stateNumber = dfa[mode].states.size();
newState.stateNumber = decisionToDFA[mode].states.size();
configs.setReadonly(true);
newState.configs = configs;
dfa[mode].states.put(newState, newState);
decisionToDFA[mode].states.put(newState, newState);
return newState;
}
@Nullable
public DFA getDFA(int mode) {
return dfa[mode];
return decisionToDFA[mode];
}
/** Get the text of the current token from an *action* in lexer not

13
runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java
View File

@ -150,12 +150,17 @@ public class ParserATNSimulator<Symbol extends Token> extends ATNSimulator {
protected ParserRuleContext<?> _outerContext;
/** Testing only! */
public ParserATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
this(null, atn, decisionToDFA);
public ParserATNSimulator(@NotNull ATN atn, @NotNull DFA[] decisionToDFA,
@NotNull PredictionContextCache sharedContextCache)
{
this(null, atn, decisionToDFA, sharedContextCache);
}
public ParserATNSimulator(@Nullable Parser parser, @NotNull ATN atn, @NotNull DFA[] decisionToDFA) {
super(atn);
public ParserATNSimulator(@Nullable Parser parser, @NotNull ATN atn,
@NotNull DFA[] decisionToDFA,
@NotNull PredictionContextCache sharedContextCache)
{
super(atn,sharedContextCache);
this.parser = parser;
this.decisionToDFA = decisionToDFA;
// DOTGenerator dot = new DOTGenerator(null);

9
runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java
View File

@ -13,14 +13,15 @@ public class PredictionContextCache {
protected Map<PredictionContext, PredictionContext> cache =
new HashMap<PredictionContext, PredictionContext>();
public PredictionContextCache(String name) {
public PredictionContextCache() {
this.name = name;
}
/** Add a context to the cache and return it. If the context already exists,
* return that one instead and do not add a new context to the cache.
* Protect shared cache from unsafe thread access.
*/
public PredictionContext add(PredictionContext ctx) {
public synchronized PredictionContext add(PredictionContext ctx) {
if ( ctx==PredictionContext.EMPTY ) return PredictionContext.EMPTY;
PredictionContext existing = cache.get(ctx);
if ( existing!=null ) {
@ -31,11 +32,11 @@ public class PredictionContextCache {
return ctx;
}
public PredictionContext get(PredictionContext ctx) {
public synchronized PredictionContext get(PredictionContext ctx) {
return cache.get(ctx);
}
public int size() {
public synchronized int size() {
return cache.size();
}
}

5
tool/playground/TestJavaLR.java
View File

@ -49,6 +49,8 @@ class TestJavaLR {
public static boolean SLL = false;
public static boolean diag = false;
public static boolean bail = false;
public static boolean x2 = false;
public static boolean threaded = false;
public static void main(String[] args) {
doAll(args);
@ -68,6 +70,8 @@ class TestJavaLR {
else if ( args[i].equals("-SLL") ) SLL = true;
else if ( args[i].equals("-bail") ) bail = true;
else if ( args[i].equals("-diag") ) diag = true;
else if ( args[i].equals("-2x") ) x2 = true;
else if ( args[i].equals("-threaded") ) threaded = true;
doFile(new File(args[i])); // parse it
}
}
@ -151,6 +155,7 @@ class TestJavaLR {
if ( printTree ) System.out.println(t.toStringTree(parser));
//System.err.println("finished "+f);
// System.out.println("cache size = "+DefaultErrorStrategy.cache.size());
lexer=null; parser=null; // force rebuild
}
}
catch (Exception e) {

8
tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View File

@ -119,6 +119,8 @@ Parser_(parser, funcs, atn, sempredFuncs, ctor, extras, superclass) ::= <<
@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
public <if(parser.abstractRecognizer)>abstract <endif>class <parser.name> extends <superclass> {
protected static final DFA[] _decisionToDFA;
protected static final PredictionContextCache _sharedContextCache =
new PredictionContextCache();
<if(parser.tokens)>
public static final int
<parser.tokens:{k | <k>=<parser.tokens.(k)>}; separator=", ", wrap, anchor>;
@ -191,7 +193,7 @@ case <f.ruleIndex> : return <f.name>_sempred((<f.ctxType>)_localctx, predIndex);
parser_ctor(p) ::= <<
public <p.name>(TokenStream input) {
super(input);
_interp = new ParserATNSimulator\<Token>(this,_ATN,_decisionToDFA);
_interp = new ParserATNSimulator\<Token>(this,_ATN,_decisionToDFA,_sharedContextCache);
}
>>
@ -716,6 +718,8 @@ Lexer(lexer, atn, actionFuncs, sempredFuncs) ::= <<
@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
public <if(lexer.abstractRecognizer)>abstract <endif>class <lexer.name> extends Lexer {
protected static final DFA[] _decisionToDFA;
protected static final PredictionContextCache _sharedContextCache =
new PredictionContextCache();
public static final int
<lexer.tokens:{k | <k>=<lexer.tokens.(k)>}; separator=", ", wrap, anchor>;
<rest(lexer.modes):{m| public static final int <m> = <i>;}; separator="\n">
@ -735,7 +739,7 @@ public <if(lexer.abstractRecognizer)>abstract <endif>class <lexer.name> extends
public <lexer.name>(CharStream input) {
super(input);
_interp = new LexerATNSimulator(this,_ATN,_decisionToDFA);
_interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache);
}
@Override

2
tool/src/org/antlr/v4/tool/interp/LexerInterpreter.java
View File

@ -56,7 +56,7 @@ public class LexerInterpreter implements TokenSource {
public LexerInterpreter(LexerGrammar g) {
Tool antlr = new Tool();
antlr.process(g,false);
interp = new LexerATNSimulator(g.atn,null);
interp = new LexerATNSimulator(g.atn,null,null);
}
public void setInput(String inputString) {

8
tool/src/org/antlr/v4/tool/interp/ParserInterpreter.java
View File

@ -38,6 +38,7 @@ import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.ParserATNSimulator;
import org.antlr.v4.runtime.atn.PredictionContextCache;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable;
@ -46,6 +47,9 @@ import org.antlr.v4.tool.Grammar;
public class ParserInterpreter {
public static class DummyParser extends Parser {
public final DFA[] decisionToDFA; // not shared for interp
public final PredictionContextCache sharedContextCache =
new PredictionContextCache();
public Grammar g;
public DummyParser(Grammar g, TokenStream input) {
super(input);
@ -89,7 +93,9 @@ public class ParserInterpreter {
Tool antlr = new Tool();
antlr.process(g,false);
parser = new DummyParser(g, input);
atnSimulator = new ParserATNSimulator<Token>(parser, g.atn, parser.decisionToDFA);
atnSimulator =
new ParserATNSimulator<Token>(parser, g.atn, parser.decisionToDFA,
parser.sharedContextCache);
}
public synchronized int predictATN(@NotNull DFA dfa, @NotNull TokenStream input,

2
tool/test/org/antlr/v4/test/BaseTest.java
View File

@ -211,7 +211,7 @@ public abstract class BaseTest {
CharStream input,
boolean adaptive)
{
LexerATNSimulator interp = new LexerATNSimulator(atn,null);
LexerATNSimulator interp = new LexerATNSimulator(atn,null,null);
List<String> tokenTypes = new ArrayList<String>();
int ttype;
boolean hitEOF = false;

2
tool/test/org/antlr/v4/test/TestATNInterpreter.java
View File

@ -273,7 +273,7 @@ public class TestATNInterpreter extends BaseTest {
int expected)
{
ATN lexatn = createATN(lg);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null,null);
List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
System.out.println(types);

8
tool/test/org/antlr/v4/test/TestATNParserPrediction.java
View File

@ -479,7 +479,7 @@ public class TestATNParserPrediction extends BaseTest {
{
Tool.internalOption_ShowATNConfigsInDFA = true;
ATN lexatn = createATN(lg);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null,null);
List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
System.out.println(types);
@ -531,7 +531,7 @@ public class TestATNParserPrediction extends BaseTest {
// sync to ensure multiple tests don't race on dfa access
Tool.internalOption_ShowATNConfigsInDFA = true;
ATN lexatn = createATN(lg);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null,null);
semanticProcess(lg);
g.importVocab(lg);
@ -546,7 +546,7 @@ public class TestATNParserPrediction extends BaseTest {
// System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e"))));
ParserATNSimulator<Token> interp =
new ParserATNSimulator<Token>(atn, new DFA[atn.getNumberOfDecisions()+1]);
new ParserATNSimulator<Token>(atn, new DFA[atn.getNumberOfDecisions()],null);
List<Integer> types = getTokenTypesViaATN(inputString, lexInterp);
System.out.println(types);
TokenStream input = new IntTokenStream(types);
@ -568,7 +568,7 @@ public class TestATNParserPrediction extends BaseTest {
{
// Tool.internalOption_ShowATNConfigsInDFA = true;
ATN lexatn = createATN(lg);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null,null);
semanticProcess(lg);
g.importVocab(lg);

2
tool/test/org/antlr/v4/test/TestGraphNodes.java
View File

@ -13,7 +13,7 @@ public class TestGraphNodes extends TestCase {
@Override
protected void setUp() throws Exception {
PredictionContext.globalNodeCount = 1;
contextCache = new PredictionContextCache("testing");
contextCache = new PredictionContextCache();
}
public boolean rootIsWildcard() { return true; }

2
tool/test/org/antlr/v4/test/TestPerformance.java
View File

@ -317,7 +317,7 @@ public class TestPerformance extends BaseTest {
System.currentTimeMillis() - startTime);
final LexerATNSimulator lexerInterpreter = sharedLexer.getInterpreter();
final DFA[] modeToDFA = lexerInterpreter.dfa;
final DFA[] modeToDFA = lexerInterpreter.decisionToDFA;
if (SHOW_DFA_STATE_STATS) {
int states = 0;
int configs = 0;