forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Make sure to serialize/deserialize the ATN before using the interpreter since the runtime alters the ATN on load

This commit is contained in:
Sam Harwell 2012-10-14 12:00:39 -05:00
parent b3b1a3750d
commit 850fe5b9c2
6 changed files with 53 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
tool/test/org/antlr/v4/test/BaseTest.java
View File

@ -32,6 +32,7 @@ package org.antlr.v4.test;
import org.antlr.v4.Tool; import org.antlr.v4.Tool;
import org.antlr.v4.automata.ATNFactory; import org.antlr.v4.automata.ATNFactory;
import org.antlr.v4.automata.ATNPrinter; import org.antlr.v4.automata.ATNPrinter;
import org.antlr.v4.automata.ATNSerializer;
import org.antlr.v4.automata.LexerATNFactory; import org.antlr.v4.automata.LexerATNFactory;
import org.antlr.v4.automata.ParserATNFactory; import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.CodeGenerator;
@ -47,6 +48,7 @@ import org.antlr.v4.runtime.TokenSource;
import org.antlr.v4.runtime.TokenStream; import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.WritableToken; import org.antlr.v4.runtime.WritableToken;
import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNSimulator;
import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.DecisionState; import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.LexerATNSimulator; import org.antlr.v4.runtime.atn.LexerATNSimulator;
@ -148,15 +150,28 @@ public abstract class BaseTest {
return tool; return tool;
} }
ATN createATN(Grammar g) { protected ATN createATN(Grammar g, boolean useSerializer) {
if ( g.atn!=null ) return g.atn; if ( g.atn==null ) {
semanticProcess(g); semanticProcess(g);
ParserATNFactory f = new ParserATNFactory(g); ParserATNFactory f;
if ( g.isLexer() ) f = new LexerATNFactory((LexerGrammar)g); if ( g.isLexer() ) {
g.atn = f.createATN(); f = new LexerATNFactory((LexerGrammar)g);
}
else {
f = new ParserATNFactory(g);
}
return g.atn; g.atn = f.createATN();
}
ATN atn = g.atn;
if (useSerializer) {
char[] serialized = ATNSerializer.getSerializedAsChars(g, atn);
return ATNSimulator.deserialize(serialized);
}
return atn;
} }
protected void semanticProcess(Grammar g) { protected void semanticProcess(Grammar g) {
@ -245,7 +260,7 @@ public abstract class BaseTest {
{ {
ErrorQueue equeue = new ErrorQueue(); ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(gtext, equeue); Grammar g = new Grammar(gtext, equeue);
ATN atn = createATN(g); ATN atn = createATN(g, false);
ATNState s = atn.ruleToStartState[g.getRule(ruleName).index]; ATNState s = atn.ruleToStartState[g.getRule(ruleName).index];
if ( s==null ) { if ( s==null ) {
System.err.println("no such rule: "+ruleName); System.err.println("no such rule: "+ruleName);
@ -266,7 +281,7 @@ public abstract class BaseTest {
{ {
ErrorQueue equeue = new ErrorQueue(); ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(gtext, equeue); Grammar g = new Grammar(gtext, equeue);
ATN atn = createATN(g); ATN atn = createATN(g, false);
DecisionState blk = atn.decisionToState.get(decision); DecisionState blk = atn.decisionToState.get(decision);
checkRuleDFA(g, blk, expecting); checkRuleDFA(g, blk, expecting);
return equeue.all; return equeue.all;
@ -292,7 +307,7 @@ public abstract class BaseTest {
{ {
ErrorQueue equeue = new ErrorQueue(); ErrorQueue equeue = new ErrorQueue();
LexerGrammar g = new LexerGrammar(gtext, equeue); LexerGrammar g = new LexerGrammar(gtext, equeue);
g.atn = createATN(g); g.atn = createATN(g, false);
// LexerATNToDFAConverter conv = new LexerATNToDFAConverter(g); // LexerATNToDFAConverter conv = new LexerATNToDFAConverter(g);
// DFA dfa = conv.createDFA(modeName); // DFA dfa = conv.createDFA(modeName);
// g.setLookaheadDFA(0, dfa); // only one decision to worry about // g.setLookaheadDFA(0, dfa); // only one decision to worry about

2
tool/test/org/antlr/v4/test/TestATNDeserialization.java
View File

@ -144,7 +144,7 @@ public class TestATNDeserialization extends BaseTest {
} }
protected void checkDeserializationIsStable(Grammar g) { protected void checkDeserializationIsStable(Grammar g) {
ATN atn = createATN(g); ATN atn = createATN(g, false);
char[] data = Utils.toCharArray(ATNSerializer.getSerialized(g, atn)); char[] data = Utils.toCharArray(ATNSerializer.getSerialized(g, atn));
String atnData = ATNSerializer.getDecoded(g, atn); String atnData = ATNSerializer.getDecoded(g, atn);
ATN atn2 = ParserATNSimulator.deserialize(data); ATN atn2 = ParserATNSimulator.deserialize(data);

2
tool/test/org/antlr/v4/test/TestATNInterpreter.java
View File

@ -271,7 +271,7 @@ public class TestATNInterpreter extends BaseTest {
String inputString, String inputString,
int expected) int expected)
{ {
ATN lexatn = createATN(lg); ATN lexatn = createATN(lg, true);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,new DFA[1],null); LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,new DFA[1],null);
IntegerList types = getTokenTypesViaATN(inputString, lexInterp); IntegerList types = getTokenTypesViaATN(inputString, lexInterp);
System.out.println(types); System.out.println(types);

2
tool/test/org/antlr/v4/test/TestATNLexerInterpreter.java
View File

@ -266,7 +266,7 @@ public class TestATNLexerInterpreter extends BaseTest {
} }
protected RecognitionException checkLexerMatches(LexerGrammar lg, String inputString, String expecting) { protected RecognitionException checkLexerMatches(LexerGrammar lg, String inputString, String expecting) {
ATN atn = createATN(lg); ATN atn = createATN(lg, true);
CharStream input = new ANTLRInputStream(inputString); CharStream input = new ANTLRInputStream(inputString);
ATNState startState = atn.modeNameToStartState.get("DEFAULT_MODE"); ATNState startState = atn.modeNameToStartState.get("DEFAULT_MODE");
DOTGenerator dot = new DOTGenerator(lg); DOTGenerator dot = new DOTGenerator(lg);

6
tool/test/org/antlr/v4/test/TestATNParserPrediction.java
View File

@ -455,7 +455,7 @@ public class TestATNParserPrediction extends BaseTest {
String inputString, int expectedAlt) String inputString, int expectedAlt)
{ {
Tool.internalOption_ShowATNConfigsInDFA = true; Tool.internalOption_ShowATNConfigsInDFA = true;
ATN lexatn = createATN(lg); ATN lexatn = createATN(lg, true);
LexerATNSimulator lexInterp = LexerATNSimulator lexInterp =
new LexerATNSimulator(lexatn,new DFA[1],new PredictionContextCache()); new LexerATNSimulator(lexatn,new DFA[1],new PredictionContextCache());
IntegerList types = getTokenTypesViaATN(inputString, lexInterp); IntegerList types = getTokenTypesViaATN(inputString, lexInterp);
@ -508,7 +508,7 @@ public class TestATNParserPrediction extends BaseTest {
{ {
// sync to ensure multiple tests don't race on dfa access // sync to ensure multiple tests don't race on dfa access
Tool.internalOption_ShowATNConfigsInDFA = true; Tool.internalOption_ShowATNConfigsInDFA = true;
ATN lexatn = createATN(lg); ATN lexatn = createATN(lg, true);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null,null); LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn,null,null);
semanticProcess(lg); semanticProcess(lg);
@ -545,7 +545,7 @@ public class TestATNParserPrediction extends BaseTest {
String[] inputString, String[] dfaString) String[] inputString, String[] dfaString)
{ {
// Tool.internalOption_ShowATNConfigsInDFA = true; // Tool.internalOption_ShowATNConfigsInDFA = true;
ATN lexatn = createATN(lg); ATN lexatn = createATN(lg, true);
LexerATNSimulator lexInterp = LexerATNSimulator lexInterp =
new LexerATNSimulator(lexatn,new DFA[1], new PredictionContextCache()); new LexerATNSimulator(lexatn,new DFA[1], new PredictionContextCache());

44
tool/test/org/antlr/v4/test/TestATNSerialization.java
View File

@ -54,7 +54,7 @@ public class TestATNSerialization extends BaseTest {
"2->3 ATOM 1,0,0\n" + "2->3 ATOM 1,0,0\n" +
"3->4 ATOM 2,0,0\n" + "3->4 ATOM 2,0,0\n" +
"4->1 EPSILON 0,0,0\n"; "4->1 EPSILON 0,0,0\n";
ATN atn = createATN(g); ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn); String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -76,7 +76,7 @@ public class TestATNSerialization extends BaseTest {
"2->3 ATOM 1,0,0\n" + "2->3 ATOM 1,0,0\n" +
"3->4 ATOM -1,0,0\n" + "3->4 ATOM -1,0,0\n" +
"4->1 EPSILON 0,0,0\n"; "4->1 EPSILON 0,0,0\n";
ATN atn = createATN(g); ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn); String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -97,7 +97,7 @@ public class TestATNSerialization extends BaseTest {
"0->2 EPSILON 0,0,0\n" + "0->2 EPSILON 0,0,0\n" +
"2->3 SET 0,0,0\n" + "2->3 SET 0,0,0\n" +
"3->1 EPSILON 0,0,0\n"; "3->1 EPSILON 0,0,0\n";
ATN atn = createATN(g); ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn); String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -119,7 +119,7 @@ public class TestATNSerialization extends BaseTest {
"0->2 EPSILON 0,0,0\n" + "0->2 EPSILON 0,0,0\n" +
"2->3 NOT_SET 0,0,0\n" + "2->3 NOT_SET 0,0,0\n" +
"3->1 EPSILON 0,0,0\n"; "3->1 EPSILON 0,0,0\n";
ATN atn = createATN(g); ATN atn = createATN(g, true);
DOTGenerator gen = new DOTGenerator(g); DOTGenerator gen = new DOTGenerator(g);
System.out.println(gen.getDOT(atn.ruleToStartState[0])); System.out.println(gen.getDOT(atn.ruleToStartState[0]));
String result = ATNSerializer.getDecoded(g, atn); String result = ATNSerializer.getDecoded(g, atn);
@ -142,7 +142,7 @@ public class TestATNSerialization extends BaseTest {
"0->2 EPSILON 0,0,0\n" + "0->2 EPSILON 0,0,0\n" +
"2->3 WILDCARD 0,0,0\n" + "2->3 WILDCARD 0,0,0\n" +
"3->1 EPSILON 0,0,0\n"; "3->1 EPSILON 0,0,0\n";
ATN atn = createATN(g); ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn); String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -170,7 +170,7 @@ public class TestATNSerialization extends BaseTest {
"5->3 EPSILON 0,0,0\n" + "5->3 EPSILON 0,0,0\n" +
"6->1 EPSILON 0,0,0\n" + "6->1 EPSILON 0,0,0\n" +
"0:5\n"; "0:5\n";
ATN atn = createATN(g); ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn); String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -205,7 +205,7 @@ public class TestATNSerialization extends BaseTest {
"8->5 EPSILON 0,0,0\n" + "8->5 EPSILON 0,0,0\n" +
"9->1 EPSILON 0,0,0\n" + "9->1 EPSILON 0,0,0\n" +
"0:8\n"; "0:8\n";
ATN atn = createATN(g); ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn); String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -237,7 +237,7 @@ public class TestATNSerialization extends BaseTest {
"7->8 ATOM 2,0,0\n" + "7->8 ATOM 2,0,0\n" +
"8->1 EPSILON 0,0,0\n" + "8->1 EPSILON 0,0,0\n" +
"0:5\n"; "0:5\n";
ATN atn = createATN(g); ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn); String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -266,7 +266,7 @@ public class TestATNSerialization extends BaseTest {
"5->1 EPSILON 0,0,0\n" + "5->1 EPSILON 0,0,0\n" +
"6->7 ATOM 1,0,0\n" + "6->7 ATOM 1,0,0\n" +
"7->3 EPSILON 0,0,0\n"; "7->3 EPSILON 0,0,0\n";
ATN atn = createATN(g); ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn); String result = ATNSerializer.getDecoded(g, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -299,7 +299,7 @@ public class TestATNSerialization extends BaseTest {
"7->8 ATOM 98,0,0\n" + "7->8 ATOM 98,0,0\n" +
"8->4 EPSILON 0,0,0\n" + "8->4 EPSILON 0,0,0\n" +
"0:0\n"; "0:0\n";
ATN atn = createATN(lg); ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn); String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -322,7 +322,7 @@ public class TestATNSerialization extends BaseTest {
"3->4 RANGE 48,57,0\n" + "3->4 RANGE 48,57,0\n" +
"4->2 EPSILON 0,0,0\n" + "4->2 EPSILON 0,0,0\n" +
"0:0\n"; "0:0\n";
ATN atn = createATN(lg); ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn); String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -347,7 +347,7 @@ public class TestATNSerialization extends BaseTest {
"4->5 ATOM -1,0,0\n" + "4->5 ATOM -1,0,0\n" +
"5->2 EPSILON 0,0,0\n" + "5->2 EPSILON 0,0,0\n" +
"0:0\n"; "0:0\n";
ATN atn = createATN(lg); ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn); String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -376,7 +376,7 @@ public class TestATNSerialization extends BaseTest {
"6->2 EPSILON 0,0,0\n" + "6->2 EPSILON 0,0,0\n" +
"0:0\n" + "0:0\n" +
"1:5\n"; "1:5\n";
ATN atn = createATN(lg); ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn); String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -407,7 +407,7 @@ public class TestATNSerialization extends BaseTest {
"7->2 EPSILON 0,0,0\n" + "7->2 EPSILON 0,0,0\n" +
"0:0\n" + "0:0\n" +
"1:6\n"; "1:6\n";
ATN atn = createATN(lg); ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn); String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -454,7 +454,7 @@ public class TestATNSerialization extends BaseTest {
"13->14 ACTION 2,1,0\n" + "13->14 ACTION 2,1,0\n" +
"14->6 EPSILON 0,0,0\n" + "14->6 EPSILON 0,0,0\n" +
"0:0\n"; "0:0\n";
ATN atn = createATN(lg); ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn); String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -478,7 +478,7 @@ public class TestATNSerialization extends BaseTest {
"3->4 NOT_SET 0,0,0\n" + "3->4 NOT_SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" + "4->2 EPSILON 0,0,0\n" +
"0:0\n"; "0:0\n";
ATN atn = createATN(lg); ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn); String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -502,7 +502,7 @@ public class TestATNSerialization extends BaseTest {
"3->4 SET 0,0,0\n" + "3->4 SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" + "4->2 EPSILON 0,0,0\n" +
"0:0\n"; "0:0\n";
ATN atn = createATN(lg); ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn); String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -526,7 +526,7 @@ public class TestATNSerialization extends BaseTest {
"3->4 NOT_SET 0,0,0\n" + "3->4 NOT_SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" + "4->2 EPSILON 0,0,0\n" +
"0:0\n"; "0:0\n";
ATN atn = createATN(lg); ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn); String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -589,7 +589,7 @@ public class TestATNSerialization extends BaseTest {
"0:0\n" + "0:0\n" +
"1:1\n" + "1:1\n" +
"2:11\n"; "2:11\n";
ATN atn = createATN(lg); ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn); String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -616,7 +616,7 @@ public class TestATNSerialization extends BaseTest {
"4->5 NOT_SET 1,0,0\n" + "4->5 NOT_SET 1,0,0\n" +
"5->2 EPSILON 0,0,0\n" + "5->2 EPSILON 0,0,0\n" +
"0:0\n"; "0:0\n";
ATN atn = createATN(lg); ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn); String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -673,7 +673,7 @@ public class TestATNSerialization extends BaseTest {
"17->9 EPSILON 0,0,0\n" + "17->9 EPSILON 0,0,0\n" +
"0:0\n" + "0:0\n" +
"1:1\n"; "1:1\n";
ATN atn = createATN(lg); ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn); String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }
@ -724,7 +724,7 @@ public class TestATNSerialization extends BaseTest {
"0:0\n" + "0:0\n" +
"1:1\n" + "1:1\n" +
"2:2\n"; "2:2\n";
ATN atn = createATN(lg); ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn); String result = ATNSerializer.getDecoded(lg, atn);
assertEquals(expecting, result); assertEquals(expecting, result);
} }