jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Move ATNSerializer to runtime

This commit is contained in:
Sam Harwell 2013-11-21 13:10:48 -06:00
parent bc59f30857
commit fb1880d82c
8 changed files with 109 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
tool/src/org/antlr/v4/automata/ATNSerializer.java → runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java
View File

@ -28,32 +28,13 @@
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.automata;
package org.antlr.v4.runtime.atn;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNSimulator;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.ATNType;
import org.antlr.v4.runtime.atn.ActionTransition;
import org.antlr.v4.runtime.atn.AtomTransition;
import org.antlr.v4.runtime.atn.BlockStartState;
import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.LoopEndState;
import org.antlr.v4.runtime.atn.PrecedencePredicateTransition;
import org.antlr.v4.runtime.atn.PredicateTransition;
import org.antlr.v4.runtime.atn.RangeTransition;
import org.antlr.v4.runtime.atn.RuleStartState;
import org.antlr.v4.runtime.atn.RuleTransition;
import org.antlr.v4.runtime.atn.SetTransition;
import org.antlr.v4.runtime.atn.Transition;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.runtime.misc.Utils;
import java.io.InvalidClassException;
import java.util.ArrayList;
@ -64,14 +45,20 @@ import java.util.Map;
import java.util.UUID;
public class ATNSerializer {
public Grammar g;
public ATN atn;
private List<String> tokenNames;
public ATNSerializer(Grammar g, ATN atn) {
this.g = g;
public ATNSerializer(ATN atn) {
assert atn.grammarType != null;
this.atn = atn;
}
public ATNSerializer(ATN atn, List<String> tokenNames) {
assert atn.grammarType != null;
this.atn = atn;
this.tokenNames = tokenNames;
}
/** Serialize state descriptors, edge descriptors, and decision->state map
* into list of ints:
*
@ -99,21 +86,8 @@ public class ATNSerializer {
serializeUUID(data, ATNSimulator.SERIALIZED_UUID);
// convert grammar type to ATN const to avoid dependence on ANTLRParser
switch (g.getType()) {
case ANTLRParser.LEXER:
data.add(ATNType.LEXER.ordinal());
break;
case ANTLRParser.PARSER:
case ANTLRParser.COMBINED:
data.add(ATNType.PARSER.ordinal());
break;
default:
throw new UnsupportedOperationException("Invalid grammar type.");
}
data.add(g.getMaxTokenType());
data.add(atn.grammarType.ordinal());
data.add(atn.maxTokenType);
int nedges = 0;
Map<IntervalSet, Integer> setIndices = new HashMap<IntervalSet, Integer>();
@ -189,20 +163,19 @@ public class ATNSerializer {
for (int r=0; r<nrules; r++) {
ATNState ruleStartState = atn.ruleToStartState[r];
data.add(ruleStartState.stateNumber);
if ( g.isLexer() ) {
if (atn.grammarType == ATNType.LEXER) {
if (atn.ruleToTokenType[r] == Token.EOF) {
data.add(Character.MAX_VALUE);
}
else {
data.add(atn.ruleToTokenType[r]);
}
String ruleName = g.rules.getKey(r);
Rule rule = g.getRule(ruleName);
if (rule.actionIndex == -1) {
if (atn.ruleToActionIndex[r] == -1) {
data.add(Character.MAX_VALUE);
}
else {
data.add(rule.actionIndex);
data.add(atn.ruleToActionIndex[r]);
}
}
}
@ -407,7 +380,7 @@ public class ATNSerializer {
int nrules = ATNSimulator.toInt(data[p++]);
for (int i=0; i<nrules; i++) {
int s = ATNSimulator.toInt(data[p++]);
if ( g.isLexer() ) {
if (atn.grammarType == ATNType.LEXER) {
int arg1 = ATNSimulator.toInt(data[p++]);
int arg2 = ATNSimulator.toInt(data[p++]);
if (arg2 == Character.MAX_VALUE) {
@ -467,27 +440,62 @@ public class ATNSerializer {
public String getTokenName(int t) {
if ( t==-1 ) return "EOF";
if ( g!=null ) return g.getTokenDisplayName(t);
if ( atn.grammarType == ATNType.LEXER &&
t >= Character.MIN_VALUE && t <= Character.MAX_VALUE )
{
switch (t) {
case '\n':
return "'\\n'";
case '\r':
return "'\\r'";
case '\t':
return "'\\t'";
case '\b':
return "'\\b'";
case '\f':
return "'\\f'";
case '\\':
return "'\\\\'";
case '\'':
return "'\\''";
default:
if ( Character.UnicodeBlock.of((char)t)==Character.UnicodeBlock.BASIC_LATIN &&
!Character.isISOControl((char)t) ) {
return '\''+Character.toString((char)t)+'\'';
}
// turn on the bit above max "\uFFFF" value so that we pad with zeros
// then only take last 4 digits
String hex = Integer.toHexString(t|0x10000).toUpperCase().substring(1,5);
String unicodeStr = "'\\u"+hex+"'";
return unicodeStr;
}
}
if (tokenNames != null && t >= 0 && t < tokenNames.size()) {
return tokenNames.get(t);
}
return String.valueOf(t);
}
/** Used by Java target to encode short/int array as chars in string. */
public static String getSerializedAsString(Grammar g, ATN atn) {
return new String(getSerializedAsChars(g, atn));
public static String getSerializedAsString(ATN atn) {
return new String(getSerializedAsChars(atn));
}
public static IntegerList getSerialized(Grammar g, ATN atn) {
return new ATNSerializer(g, atn).serialize();
public static IntegerList getSerialized(ATN atn) {
return new ATNSerializer(atn).serialize();
}
public static char[] getSerializedAsChars(Grammar g, ATN atn) {
return Utils.toCharArray(getSerialized(g, atn));
public static char[] getSerializedAsChars(ATN atn) {
return Utils.toCharArray(getSerialized(atn));
}
public static String getDecoded(Grammar g, ATN atn) {
IntegerList serialized = getSerialized(g, atn);
public static String getDecoded(ATN atn, List<String> tokenNames) {
IntegerList serialized = getSerialized(atn);
char[] data = Utils.toCharArray(serialized);
return new ATNSerializer(g, atn).decode(data);
return new ATNSerializer(atn, tokenNames).decode(data);
}
private void serializeUUID(IntegerList data, UUID uuid) {

9
runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java
View File

@ -146,4 +146,13 @@ public class Utils {
}
return m;
}
public static char[] toCharArray(IntegerList data) {
if ( data==null ) return null;
char[] cdata = new char[data.size()];
for (int i=0; i<data.size(); i++) {
cdata[i] = (char)data.get(i);
}
return cdata;
}
}

4
tool/src/org/antlr/v4/codegen/model/SerializedATN.java
View File

@ -30,9 +30,9 @@
package org.antlr.v4.codegen.model;
import org.antlr.v4.automata.ATNSerializer;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNSerializer;
import org.antlr.v4.runtime.misc.IntegerList;
import java.util.ArrayList;
@ -43,7 +43,7 @@ public class SerializedATN extends OutputModelObject {
public List<String> serialized;
public SerializedATN(OutputModelFactory factory, ATN atn) {
super(factory);
IntegerList data = ATNSerializer.getSerialized(factory.getGrammar(), atn);
IntegerList data = ATNSerializer.getSerialized(atn);
serialized = new ArrayList<String>(data.size());
for (int c : data.toArray()) {
String encoded = factory.getGenerator().getTarget().encodeIntAsCharEscape(c == -1 ? Character.MAX_VALUE : c);

9
tool/src/org/antlr/v4/misc/Utils.java
View File

@ -124,15 +124,6 @@ public class Utils {
return Character.toLowerCase(s.charAt(0)) + s.substring(1);
}
public static char[] toCharArray(IntegerList data) {
if ( data==null ) return null;
char[] cdata = new char[data.size()];
for (int i=0; i<data.size(); i++) {
cdata[i] = (char)data.get(i);
}
return cdata;
}
/** apply methodName to list and return list of results. method has
* no args. This pulls data out of a list essentially.
*/

4
tool/test/org/antlr/v4/test/BaseTest.java
View File

@ -32,7 +32,6 @@ package org.antlr.v4.test;
import org.antlr.v4.Tool;
import org.antlr.v4.automata.ATNFactory;
import org.antlr.v4.automata.ATNPrinter;
import org.antlr.v4.automata.ATNSerializer;
import org.antlr.v4.automata.LexerATNFactory;
import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.codegen.CodeGenerator;
@ -50,6 +49,7 @@ import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.WritableToken;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNDeserializer;
import org.antlr.v4.runtime.atn.ATNSerializer;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
@ -184,7 +184,7 @@ public abstract class BaseTest {
ATN atn = g.atn;
if (useSerializer) {
char[] serialized = ATNSerializer.getSerializedAsChars(g, atn);
char[] serialized = ATNSerializer.getSerializedAsChars(atn);
return new ATNDeserializer().deserialize(serialized);
}

12
tool/test/org/antlr/v4/test/TestATNDeserialization.java
View File

@ -30,16 +30,18 @@
package org.antlr.v4.test;
import org.antlr.v4.automata.ATNSerializer;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNDeserializer;
import org.antlr.v4.runtime.atn.ATNSerializer;
import org.antlr.v4.runtime.misc.Utils;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar;
import org.junit.Test;
import static org.junit.Assert.*;
import java.util.Arrays;
public class TestATNDeserialization extends BaseTest {
@Test public void testSimpleNoBlock() throws Exception {
Grammar g = new Grammar(
@ -177,10 +179,10 @@ public class TestATNDeserialization extends BaseTest {
protected void checkDeserializationIsStable(Grammar g) {
ATN atn = createATN(g, false);
char[] data = Utils.toCharArray(ATNSerializer.getSerialized(g, atn));
String atnData = ATNSerializer.getDecoded(g, atn);
char[] data = Utils.toCharArray(ATNSerializer.getSerialized(atn));
String atnData = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames()));
ATN atn2 = new ATNDeserializer().deserialize(data);
String atn2Data = ATNSerializer.getDecoded(g, atn2);
String atn2Data = ATNSerializer.getDecoded(atn2, Arrays.asList(g.getTokenNames()));
assertEquals(atnData, atn2Data);
}

48
tool/test/org/antlr/v4/test/TestATNSerialization.java
View File

@ -30,13 +30,15 @@
package org.antlr.v4.test;
import org.antlr.v4.automata.ATNSerializer;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNSerializer;
import org.antlr.v4.tool.DOTGenerator;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar;
import org.junit.Test;
import java.util.Arrays;
import static org.junit.Assert.*;
public class TestATNSerialization extends BaseTest {
@ -58,7 +60,7 @@ public class TestATNSerialization extends BaseTest {
"3->4 ATOM 2,0,0\n" +
"4->1 EPSILON 0,0,0\n";
ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames()));
assertEquals(expecting, result);
}
@ -80,7 +82,7 @@ public class TestATNSerialization extends BaseTest {
"3->4 ATOM 0,0,1\n" +
"4->1 EPSILON 0,0,0\n";
ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames()));
assertEquals(expecting, result);
}
@ -101,7 +103,7 @@ public class TestATNSerialization extends BaseTest {
"2->3 SET 0,0,0\n" +
"3->1 EPSILON 0,0,0\n";
ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames()));
assertEquals(expecting, result);
}
@ -125,7 +127,7 @@ public class TestATNSerialization extends BaseTest {
ATN atn = createATN(g, true);
DOTGenerator gen = new DOTGenerator(g);
System.out.println(gen.getDOT(atn.ruleToStartState[0]));
String result = ATNSerializer.getDecoded(g, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames()));
assertEquals(expecting, result);
}
@ -146,7 +148,7 @@ public class TestATNSerialization extends BaseTest {
"2->3 WILDCARD 0,0,0\n" +
"3->1 EPSILON 0,0,0\n";
ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames()));
assertEquals(expecting, result);
}
@ -174,7 +176,7 @@ public class TestATNSerialization extends BaseTest {
"6->1 EPSILON 0,0,0\n" +
"0:5\n";
ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames()));
assertEquals(expecting, result);
}
@ -209,7 +211,7 @@ public class TestATNSerialization extends BaseTest {
"9->1 EPSILON 0,0,0\n" +
"0:8\n";
ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames()));
assertEquals(expecting, result);
}
@ -241,7 +243,7 @@ public class TestATNSerialization extends BaseTest {
"8->1 EPSILON 0,0,0\n" +
"0:5\n";
ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames()));
assertEquals(expecting, result);
}
@ -270,7 +272,7 @@ public class TestATNSerialization extends BaseTest {
"6->7 ATOM 1,0,0\n" +
"7->3 EPSILON 0,0,0\n";
ATN atn = createATN(g, true);
String result = ATNSerializer.getDecoded(g, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames()));
assertEquals(expecting, result);
}
@ -303,7 +305,7 @@ public class TestATNSerialization extends BaseTest {
"8->4 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@ -326,7 +328,7 @@ public class TestATNSerialization extends BaseTest {
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@ -351,7 +353,7 @@ public class TestATNSerialization extends BaseTest {
"5->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@ -380,7 +382,7 @@ public class TestATNSerialization extends BaseTest {
"0:0\n" +
"1:5\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@ -411,7 +413,7 @@ public class TestATNSerialization extends BaseTest {
"0:0\n" +
"1:6\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@ -458,7 +460,7 @@ public class TestATNSerialization extends BaseTest {
"14->6 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@ -482,7 +484,7 @@ public class TestATNSerialization extends BaseTest {
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@ -506,7 +508,7 @@ public class TestATNSerialization extends BaseTest {
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@ -530,7 +532,7 @@ public class TestATNSerialization extends BaseTest {
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@ -593,7 +595,7 @@ public class TestATNSerialization extends BaseTest {
"1:1\n" +
"2:11\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@ -620,7 +622,7 @@ public class TestATNSerialization extends BaseTest {
"5->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@ -677,7 +679,7 @@ public class TestATNSerialization extends BaseTest {
"0:0\n" +
"1:1\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@ -728,7 +730,7 @@ public class TestATNSerialization extends BaseTest {
"1:1\n" +
"2:2\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(lg, atn);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}

2
tool/test/org/antlr/v4/test/TestParseErrors.java
View File

@ -30,7 +30,7 @@
package org.antlr.v4.test;
import org.antlr.v4.automata.ATNSerializer;
import org.antlr.v4.runtime.atn.ATNSerializer;
import org.junit.Test;
import static org.junit.Assert.*;