Use IntegerList in the tool

This commit is contained in:
Sam Harwell 2012-07-30 14:58:28 -05:00
parent dcdcad8feb
commit 7d4f71d829
8 changed files with 30 additions and 35 deletions

View File

@ -32,6 +32,7 @@ package org.antlr.v4.automata;
import org.antlr.v4.misc.Utils; import org.antlr.v4.misc.Utils;
import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.IntervalSet; import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.Grammar;
@ -71,8 +72,8 @@ public class ATNSerializer {
* *
* Convenient to pack into unsigned shorts to make as Java string. * Convenient to pack into unsigned shorts to make as Java string.
*/ */
public List<Integer> serialize() { public IntegerList serialize() {
List<Integer> data = new ArrayList<Integer>(); IntegerList data = new IntegerList();
// convert grammar type to ATN const to avoid dependence on ANTLRParser // convert grammar type to ATN const to avoid dependence on ANTLRParser
if ( g.getType()== ANTLRParser.LEXER ) data.add(ATN.LEXER); if ( g.getType()== ANTLRParser.LEXER ) data.add(ATN.LEXER);
else if ( g.getType()== ANTLRParser.PARSER ) data.add(ATN.PARSER); else if ( g.getType()== ANTLRParser.PARSER ) data.add(ATN.PARSER);
@ -277,7 +278,7 @@ public class ATNSerializer {
return new String(Utils.toCharArray(getSerialized(g, atn))); return new String(Utils.toCharArray(getSerialized(g, atn)));
} }
public static List<Integer> getSerialized(Grammar g, ATN atn) { public static IntegerList getSerialized(Grammar g, ATN atn) {
return new ATNSerializer(g, atn).serialize(); return new ATNSerializer(g, atn).serialize();
} }
@ -286,7 +287,7 @@ public class ATNSerializer {
} }
public static String getDecoded(Grammar g, ATN atn) { public static String getDecoded(Grammar g, ATN atn) {
List<Integer> serialized = getSerialized(g, atn); IntegerList serialized = getSerialized(g, atn);
char[] data = Utils.toCharArray(serialized); char[] data = Utils.toCharArray(serialized);
return new ATNSerializer(g, atn).decode(data); return new ATNSerializer(g, atn).decode(data);
} }

View File

@ -32,6 +32,7 @@ package org.antlr.v4.codegen.model;
import org.antlr.v4.automata.ATNSerializer; import org.antlr.v4.automata.ATNSerializer;
import org.antlr.v4.codegen.OutputModelFactory; import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.misc.IntegerList;
import java.util.*; import java.util.*;
@ -40,9 +41,9 @@ public class SerializedATN extends OutputModelObject {
public List<String> serialized; public List<String> serialized;
public SerializedATN(OutputModelFactory factory, ATN atn) { public SerializedATN(OutputModelFactory factory, ATN atn) {
super(factory); super(factory);
List<Integer> data = ATNSerializer.getSerialized(factory.getGrammar(), atn); IntegerList data = ATNSerializer.getSerialized(factory.getGrammar(), atn);
serialized = new ArrayList<String>(data.size()); serialized = new ArrayList<String>(data.size());
for (int c : data) { for (int c : data.toArray()) {
String encoded = factory.getGenerator().target.encodeIntAsCharEscape(c == -1 ? Character.MAX_VALUE : c); String encoded = factory.getGenerator().target.encodeIntAsCharEscape(c == -1 ? Character.MAX_VALUE : c);
serialized.add(encoded); serialized.add(encoded);
} }

View File

@ -29,6 +29,7 @@
package org.antlr.v4.misc; package org.antlr.v4.misc;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.tool.ast.GrammarAST; import org.antlr.v4.tool.ast.GrammarAST;
import java.util.*; import java.util.*;
@ -137,13 +138,6 @@ public class Utils {
// return x; // return x;
// } // }
public static int[] toIntArray(List<Integer> list) {
if ( list==null ) return null;
int[] a = new int[list.size()];
for (int i=0; i<list.size(); i++) a[i] = list.get(i);
return a;
}
public static String capitalize(String s) { public static String capitalize(String s) {
return Character.toUpperCase(s.charAt(0)) + s.substring(1); return Character.toUpperCase(s.charAt(0)) + s.substring(1);
} }
@ -152,11 +146,11 @@ public class Utils {
return Character.toLowerCase(s.charAt(0)) + s.substring(1); return Character.toLowerCase(s.charAt(0)) + s.substring(1);
} }
public static char[] toCharArray(List<Integer> data) { public static char[] toCharArray(IntegerList data) {
if ( data==null ) return null; if ( data==null ) return null;
char[] cdata = new char[data.size()]; char[] cdata = new char[data.size()];
for (int i=0; i<data.size(); i++) { for (int i=0; i<data.size(); i++) {
cdata[i] = (char)(int)data.get(i); cdata[i] = (char)data.get(i);
} }
return cdata; return cdata;
} }

View File

@ -33,6 +33,7 @@ import org.antlr.v4.misc.Utils;
import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.dfa.DFAState; import org.antlr.v4.runtime.dfa.DFAState;
import org.antlr.v4.runtime.misc.IntegerList;
import org.stringtemplate.v4.ST; import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupDir; import org.stringtemplate.v4.STGroupDir;
@ -120,9 +121,9 @@ public class DOTGenerator {
if ( alts!=null ) { if ( alts!=null ) {
buf.append("\\n"); buf.append("\\n");
// separate alts // separate alts
List<Integer> altList = new ArrayList<Integer>(); IntegerList altList = new IntegerList();
altList.addAll(alts); altList.addAll(alts);
Collections.sort(altList); altList.sort();
Set<ATNConfig> configurations = s.configset; Set<ATNConfig> configurations = s.configset;
for (int altIndex = 0; altIndex < altList.size(); altIndex++) { for (int altIndex = 0; altIndex < altList.size(); altIndex++) {
int alt = altList.get(altIndex); int alt = altList.get(altIndex);

View File

@ -46,6 +46,7 @@ import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntSet; import org.antlr.v4.runtime.misc.IntSet;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.runtime.misc.IntervalSet; import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.misc.Nullable; import org.antlr.v4.runtime.misc.Nullable;
@ -58,7 +59,6 @@ import org.antlr.v4.tool.ast.TerminalAST;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
@ -466,9 +466,9 @@ public class Grammar implements AttributeResolver {
return tokenName; return tokenName;
} }
public List<String> getTokenDisplayNames(Collection<Integer> types) { public List<String> getTokenDisplayNames(IntegerList types) {
List<String> names = new ArrayList<String>(); List<String> names = new ArrayList<String>();
for (int t : types) names.add(getTokenDisplayName(t)); for (int t : types.toArray()) names.add(getTokenDisplayName(t));
return names; return names;
} }

View File

@ -50,6 +50,7 @@ import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.DecisionState; import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.LexerATNSimulator; import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.Nullable; import org.antlr.v4.runtime.misc.Nullable;
import org.antlr.v4.semantics.SemanticPipeline; import org.antlr.v4.semantics.SemanticPipeline;
@ -184,8 +185,8 @@ public abstract class BaseTest {
// dfa.minimized = dmin.minimize(); // dfa.minimized = dmin.minimize();
// } // }
List<Integer> getTypesFromString(Grammar g, String expecting) { IntegerList getTypesFromString(Grammar g, String expecting) {
List<Integer> expectingTokenTypes = new ArrayList<Integer>(); IntegerList expectingTokenTypes = new IntegerList();
if ( expecting!=null && !expecting.trim().isEmpty() ) { if ( expecting!=null && !expecting.trim().isEmpty() ) {
for (String tname : expecting.replace(" ", "").split(",")) { for (String tname : expecting.replace(" ", "").split(",")) {
int ttype = g.getTokenType(tname); int ttype = g.getTokenType(tname);
@ -195,9 +196,9 @@ public abstract class BaseTest {
return expectingTokenTypes; return expectingTokenTypes;
} }
public List<Integer> getTokenTypesViaATN(String input, LexerATNSimulator lexerATN) { public IntegerList getTokenTypesViaATN(String input, LexerATNSimulator lexerATN) {
ANTLRInputStream in = new ANTLRInputStream(input); ANTLRInputStream in = new ANTLRInputStream(input);
List<Integer> tokenTypes = new ArrayList<Integer>(); IntegerList tokenTypes = new IntegerList();
int ttype; int ttype;
do { do {
ttype = lexerATN.matchATN(in); ttype = lexerATN.matchATN(in);
@ -1065,9 +1066,9 @@ public abstract class BaseTest {
public void assertNull(Object object) { try {Assert.assertNull(object);} catch (Error e) {lastTestFailed=true; throw e;} } public void assertNull(Object object) { try {Assert.assertNull(object);} catch (Error e) {lastTestFailed=true; throw e;} }
public static class IntTokenStream implements TokenStream { public static class IntTokenStream implements TokenStream {
List<Integer> types; IntegerList types;
int p=0; int p=0;
public IntTokenStream(List<Integer> types) { this.types = types; } public IntTokenStream(IntegerList types) { this.types = types; }
@Override @Override
public void consume() { p++; } public void consume() { p++; }

View File

@ -7,6 +7,7 @@ import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.BlockStartState; import org.antlr.v4.runtime.atn.BlockStartState;
import org.antlr.v4.runtime.atn.LexerATNSimulator; import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.tool.DOTGenerator; import org.antlr.v4.tool.DOTGenerator;
import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar; import org.antlr.v4.tool.LexerGrammar;
@ -14,9 +15,6 @@ import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.interp.ParserInterpreter; import org.antlr.v4.tool.interp.ParserInterpreter;
import org.junit.Test; import org.junit.Test;
import java.util.List;
// NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH // NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH
// NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH // NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH
// NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH // NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH
@ -275,7 +273,7 @@ public class TestATNInterpreter extends BaseTest {
{ {
ATN lexatn = createATN(lg); ATN lexatn = createATN(lg);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn); LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn);
List<Integer> types = getTokenTypesViaATN(inputString, lexInterp); IntegerList types = getTokenTypesViaATN(inputString, lexInterp);
System.out.println(types); System.out.println(types);
semanticProcess(lg); semanticProcess(lg);

View File

@ -37,6 +37,7 @@ import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream; import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.tool.DOTGenerator; import org.antlr.v4.tool.DOTGenerator;
import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar; import org.antlr.v4.tool.LexerGrammar;
@ -44,8 +45,6 @@ import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.interp.ParserInterpreter; import org.antlr.v4.tool.interp.ParserInterpreter;
import org.junit.Test; import org.junit.Test;
import java.util.List;
// NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH // NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH
// NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH // NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH
// NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH // NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH
@ -477,7 +476,7 @@ public class TestATNParserPrediction extends BaseTest {
Tool.internalOption_ShowATNConfigsInDFA = true; Tool.internalOption_ShowATNConfigsInDFA = true;
ATN lexatn = createATN(lg); ATN lexatn = createATN(lg);
LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn); LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn);
List<Integer> types = getTokenTypesViaATN(inputString, lexInterp); IntegerList types = getTokenTypesViaATN(inputString, lexInterp);
System.out.println(types); System.out.println(types);
semanticProcess(lg); semanticProcess(lg);
@ -542,7 +541,7 @@ public class TestATNParserPrediction extends BaseTest {
// System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e")))); // System.out.println(dot.getDOT(atn.ruleToStartState.get(g.getRule("e"))));
ParserATNSimulator<Token> interp = new ParserATNSimulator<Token>(atn); ParserATNSimulator<Token> interp = new ParserATNSimulator<Token>(atn);
List<Integer> types = getTokenTypesViaATN(inputString, lexInterp); IntegerList types = getTokenTypesViaATN(inputString, lexInterp);
System.out.println(types); System.out.println(types);
TokenStream input = new IntTokenStream(types); TokenStream input = new IntTokenStream(types);
try { try {
@ -572,7 +571,7 @@ public class TestATNParserPrediction extends BaseTest {
ParserInterpreter interp = new ParserInterpreter(g, null); ParserInterpreter interp = new ParserInterpreter(g, null);
for (int i=0; i<inputString.length; i++) { for (int i=0; i<inputString.length; i++) {
// Check DFA // Check DFA
List<Integer> types = getTokenTypesViaATN(inputString[i], lexInterp); IntegerList types = getTokenTypesViaATN(inputString[i], lexInterp);
System.out.println(types); System.out.println(types);
TokenStream input = new IntTokenStream(types); TokenStream input = new IntTokenStream(types);
try { try {