forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Merge pull request #1633 from bhamiltoncx/unicode 

New extended Unicode escape \u{10ABCD} to support Unicode literals > U+FFFF
This commit is contained in:
Terence Parr 2017-02-23 14:06:44 -08:00 committed by GitHub
parent ce09abb480 4589ff56ea
commit 4ec294faff
25 changed files with 1402 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

278
runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/SetsDescriptors.java
View File

@ -402,4 +402,282 @@ public class SetsDescriptors {
public String grammar;
}
public static class UnicodeUnescapedBMPSet extends BaseParserTestDescriptor {
public String input = "a\u00E4\u3042\u4E9Cc";
public String output = "a\u00E4\u3042\u4E9Cc\n";
public String errors = null;
public String startRule = "a";
public String grammarName = "T";
/**
grammar T;
a : LETTERS {<InputText():writeln()>} ;
// These are actually not escaped -- Java passes the
// raw unescaped Unicode values to the grammar compiler.
LETTERS : ('a'|'\u00E4'|'\u4E9C'|'\u3042')* 'c';
*/
@CommentHasStringValue
public String grammar;
}
public static class UnicodeUnescapedBMPRangeSet extends BaseParserTestDescriptor {
public String input = "a\u00E1\u00E4\u00E1\u00E2\u00E5d";
public String output = "a\u00E1\u00E4\u00E1\u00E2\u00E5d\n";
public String errors = null;
public String startRule = "a";
public String grammarName = "T";
/**
grammar T;
a : LETTERS* 'd' {<InputText():writeln()>} ;
// These are actually not escaped -- Java passes the
// raw unescaped Unicode values to the grammar compiler.
LETTERS : ('a'|'\u00E0'..'\u00E5');
*/
@CommentHasStringValue
public String grammar;
}
public static class UnicodeEscapedBMPSet extends BaseParserTestDescriptor {
public String input = "a\u00E4\u3042\u4E9Cc";
public String output = "a\u00E4\u3042\u4E9Cc\n";
public String errors = null;
public String startRule = "a";
public String grammarName = "T";
/**
grammar T;
a : LETTERS {<InputText():writeln()>} ;
// Note the double-backslash to avoid Java passing
// unescaped values as part of the grammar.
LETTERS : ('a'|'\\u00E4'|'\\u4E9C'|'\\u3042')* 'c';
*/
@CommentHasStringValue
public String grammar;
}
public static class UnicodeEscapedBMPRangeSet extends BaseParserTestDescriptor {
public String input = "a\u00E1\u00E4\u00E1\u00E2\u00E5d";
public String output = "a\u00E1\u00E4\u00E1\u00E2\u00E5d\n";
public String errors = null;
public String startRule = "a";
public String grammarName = "T";
/**
grammar T;
a : LETTERS* 'd' {<InputText():writeln()>} ;
// Note the double-backslash to avoid Java passing
// unescaped values as part of the grammar.
LETTERS : ('a'|'\\u00E0'..'\\u00E5');
*/
@CommentHasStringValue
public String grammar;
}
// TODO(bhamiltoncx): This needs to be an error, the V3
// runtime used by the tool doesn't really understand unescaped code points >
// U+FFFF.
// public static class UnicodeUnescapedSMPSet extends BaseParserTestDescriptor {
// public String input = new StringBuilder()
// .append("a")
// .appendCodePoint(0x1D5C2)
// .appendCodePoint(0x1D5CE)
// .appendCodePoint(0x1D5BA)
// .append("c")
// .toString();
// public String output = new StringBuilder()
// .append("a")
// .appendCodePoint(0x1D5C2)
// .appendCodePoint(0x1D5CE)
// .appendCodePoint(0x1D5BA)
// .append("c\n")
// .toString();
// public String errors = null;
// public String startRule = "a";
// public String grammarName = "T";
// /**
// grammar T;
// a : LETTERS {<InputText():writeln()>} ;
// // These are actually not escaped -- Java passes the
// // raw unescaped Unicode values to the grammar compiler.
// //
// // Each sequence is the UTF-16 encoding of a raw Unicode
// // SMP code point.
// LETTERS : ('a'|'\uD835\uDDBA'|'\uD835\uDDBE'|'\uD835\uDDC2'|'\uD835\uDDC8'|'\uD835\uDDCE')* 'c';
// */
// @CommentHasStringValue
// public String grammar;
// }
public static class UnicodeEscapedSMPSet extends BaseParserTestDescriptor {
public String input = new StringBuilder()
.append("a")
.appendCodePoint(0x1D5C2)
.appendCodePoint(0x1D5CE)
.appendCodePoint(0x1D5BA)
.append("c")
.toString();
public String output = new StringBuilder()
.append("a")
.appendCodePoint(0x1D5C2)
.appendCodePoint(0x1D5CE)
.appendCodePoint(0x1D5BA)
.append("c\n")
.toString();
public String errors = null;
public String startRule = "a";
public String grammarName = "T";
/**
grammar T;
a : LETTERS {<InputText():writeln()>} ;
// Note the double-backslash to avoid Java passing
// unescaped values as part of the grammar.
LETTERS : ('a'|'\\u{1D5BA}'|'\\u{1D5BE}'|'\\u{1D5C2}'|'\\u{1D5C8}'|'\\u{1D5CE}')* 'c';
*/
@CommentHasStringValue
public String grammar;
}
// Turns out Tool.java uses ANTLR 3's runtime, which means it can't use
// CodePointCharStream to understand unescaped code points > U+FFFF.
//
// TODO(bhamiltoncx): This needs to be an error, since we don't currently plan
// to port Tool.java to use ANTLR 4's runtime.
// public static class UnicodeUnescapedSMPRangeSet extends BaseParserTestDescriptor {
// public String input = new StringBuilder()
// .append("a")
// .appendCodePoint(0x1D5C2)
// .appendCodePoint(0x1D5CE)
// .appendCodePoint(0x1D5BA)
// .append("d")
// .toString();
// public String output = new StringBuilder()
// .append("a")
// .appendCodePoint(0x1D5C2)
// .appendCodePoint(0x1D5CE)
// .appendCodePoint(0x1D5BA)
// .append("d\n")
// .toString();
// public String errors = null;
// public String startRule = "a";
// public String grammarName = "T";
// /**
// grammar T;
// a : LETTERS* 'd' {<InputText():writeln()>} ;
// // These are actually not escaped -- Java passes the
// // raw unescaped Unicode values to the grammar compiler.
// LETTERS : ('a'|'\uD83D\uDE00'..'\uD83E\uDD43');
// */
// @CommentHasStringValue
// public String grammar;
// }
public static class UnicodeEscapedSMPRangeSet extends BaseParserTestDescriptor {
public String input = new StringBuilder()
.append("a")
.appendCodePoint(0x1F609)
.appendCodePoint(0x1F942)
.appendCodePoint(0x1F700)
.append("d")
.toString();
public String output = new StringBuilder()
.append("a")
.appendCodePoint(0x1F609)
.appendCodePoint(0x1F942)
.appendCodePoint(0x1F700)
.append("d\n")
.toString();
public String errors = null;
public String startRule = "a";
public String grammarName = "T";
/**
grammar T;
a : LETTERS* 'd' {<InputText():writeln()>} ;
// Note the double-backslash to avoid Java passing
// unescaped values as part of the grammar.
LETTERS : ('a'|'\\u{1F600}'..'\\u{1F943}');
*/
@CommentHasStringValue
public String grammar;
}
public static class UnicodeEscapedSMPRangeSetMismatch extends BaseParserTestDescriptor {
// Test the code points just before and just after the range.
public String input = new StringBuilder()
.append("a")
.appendCodePoint(0x1F5FF)
.appendCodePoint(0x1F944)
.append("d")
.toString();
public String output = "ad\n";
public String errors = new StringBuilder()
.append("line 1:1 token recognition error at: '")
.appendCodePoint(0x1F5FF)
.append("'\n")
.append("line 1:2 token recognition error at: '")
.appendCodePoint(0x1F944)
.append("'\n")
.toString();
public String startRule = "a";
public String grammarName = "T";
/**
grammar T;
a : LETTERS* 'd' {<InputText():writeln()>} ;
// Note the double-backslash to avoid Java passing
// unescaped values as part of the grammar.
LETTERS : ('a'|'\\u{1F600}'..'\\u{1F943}');
*/
@CommentHasStringValue
public String grammar;
}
public static class UnicodeNegatedBMPSetIncludesSMPCodePoints extends BaseParserTestDescriptor {
public String input = "a\uD83D\uDE33\uD83D\uDE21\uD83D\uDE1D\uD83E\uDD13c";
public String output = "a\uD83D\uDE33\uD83D\uDE21\uD83D\uDE1D\uD83E\uDD13c\n";
public String errors = null;
public String startRule = "a";
public String grammarName = "T";
/**
grammar T;
a : LETTERS {<InputText():writeln()>} ;
LETTERS : 'a' ~('b')+ 'c';
*/
@CommentHasStringValue
public String grammar;
}
public static class UnicodeNegatedSMPSetIncludesBMPCodePoints extends BaseParserTestDescriptor {
public String input = "abc";
public String output = "abc\n";
public String errors = null;
public String startRule = "a";
public String grammarName = "T";
/**
grammar T;
a : LETTERS {<InputText():writeln()>} ;
LETTERS : 'a' ~('\\u{1F600}'..'\\u{1F943}')+ 'c';
*/
@CommentHasStringValue
public String grammar;
}
}

32
runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/ATNDeserializer.cs
View File

@ -22,6 +22,18 @@ namespace Antlr4.Runtime.Atn
/// <remarks>This is the earliest supported serialized UUID.</remarks>
private static readonly Guid BaseSerializedUuid;
/// <summary>
/// This UUID indicates the serialized ATN contains two sets of
/// IntervalSets, where the second set's values are encoded as
/// 32-bit integers to support the full Unicode SMP range up to U+10FFFF.
/// </summary>
/// <remarks>
/// This UUID indicates the serialized ATN contains two sets of
/// IntervalSets, where the second set's values are encoded as
/// 32-bit integers to support the full Unicode SMP range up to U+10FFFF.
/// </remarks>
private static readonly Guid AddedUnicodeSmp;
/// <summary>
/// This list contains all of the currently supported UUIDs, ordered by when
/// the feature first appeared in this branch.
@ -39,14 +51,18 @@ namespace Antlr4.Runtime.Atn
static ATNDeserializer()
{
BaseSerializedUuid = new Guid("AADB8D7E-AEEF-4415-AD2B-8204D6CF042E");
AddedUnicodeSmp = new Guid("59627784-3BE5-417A-B9EB-8131A7286089");
SupportedUuids = new List<Guid>();
SupportedUuids.Add(BaseSerializedUuid);
SerializedUuid = BaseSerializedUuid;
SupportedUuids.Add(AddedUnicodeSmp);
SerializedUuid = AddedUnicodeSmp;
}
[NotNull]
private readonly ATNDeserializationOptions deserializationOptions;
private Guid uuid;
public ATNDeserializer()
: this(ATNDeserializationOptions.Default)
{
@ -115,7 +131,11 @@ namespace Antlr4.Runtime.Atn
ReadStates (atn);
ReadRules (atn);
ReadModes (atn);
IList<IntervalSet> sets = ReadSets (atn);
IList<IntervalSet> sets = new List<IntervalSet>();
ReadSets (atn, sets, this.ReadInt);
if (IsFeatureSupported(AddedUnicodeSmp, uuid)) {
ReadSets (atn, sets, this.ReadInt32);
}
ReadEdges (atn, sets);
ReadDecisions (atn);
ReadLexerActions (atn);
@ -378,12 +398,11 @@ namespace Antlr4.Runtime.Atn
}
}
protected internal virtual IList<IntervalSet> ReadSets(ATN atn)
protected internal virtual void ReadSets(ATN atn, IList<IntervalSet> sets, Func<int> readUnicode)
{
//
// SETS
//
IList<IntervalSet> sets = new List<IntervalSet>();
int nsets = ReadInt();
for (int i_8 = 0; i_8 < nsets; i_8++)
{
@ -397,10 +416,9 @@ namespace Antlr4.Runtime.Atn
}
for (int j = 0; j < nintervals; j++)
{
set.Add(ReadInt(), ReadInt());
set.Add(readUnicode(), readUnicode());
}
}
return sets;
}
protected internal virtual void ReadModes(ATN atn)
@ -530,7 +548,7 @@ namespace Antlr4.Runtime.Atn
protected internal virtual void CheckUUID()
{
Guid uuid = ReadUUID();
uuid = ReadUUID();
if (!SupportedUuids.Contains(uuid))
{
string reason = string.Format(CultureInfo.CurrentCulture, "Could not deserialize ATN with UUID {0} (expected {1} or a legacy UUID).", uuid, SerializedUuid);

72
runtime/Cpp/runtime/src/atn/ATNDeserializer.cpp
View File

@ -57,6 +57,51 @@ using namespace antlrcpp;
const size_t ATNDeserializer::SERIALIZED_VERSION = 3;
namespace {
uint32_t deserializeInt32(const std::vector<uint16_t>& data, size_t offset) {
return (uint32_t)data[offset] | ((uint32_t)data[offset + 1] << 16);
}
ssize_t readUnicodeInt(const std::vector<uint16_t>& data, int& p) {
return static_cast<ssize_t>(data[p++]);
}
ssize_t readUnicodeInt32(const std::vector<uint16_t>& data, int& p) {
auto result = deserializeInt32(data, p);
p += 2;
return static_cast<ssize_t>(result);
}
// We templatize this on the function type so the optimizer can inline
// the 16- or 32-bit readUnicodeInt/readUnicodeInt32 as needed.
template <typename F>
void deserializeSets(
const std::vector<uint16_t>& data,
int& p,
std::vector<misc::IntervalSet>& sets,
F readUnicode) {
int nsets = data[p++];
for (int i = 0; i < nsets; i++) {
int nintervals = data[p++];
misc::IntervalSet set;
bool containsEof = data[p++] != 0;
if (containsEof) {
set.add(-1);
}
for (int j = 0; j < nintervals; j++) {
auto a = readUnicode(data, p);
auto b = readUnicode(data, p);
set.add(a, b);
}
sets.push_back(set);
}
}
}
ATNDeserializer::ATNDeserializer(): ATNDeserializer(ATNDeserializationOptions::getDefaultOptions()) {
}
@ -75,8 +120,12 @@ Guid ATNDeserializer::ADDED_LEXER_ACTIONS() {
return Guid("AADB8D7E-AEEF-4415-AD2B-8204D6CF042E");
}
Guid ATNDeserializer::ADDED_UNICODE_SMP() {
return Guid("59627784-3BE5-417A-B9EB-8131A7286089");
}
Guid ATNDeserializer::SERIALIZED_UUID() {
return ADDED_LEXER_ACTIONS();
return ADDED_UNICODE_SMP();
}
Guid ATNDeserializer::BASE_SERIALIZED_UUID() {
@ -84,7 +133,7 @@ Guid ATNDeserializer::BASE_SERIALIZED_UUID() {
}
std::vector<Guid>& ATNDeserializer::SUPPORTED_UUIDS() {
static std::vector<Guid> singleton = { BASE_SERIALIZED_UUID(), ADDED_PRECEDENCE_TRANSITIONS(), ADDED_LEXER_ACTIONS() };
static std::vector<Guid> singleton = { BASE_SERIALIZED_UUID(), ADDED_PRECEDENCE_TRANSITIONS(), ADDED_LEXER_ACTIONS(), ADDED_UNICODE_SMP() };
return singleton;
}
@ -239,21 +288,14 @@ ATN ATNDeserializer::deserialize(const std::vector<uint16_t>& input) {
// SETS
//
std::vector<misc::IntervalSet> sets;
int nsets = data[p++];
for (int i = 0; i < nsets; i++) {
int nintervals = data[p++];
misc::IntervalSet set;
bool containsEof = data[p++] != 0;
if (containsEof) {
set.add(-1);
}
// First, deserialize sets with 16-bit arguments <= U+FFFF.
deserializeSets(data, p, sets, readUnicodeInt);
for (int j = 0; j < nintervals; j++) {
set.add(data[p], data[p + 1], true);
p += 2;
}
sets.push_back(set);
// Next, if the ATN was serialized with the Unicode SMP feature,
// deserialize sets with 32-bit arguments <= U+10FFFF.
if (isFeatureSupported(ADDED_UNICODE_SMP(), uuid)) {
deserializeSets(data, p, sets, readUnicodeInt32);
}
//

7
runtime/Cpp/runtime/src/atn/ATNDeserializer.h
View File

@ -67,6 +67,13 @@ namespace atn {
*/
static Guid ADDED_LEXER_ACTIONS();
/**
* This UUID indicates the serialized ATN contains two sets of
* IntervalSets, where the second set's values are encoded as
* 32-bit integers to support the full Unicode SMP range up to U+10FFFF.
*/
static Guid ADDED_UNICODE_SMP();
/// This list contains all of the currently supported UUIDs, ordered by when
/// the feature first appeared in this branch.
static std::vector<Guid>& SUPPORTED_UUIDS();

9
runtime/Cpp/runtime/src/misc/Interval.cpp
View File

@ -24,14 +24,7 @@ Interval::Interval() : Interval((ssize_t)-1, -2) { // Need an explicit cast here
Interval::Interval(size_t a_, size_t b_) : Interval(symbolToNumeric(a_), symbolToNumeric(b_)) {
}
Interval::Interval(ssize_t a_, ssize_t b_, bool autoExtend) {
a = a_;
b = b_;
// XXX: temporary hack to make the full Unicode range available.
if (autoExtend && b == 0xFFFF) {
b = 0x10FFFF;
}
Interval::Interval(ssize_t a_, ssize_t b_) : a(a_), b(b_) {
}
size_t Interval::length() const {

2
runtime/Cpp/runtime/src/misc/Interval.h
View File

@ -27,7 +27,7 @@ namespace misc {
Interval();
explicit Interval(size_t a_, size_t b_); // For unsigned -> signed mappings.
Interval(ssize_t a_, ssize_t b_, bool autoExtend = false); // Automatically extend a value of 0xFFFF to 0x10FFFF.
Interval(ssize_t a_, ssize_t b_);
virtual ~Interval() {};
/// return number of elements between a and b inclusively. x..x is length 1.

8
runtime/Cpp/runtime/src/misc/IntervalSet.cpp
View File

@ -50,8 +50,8 @@ IntervalSet IntervalSet::of(ssize_t a) {
return IntervalSet({ Interval(a, a) });
}
IntervalSet IntervalSet::of(ssize_t a, ssize_t b, bool autoExtend) {
return IntervalSet({ Interval(a, b, autoExtend) });
IntervalSet IntervalSet::of(ssize_t a, ssize_t b) {
return IntervalSet({ Interval(a, b) });
}
void IntervalSet::clear() {
@ -68,8 +68,8 @@ void IntervalSet::add(ssize_t el) {
add(el, el);
}
void IntervalSet::add(ssize_t a, ssize_t b, bool autoExtend) {
add(Interval(a, b, autoExtend));
void IntervalSet::add(ssize_t a, ssize_t b) {
add(Interval(a, b));
}
void IntervalSet::add(const Interval &addition) {

4
runtime/Cpp/runtime/src/misc/IntervalSet.h
View File

@ -44,7 +44,7 @@ namespace misc {
static IntervalSet of(ssize_t a);
/// Create a set with all ints within range [a..b] (inclusive)
static IntervalSet of(ssize_t a, ssize_t b, bool autoExtend = false);
static IntervalSet of(ssize_t a, ssize_t b);
virtual void clear();
@ -58,7 +58,7 @@ namespace misc {
/// If overlap, combine ranges. For example,
/// If this is {1..5, 10..20}, adding 6..7 yields
/// {1..5, 6..7, 10..20}. Adding 4..8 yields {1..8, 10..20}.
virtual void add(ssize_t a, ssize_t b, bool autoExtend = false);
virtual void add(ssize_t a, ssize_t b);
public:
/// combine all sets in the array returned the or'd value

28
runtime/Go/antlr/atn_deserializer.go
View File

@ -15,15 +15,16 @@ import (
// This is the earliest supported serialized UUID.
// stick to serialized version for now, we don't need a UUID instance
var BaseSerializedUUID = "AADB8D7E-AEEF-4415-AD2B-8204D6CF042E"
var AddedUnicodeSMP = "59627784-3BE5-417A-B9EB-8131A7286089"
// This list contains all of the currently supported UUIDs, ordered by when
// the feature first appeared in this branch.
var SupportedUUIDs = []string{BaseSerializedUUID}
var SupportedUUIDs = []string{BaseSerializedUUID, AddedUnicodeSMP}
var SerializedVersion = 3
// This is the current serialized UUID.
var SerializedUUID = BaseSerializedUUID
var SerializedUUID = AddedUnicodeSMP
type LoopEndStateIntPair struct {
item0 *LoopEndState
@ -91,7 +92,15 @@ func (a *ATNDeserializer) DeserializeFromUInt16(data []uint16) *ATN {
a.readRules(atn)
a.readModes(atn)
sets := a.readSets(atn)
sets := make([]*IntervalSet, 0)
// First, deserialize sets with 16-bit arguments <= U+FFFF.
sets = a.readSets(atn, sets, a.readInt)
// Next, if the ATN was serialized with the Unicode SMP feature,
// deserialize sets with 32-bit arguments <= U+10FFFF.
if (a.isFeatureSupported(AddedUnicodeSMP, a.uuid)) {
sets = a.readSets(atn, sets, a.readInt32)
}
a.readEdges(atn, sets)
a.readDecisions(atn)
@ -266,8 +275,7 @@ func (a *ATNDeserializer) readModes(atn *ATN) {
}
}
func (a *ATNDeserializer) readSets(atn *ATN) []*IntervalSet {
sets := make([]*IntervalSet, 0)
func (a *ATNDeserializer) readSets(atn *ATN, sets []*IntervalSet, readUnicode func() int) []*IntervalSet {
m := a.readInt()
for i := 0; i < m; i++ {
@ -283,8 +291,8 @@ func (a *ATNDeserializer) readSets(atn *ATN) []*IntervalSet {
}
for j := 0; j < n; j++ {
i1 := a.readInt()
i2 := a.readInt()
i1 := readUnicode()
i2 := readUnicode()
iset.addRange(i1, i2)
}
@ -642,6 +650,12 @@ func (a *ATNDeserializer) readInt() int {
return int(v)
}
func (a *ATNDeserializer) readInt32() int {
var low = a.readInt()
var high = a.readInt()
return low | (high << 16)
}
//TODO
//func (a *ATNDeserializer) readLong() int64 {
// panic("Not implemented")

97
runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java
View File

@ -44,6 +44,12 @@ public class ATNDeserializer {
* {@link LexerAction} instances.
*/
private static final UUID ADDED_LEXER_ACTIONS;
/**
* This UUID indicates the serialized ATN contains two sets of
* IntervalSets, where the second set's values are encoded as
* 32-bit integers to support the full Unicode SMP range up to U+10FFFF.
*/
private static final UUID ADDED_UNICODE_SMP;
/**
* This list contains all of the currently supported UUIDs, ordered by when
* the feature first appeared in this branch.
@ -61,15 +67,58 @@ public class ATNDeserializer {
BASE_SERIALIZED_UUID = UUID.fromString("33761B2D-78BB-4A43-8B0B-4F5BEE8AACF3");
ADDED_PRECEDENCE_TRANSITIONS = UUID.fromString("1DA0C57D-6C06-438A-9B27-10BCB3CE0F61");
ADDED_LEXER_ACTIONS = UUID.fromString("AADB8D7E-AEEF-4415-AD2B-8204D6CF042E");
ADDED_UNICODE_SMP = UUID.fromString("59627784-3BE5-417A-B9EB-8131A7286089");
SUPPORTED_UUIDS = new ArrayList<UUID>();
SUPPORTED_UUIDS.add(BASE_SERIALIZED_UUID);
SUPPORTED_UUIDS.add(ADDED_PRECEDENCE_TRANSITIONS);
SUPPORTED_UUIDS.add(ADDED_LEXER_ACTIONS);
SUPPORTED_UUIDS.add(ADDED_UNICODE_SMP);
SERIALIZED_UUID = ADDED_LEXER_ACTIONS;
SERIALIZED_UUID = ADDED_UNICODE_SMP;
}
interface UnicodeDeserializer {
// Wrapper for readInt() or readInt32()
int readUnicode(char[] data, int p);
// Work around Java not allowing mutation of captured variables
// by returning amount by which to increment p after each read
int size();
}
enum UnicodeDeserializingMode {
UNICODE_BMP,
UNICODE_SMP
}
static UnicodeDeserializer getUnicodeDeserializer(UnicodeDeserializingMode mode) {
if (mode == UnicodeDeserializingMode.UNICODE_BMP) {
return new UnicodeDeserializer() {
@Override
public int readUnicode(char[] data, int p) {
return toInt(data[p]);
}
@Override
public int size() {
return 1;
}
};
} else {
return new UnicodeDeserializer() {
@Override
public int readUnicode(char[] data, int p) {
return toInt32(data, p);
}
@Override
public int size() {
return 2;
}
};
}
}
private final ATNDeserializationOptions deserializationOptions;
@ -98,7 +147,7 @@ public class ATNDeserializer {
* serialized ATN at or after the feature identified by {@code feature} was
* introduced; otherwise, {@code false}.
*/
protected boolean isFeatureSupported(UUID feature, UUID actualUuid) {
static protected boolean isFeatureSupported(UUID feature, UUID actualUuid) {
int featureIndex = SUPPORTED_UUIDS.indexOf(feature);
if (featureIndex < 0) {
return false;
@ -258,22 +307,14 @@ public class ATNDeserializer {
// SETS
//
List<IntervalSet> sets = new ArrayList<IntervalSet>();
int nsets = toInt(data[p++]);
for (int i=0; i<nsets; i++) {
int nintervals = toInt(data[p]);
p++;
IntervalSet set = new IntervalSet();
sets.add(set);
boolean containsEof = toInt(data[p++]) != 0;
if (containsEof) {
set.add(-1);
}
// First, read all sets with 16-bit Unicode code points <= U+FFFF.
p = deserializeSets(data, p, sets, getUnicodeDeserializer(UnicodeDeserializingMode.UNICODE_BMP));
for (int j=0; j<nintervals; j++) {
set.add(toInt(data[p]), toInt(data[p + 1]));
p += 2;
}
// Next, if the ATN was serialized with the Unicode SMP feature,
// deserialize sets with 32-bit arguments <= U+10FFFF.
if (isFeatureSupported(ADDED_UNICODE_SMP, uuid)) {
p = deserializeSets(data, p, sets, getUnicodeDeserializer(UnicodeDeserializingMode.UNICODE_SMP));
}
//
@ -510,6 +551,30 @@ public class ATNDeserializer {
return atn;
}
private int deserializeSets(char[] data, int p, List<IntervalSet> sets, UnicodeDeserializer unicodeDeserializer) {
int nsets = toInt(data[p++]);
for (int i=0; i<nsets; i++) {
int nintervals = toInt(data[p]);
p++;
IntervalSet set = new IntervalSet();
sets.add(set);
boolean containsEof = toInt(data[p++]) != 0;
if (containsEof) {
set.add(-1);
}
for (int j=0; j<nintervals; j++) {
int a = unicodeDeserializer.readUnicode(data, p);
p += unicodeDeserializer.size();
int b = unicodeDeserializer.readUnicode(data, p);
p += unicodeDeserializer.size();
set.add(a, b);
}
}
return p;
}
/**
* Analyze the {@link StarLoopEntryState} states in the specified ATN to set
* the {@link StarLoopEntryState#isPrecedenceDecision} field to the

165
runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java
View File

@ -14,8 +14,10 @@ import org.antlr.v4.runtime.misc.Utils;
import java.io.InvalidClassException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.UUID;
@ -24,6 +26,10 @@ public class ATNSerializer {
public ATN atn;
private List<String> tokenNames;
private interface CodePointSerializer {
void serializeCodePoint(IntegerList data, int cp);
}
public ATNSerializer(ATN atn) {
assert atn.grammarType != null;
this.atn = atn;
@ -47,9 +53,11 @@ public class ATNSerializer {
* (args are token type,actionIndex in lexer else 0,0)
* num modes,
* mode-0-start-state, mode-1-start-state, ... (parser has 0 modes)
* num sets
* set-0-interval-count intervals, set-1-interval-count intervals, ...
* num total edges,
* num unicode-bmp-sets
* bmp-set-0-interval-count intervals, bmp-set-1-interval-count intervals, ...
* num unicode-smp-sets
* smp-set-0-interval-count intervals, smp-set-1-interval-count intervals, ...
* num total edges,
* src, trg, edge-type, edge arg1, optional edge arg2 (present always), ...
* num decisions,
* decision-0-start-state, decision-1-start-state, ...
@ -66,8 +74,10 @@ public class ATNSerializer {
data.add(atn.maxTokenType);
int nedges = 0;
Map<IntervalSet, Integer> setIndices = new HashMap<IntervalSet, Integer>();
List<IntervalSet> sets = new ArrayList<IntervalSet>();
// Note that we use a LinkedHashMap as a set to
// maintain insertion order while deduplicating
// entries with the same key.
Map<IntervalSet, Boolean> sets = new LinkedHashMap<>();
// dump states, count edges and collect sets while doing so
IntegerList nonGreedyStates = new IntegerList();
@ -114,10 +124,7 @@ public class ATNSerializer {
int edgeType = Transition.serializationTypes.get(t.getClass());
if ( edgeType == Transition.SET || edgeType == Transition.NOT_SET ) {
SetTransition st = (SetTransition)t;
if (!setIndices.containsKey(st.set)) {
sets.add(st.set);
setIndices.put(st.set, sets.size() - 1);
}
sets.put(st.set, true);
}
}
}
@ -156,34 +163,40 @@ public class ATNSerializer {
data.add(modeStartState.stateNumber);
}
}
int nsets = sets.size();
data.add(nsets);
for (IntervalSet set : sets) {
boolean containsEof = set.contains(Token.EOF);
if (containsEof && set.getIntervals().get(0).b == Token.EOF) {
data.add(set.getIntervals().size() - 1);
List<IntervalSet> bmpSets = new ArrayList<>();
List<IntervalSet> smpSets = new ArrayList<>();
for (IntervalSet set : sets.keySet()) {
if (set.getMaxElement() <= Character.MAX_VALUE) {
bmpSets.add(set);
} else {
smpSets.add(set);
}
else {
data.add(set.getIntervals().size());
}
data.add(containsEof ? 1 : 0);
for (Interval I : set.getIntervals()) {
if (I.a == Token.EOF) {
if (I.b == Token.EOF) {
continue;
}
else {
data.add(0);
}
}
serializeSets(
data,
bmpSets,
new CodePointSerializer() {
@Override
public void serializeCodePoint(IntegerList data, int cp) {
data.add(cp);
}
else {
data.add(I.a);
});
serializeSets(
data,
smpSets,
new CodePointSerializer() {
@Override
public void serializeCodePoint(IntegerList data, int cp) {
serializeInt(data, cp);
}
data.add(I.b);
}
});
Map<IntervalSet, Integer> setIndices = new HashMap<>();
int setIndex = 0;
for (IntervalSet bmpSet : bmpSets) {
setIndices.put(bmpSet, setIndex++);
}
for (IntervalSet smpSet : smpSets) {
setIndices.put(smpSet, setIndex++);
}
data.add(nedges);
@ -359,6 +372,42 @@ public class ATNSerializer {
return data;
}
private static void serializeSets(
IntegerList data,
Collection<IntervalSet> sets,
CodePointSerializer codePointSerializer)
{
int nSets = sets.size();
data.add(nSets);
for (IntervalSet set : sets) {
boolean containsEof = set.contains(Token.EOF);
if (containsEof && set.getIntervals().get(0).b == Token.EOF) {
data.add(set.getIntervals().size() - 1);
}
else {
data.add(set.getIntervals().size());
}
data.add(containsEof ? 1 : 0);
for (Interval I : set.getIntervals()) {
if (I.a == Token.EOF) {
if (I.b == Token.EOF) {
continue;
}
else {
codePointSerializer.serializeCodePoint(data, 0);
}
}
else {
codePointSerializer.serializeCodePoint(data, I.a);
}
codePointSerializer.serializeCodePoint(data, I.b);
}
}
}
public String decode(char[] data) {
data = data.clone();
// don't adjust the first value since that's the version number
@ -437,25 +486,10 @@ public class ATNSerializer {
int s = ATNDeserializer.toInt(data[p++]);
buf.append("mode ").append(i).append(":").append(s).append('\n');
}
int nsets = ATNDeserializer.toInt(data[p++]);
for (int i=0; i<nsets; i++) {
int nintervals = ATNDeserializer.toInt(data[p++]);
buf.append(i).append(":");
boolean containsEof = data[p++] != 0;
if (containsEof) {
buf.append(getTokenName(Token.EOF));
}
for (int j=0; j<nintervals; j++) {
if ( containsEof || j>0 ) {
buf.append(", ");
}
buf.append(getTokenName(ATNDeserializer.toInt(data[p]))).append("..").append(getTokenName(ATNDeserializer.toInt(data[p + 1])));
p += 2;
}
buf.append("\n");
}
int numBMPSets = ATNDeserializer.toInt(data[p++]);
p = appendSets(buf, data, p, numBMPSets, 0, ATNDeserializer.getUnicodeDeserializer(ATNDeserializer.UnicodeDeserializingMode.UNICODE_BMP));
int numSMPSets = ATNDeserializer.toInt(data[p++]);
p = appendSets(buf, data, p, numSMPSets, numBMPSets, ATNDeserializer.getUnicodeDeserializer(ATNDeserializer.UnicodeDeserializingMode.UNICODE_SMP));
int nedges = ATNDeserializer.toInt(data[p++]);
for (int i=0; i<nedges; i++) {
int src = ATNDeserializer.toInt(data[p]);
@ -491,6 +525,31 @@ public class ATNSerializer {
return buf.toString();
}
private int appendSets(StringBuilder buf, char[] data, int p, int nsets, int setIndexOffset, ATNDeserializer.UnicodeDeserializer unicodeDeserializer) {
for (int i=0; i<nsets; i++) {
int nintervals = ATNDeserializer.toInt(data[p++]);
buf.append(i+setIndexOffset).append(":");
boolean containsEof = data[p++] != 0;
if (containsEof) {
buf.append(getTokenName(Token.EOF));
}
for (int j=0; j<nintervals; j++) {
if ( containsEof || j>0 ) {
buf.append(", ");
}
int a = unicodeDeserializer.readUnicode(data, p);
p += unicodeDeserializer.size();
int b = unicodeDeserializer.readUnicode(data, p);
p += unicodeDeserializer.size();
buf.append(getTokenName(a)).append("..").append(getTokenName(b));
}
buf.append("\n");
}
return p;
}
public String getTokenName(int t) {
if ( t==-1 ) return "EOF";

32
runtime/JavaScript/src/antlr4/atn/ATNDeserializer.js
View File

@ -51,14 +51,21 @@ var LexerModeAction = LexerActions.LexerModeAction;
// stick to serialized version for now, we don't need a UUID instance
var BASE_SERIALIZED_UUID = "AADB8D7E-AEEF-4415-AD2B-8204D6CF042E";
//
// This UUID indicates the serialized ATN contains two sets of
// IntervalSets, where the second set's values are encoded as
// 32-bit integers to support the full Unicode SMP range up to U+10FFFF.
//
var ADDED_UNICODE_SMP = "59627784-3BE5-417A-B9EB-8131A7286089";
// This list contains all of the currently supported UUIDs, ordered by when
// the feature first appeared in this branch.
var SUPPORTED_UUIDS = [ BASE_SERIALIZED_UUID ];
var SUPPORTED_UUIDS = [ BASE_SERIALIZED_UUID, ADDED_UNICODE_SMP ];
var SERIALIZED_VERSION = 3;
// This is the current serialized UUID.
var SERIALIZED_UUID = BASE_SERIALIZED_UUID;
var SERIALIZED_UUID = ADDED_UNICODE_SMP;
function initArray( length, value) {
var tmp = [];
@ -91,11 +98,11 @@ function ATNDeserializer (options) {
// introduced; otherwise, {@code false}.
ATNDeserializer.prototype.isFeatureSupported = function(feature, actualUuid) {
var idx1 = SUPPORTED_UUIDS.index(feature);
var idx1 = SUPPORTED_UUIDS.indexOf(feature);
if (idx1<0) {
return false;
}
var idx2 = SUPPORTED_UUIDS.index(actualUuid);
var idx2 = SUPPORTED_UUIDS.indexOf(actualUuid);
return idx2 >= idx1;
};
@ -107,7 +114,14 @@ ATNDeserializer.prototype.deserialize = function(data) {
this.readStates(atn);
this.readRules(atn);
this.readModes(atn);
var sets = this.readSets(atn);
var sets = [];
// First, deserialize sets with 16-bit arguments <= U+FFFF.
this.readSets(atn, sets, this.readInt.bind(this));
// Next, if the ATN was serialized with the Unicode SMP feature,
// deserialize sets with 32-bit arguments <= U+10FFFF.
if (this.isFeatureSupported(ADDED_UNICODE_SMP, this.uuid)) {
this.readSets(atn, sets, this.readInt32.bind(this));
}
this.readEdges(atn, sets);
this.readDecisions(atn);
this.readLexerActions(atn);
@ -244,8 +258,7 @@ ATNDeserializer.prototype.readModes = function(atn) {
}
};
ATNDeserializer.prototype.readSets = function(atn) {
var sets = [];
ATNDeserializer.prototype.readSets = function(atn, sets, readUnicode) {
var m = this.readInt();
for (var i=0; i<m; i++) {
var iset = new IntervalSet();
@ -256,12 +269,11 @@ ATNDeserializer.prototype.readSets = function(atn) {
iset.addOne(-1);
}
for (var j=0; j<n; j++) {
var i1 = this.readInt();
var i2 = this.readInt();
var i1 = readUnicode();
var i2 = readUnicode();
iset.addRange(i1, i2);
}
}
return sets;
};
ATNDeserializer.prototype.readEdges = function(atn, sets) {

18
runtime/Python2/src/antlr4/Lexer.py
View File

@ -278,7 +278,7 @@ class Lexer(Recognizer, TokenSource):
start = self._tokenStartCharIndex
stop = self._input.index
text = self._input.getText(start, stop)
msg = "token recognition error at: '" + self.getErrorDisplay(text) + "'"
msg = u"token recognition error at: '" + self.getErrorDisplay(text) + u"'"
listener = self.getErrorListenerDispatch()
listener.syntaxError(self, None, self._tokenStartLine, self._tokenStartColumn, msg, e)
@ -291,17 +291,17 @@ class Lexer(Recognizer, TokenSource):
def getErrorDisplayForChar(self, c):
if ord(c[0])==Token.EOF:
return "<EOF>"
elif c=='\n':
return "\\n"
elif c=='\t':
return "\\t"
elif c=='\r':
return "\\r"
elif c==u'\n':
return u"\\n"
elif c==u'\t':
return u"\\t"
elif c==u'\r':
return u"\\r"
else:
return unicode(c)
return c
def getCharErrorDisplay(self, c):
return "'" + self.getErrorDisplayForChar(c) + "'"
return u"'" + self.getErrorDisplayForChar(c) + u"'"
# Lexers can normally match any char in it's vocabulary after matching
# a token, so do the easy thing and just kill a character and hope

25
runtime/Python2/src/antlr4/atn/ATNDeserializer.py
View File

@ -13,14 +13,19 @@ from antlr4.atn.ATNDeserializationOptions import ATNDeserializationOptions
# This is the earliest supported serialized UUID.
BASE_SERIALIZED_UUID = UUID("AADB8D7E-AEEF-4415-AD2B-8204D6CF042E")
# This UUID indicates the serialized ATN contains two sets of
# IntervalSets, where the second set's values are encoded as
# 32-bit integers to support the full Unicode SMP range up to U+10FFFF.
ADDED_UNICODE_SMP = UUID("59627784-3BE5-417A-B9EB-8131A7286089")
# This list contains all of the currently supported UUIDs, ordered by when
# the feature first appeared in this branch.
SUPPORTED_UUIDS = [ BASE_SERIALIZED_UUID ]
SUPPORTED_UUIDS = [ BASE_SERIALIZED_UUID, ADDED_UNICODE_SMP ]
SERIALIZED_VERSION = 3
# This is the current serialized UUID.
SERIALIZED_UUID = BASE_SERIALIZED_UUID
SERIALIZED_UUID = ADDED_UNICODE_SMP
class ATNDeserializer (object):
@ -59,7 +64,13 @@ class ATNDeserializer (object):
self.readStates(atn)
self.readRules(atn)
self.readModes(atn)
sets = self.readSets(atn)
sets = []
# First, read all sets with 16-bit Unicode code points <= U+FFFF.
self.readSets(atn, sets, self.readInt)
# Next, if the ATN was serialized with the Unicode SMP feature,
# deserialize sets with 32-bit arguments <= U+10FFFF.
if self.isFeatureSupported(ADDED_UNICODE_SMP, self.uuid):
self.readSets(atn, sets, self.readInt32)
self.readEdges(atn, sets)
self.readDecisions(atn)
self.readLexerActions(atn)
@ -170,8 +181,7 @@ class ATNDeserializer (object):
s = self.readInt()
atn.modeToStartState.append(atn.states[s])
def readSets(self, atn):
sets = []
def readSets(self, atn, sets, readUnicode):
m = self.readInt()
for i in range(0, m):
iset = IntervalSet()
@ -181,10 +191,9 @@ class ATNDeserializer (object):
if containsEof!=0:
iset.addOne(-1)
for j in range(0, n):
i1 = self.readInt()
i2 = self.readInt()
i1 = readUnicode()
i2 = readUnicode()
iset.addRange(Interval(i1, i2 + 1)) # range upper limit is exclusive
return sets
def readEdges(self, atn, sets):
nedges = self.readInt()

26
runtime/Python3/src/antlr4/atn/ATNDeserializer.py
View File

@ -4,6 +4,7 @@
#/
from uuid import UUID
from io import StringIO
from typing import Callable
from antlr4.Token import Token
from antlr4.atn.ATN import ATN
from antlr4.atn.ATNType import ATNType
@ -15,14 +16,19 @@ from antlr4.atn.ATNDeserializationOptions import ATNDeserializationOptions
# This is the earliest supported serialized UUID.
BASE_SERIALIZED_UUID = UUID("AADB8D7E-AEEF-4415-AD2B-8204D6CF042E")
# This UUID indicates the serialized ATN contains two sets of
# IntervalSets, where the second set's values are encoded as
# 32-bit integers to support the full Unicode SMP range up to U+10FFFF.
ADDED_UNICODE_SMP = UUID("59627784-3BE5-417A-B9EB-8131A7286089")
# This list contains all of the currently supported UUIDs, ordered by when
# the feature first appeared in this branch.
SUPPORTED_UUIDS = [ BASE_SERIALIZED_UUID ]
SUPPORTED_UUIDS = [ BASE_SERIALIZED_UUID, ADDED_UNICODE_SMP ]
SERIALIZED_VERSION = 3
# This is the current serialized UUID.
SERIALIZED_UUID = BASE_SERIALIZED_UUID
SERIALIZED_UUID = ADDED_UNICODE_SMP
class ATNDeserializer (object):
@ -58,7 +64,13 @@ class ATNDeserializer (object):
self.readStates(atn)
self.readRules(atn)
self.readModes(atn)
sets = self.readSets(atn)
sets = []
# First, read all sets with 16-bit Unicode code points <= U+FFFF.
self.readSets(atn, sets, self.readInt)
# Next, if the ATN was serialized with the Unicode SMP feature,
# deserialize sets with 32-bit arguments <= U+10FFFF.
if self.isFeatureSupported(ADDED_UNICODE_SMP, self.uuid):
self.readSets(atn, sets, self.readInt32)
self.readEdges(atn, sets)
self.readDecisions(atn)
self.readLexerActions(atn)
@ -170,8 +182,7 @@ class ATNDeserializer (object):
s = self.readInt()
atn.modeToStartState.append(atn.states[s])
def readSets(self, atn:ATN):
sets = []
def readSets(self, atn:ATN, sets:list, readUnicode:Callable[[], int]):
m = self.readInt()
for i in range(0, m):
iset = IntervalSet()
@ -181,10 +192,9 @@ class ATNDeserializer (object):
if containsEof!=0:
iset.addOne(-1)
for j in range(0, n):
i1 = self.readInt()
i2 = self.readInt()
i1 = readUnicode()
i2 = readUnicode()
iset.addRange(range(i1, i2 + 1)) # range upper limit is exclusive
return sets
def readEdges(self, atn:ATN, sets:list):
nedges = self.readInt()

72
runtime/Swift/Sources/Antlr4/atn/ATNDeserializer.swift
View File

@ -26,21 +26,30 @@ public class ATNDeserializer {
/// for the addition of lexer actions encoded as a sequence of
/// {@link org.antlr.v4.runtime.atn.LexerAction} instances.
private static let ADDED_LEXER_ACTIONS: UUID = UUID(uuidString: "AADB8D7E-AEEF-4415-AD2B-8204D6CF042E")!
/// This list contains all of the currently supported UUIDs, ordered by when
/// the feature first appeared in this branch.
/// This UUID indicates the serialized ATN contains two sets of
/// IntervalSets, where the second set's values are encoded as
/// 32-bit integers to support the full Unicode SMP range up to U+10FFFF.
private static let ADDED_UNICODE_SMP: UUID = UUID(uuidString: "59627784-3BE5-417A-B9EB-8131A7286089")!
/**
* This list contains all of the currently supported UUIDs, ordered by when
* the feature first appeared in this branch.
*/
private static let SUPPORTED_UUIDS: Array<UUID> = {
var suuid = Array<UUID>()
suuid.append(ATNDeserializer.BASE_SERIALIZED_UUID)
suuid.append(ATNDeserializer.ADDED_PRECEDENCE_TRANSITIONS)
suuid.append(ATNDeserializer.ADDED_LEXER_ACTIONS)
suuid.append(ATNDeserializer.ADDED_UNICODE_SMP)
return suuid
}()
/// This is the current serialized UUID.
public static let SERIALIZED_UUID: UUID = {
// SERIALIZED_UUID = ADDED_LEXER_ACTIONS;
return UUID(uuidString: "AADB8D7E-AEEF-4415-AD2B-8204D6CF042E")!
// SERIALIZED_UUID = ADDED_UNICODE_SMP;
return UUID(uuidString: "59627784-3BE5-417A-B9EB-8131A7286089")!
}()
@ -245,24 +254,14 @@ public class ATNDeserializer {
// SETS
//
var sets: Array<IntervalSet> = Array<IntervalSet>()
let nsets: Int = toInt(data[p])
p += 1
for _ in 0..<nsets {
let nintervals: Int = toInt(data[p])
p += 1
let set: IntervalSet = try IntervalSet()
sets.append(set)
let containsEof: Bool = toInt(data[p]) != 0
p += 1
if containsEof {
try set.add(-1)
}
// First, deserialize sets with 16-bit arguments <= U+FFFF.
try readSets(data, &p, &sets, readUnicodeInt)
for _ in 0..<nintervals {
try set.add(toInt(data[p]), toInt(data[p + 1]))
p += 2
}
// Next, if the ATN was serialized with the Unicode SMP feature,
// deserialize sets with 32-bit arguments <= U+10FFFF.
if isFeatureSupported(ATNDeserializer.ADDED_UNICODE_SMP, uuid) {
try readSets(data, &p, &sets, readUnicodeInt32)
}
//
@ -521,6 +520,39 @@ public class ATNDeserializer {
return atn
}
private func readUnicodeInt(_ data: [Character], _ p: inout Int) -> Int {
let result: Int = toInt(data[p])
p += 1
return result
}
private func readUnicodeInt32(_ data: [Character], _ p: inout Int) -> Int {
let result: Int = toInt32(data, p)
p += 2
return result
}
private func readSets(_ data: [Character], _ p: inout Int, _ sets: inout Array<IntervalSet>, _ readUnicode: ([Character], inout Int) -> Int) throws {
let nsets: Int = toInt(data[p])
p += 1
for _ in 0..<nsets {
let nintervals: Int = toInt(data[p])
p += 1
let set: IntervalSet = try IntervalSet()
sets.append(set)
let containsEof: Bool = toInt(data[p]) != 0
p += 1
if containsEof {
try set.add(-1)
}
for _ in 0..<nintervals {
try set.add(readUnicode(data, &p), readUnicode(data, &p))
}
}
}
public func deserializeFromJson(_ jsonStr: String) -> ATN {
// let jsonStr = Utils.readFile2String(jsonFileName)
guard !jsonStr.isEmpty else {

92
tool-testsuite/test/org/antlr/v4/test/tool/TestATNLexerInterpreter.java
View File

@ -6,8 +6,8 @@
package org.antlr.v4.test.tool;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.misc.Utils;
@ -121,6 +121,94 @@ public class TestATNLexerInterpreter extends BaseJavaToolTest {
checkLexerMatches(lg, "c", expecting);
}
@Test public void testLexerSetUnicodeBMP() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ('\u611B'|'\u611C')\n ;");
String expecting = "ID, EOF";
checkLexerMatches(lg, "\u611B", expecting);
}
@Test public void testLexerNotSetUnicodeBMP() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ~('\u611B'|'\u611C')\n ;");
String expecting = "ID, EOF";
checkLexerMatches(lg, "\u611D", expecting);
}
@Test public void testLexerNotSetUnicodeBMPMatchesSMP() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ~('\u611B'|'\u611C')\n ;");
String expecting = "ID, EOF";
checkLexerMatches(lg, new StringBuilder().appendCodePoint(0x1F4A9).toString(), expecting);
}
@Test public void testLexerSetUnicodeSMP() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ('\\u{1F4A9}'|'\\u{1F4AA}')\n ;");
String expecting = "ID, EOF";
checkLexerMatches(lg, new StringBuilder().appendCodePoint(0x1F4A9).toString(), expecting);
}
@Test public void testLexerNotBMPSetMatchesUnicodeSMP() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ~('a'|'b')\n ;");
String expecting = "ID, EOF";
checkLexerMatches(lg, new StringBuilder().appendCodePoint(0x1F4A9).toString(), expecting);
}
@Test public void testLexerNotBMPSetMatchesBMP() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ~('a'|'b')\n ;");
String expecting = "ID, EOF";
checkLexerMatches(lg, "\u611B", expecting);
}
@Test public void testLexerNotBMPSetMatchesSMP() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ~('a'|'b')\n ;");
String expecting = "ID, EOF";
checkLexerMatches(lg, new StringBuilder().appendCodePoint(0x1F4A9).toString(), expecting);
}
@Test public void testLexerNotSMPSetMatchesBMP() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ~('\\u{1F4A9}'|'\\u{1F4AA}')\n ;");
String expecting = "ID, EOF";
checkLexerMatches(lg, "\u611B", expecting);
}
@Test public void testLexerNotSMPSetMatchesSMP() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ~('\\u{1F4A9}'|'\\u{1F4AA}')\n ;");
String expecting = "ID, EOF";
checkLexerMatches(lg, new StringBuilder().appendCodePoint(0x1D7C0).toString(), expecting);
}
@Test public void testLexerRangeUnicodeSMP() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ('\\u{1F4A9}'..'\\u{1F4B0}')\n ;");
String expecting = "ID, EOF";
checkLexerMatches(lg, new StringBuilder().appendCodePoint(0x1F4AF).toString(), expecting);
}
@Test public void testLexerRangeUnicodeBMPToSMP() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ('\\u611B'..'\\u{1F4B0}')\n ;");
String expecting = "ID, EOF";
checkLexerMatches(lg, new StringBuilder().appendCodePoint(0x12001).toString(), expecting);
}
@Test public void testLexerKeywordIDAmbiguity() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
@ -293,7 +381,7 @@ public class TestATNLexerInterpreter extends BaseJavaToolTest {
protected void checkLexerMatches(LexerGrammar lg, String inputString, String expecting) {
ATN atn = createATN(lg, true);
CharStream input = new ANTLRInputStream(inputString);
CharStream input = CharStreams.createWithString(inputString);
ATNState startState = atn.modeNameToStartState.get("DEFAULT_MODE");
DOTGenerator dot = new DOTGenerator(lg);
// System.out.println(dot.getDOT(startState, true));

323
tool-testsuite/test/org/antlr/v4/test/tool/TestATNSerialization.java
View File

@ -291,6 +291,113 @@ public class TestATNSerialization extends BaseJavaToolTest {
assertEquals(expecting, result);
}
@Test public void testLexerUnicodeSMPLiteralSerializedToSet() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"INT : '\\u{1F4A9}' ;");
String expecting =
"max type 1\n" +
"0:TOKEN_START -1\n" +
"1:RULE_START 0\n" +
"2:RULE_STOP 0\n" +
"3:BASIC 0\n" +
"4:BASIC 0\n" +
"rule 0:1 1\n" +
"mode 0:0\n" +
"0:128169..128169\n" +
"0->1 EPSILON 0,0,0\n" +
"1->3 EPSILON 0,0,0\n" +
"3->4 SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@Test public void testLexerUnicodeSMPRangeSerializedToSet() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"INT : ('a'..'\\u{1F4A9}') ;");
String expecting =
"max type 1\n" +
"0:TOKEN_START -1\n" +
"1:RULE_START 0\n" +
"2:RULE_STOP 0\n" +
"3:BASIC 0\n" +
"4:BASIC 0\n" +
"rule 0:1 1\n" +
"mode 0:0\n" +
"0:'a'..128169\n" +
"0->1 EPSILON 0,0,0\n" +
"1->3 EPSILON 0,0,0\n" +
"3->4 SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@Test public void testLexerUnicodeSMPSetSerializedAfterBMPSet() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"SMP : ('\\u{1F4A9}' | '\\u{1F4AF}') ;\n"+
"BMP : ('a' | 'x') ;");
String expecting =
"max type 2\n" +
"0:TOKEN_START -1\n" +
"1:RULE_START 0\n" +
"2:RULE_STOP 0\n" +
"3:RULE_START 1\n" +
"4:RULE_STOP 1\n" +
"5:BASIC 0\n" +
"6:BASIC 0\n" +
"7:BASIC 1\n" +
"8:BASIC 1\n" +
"rule 0:1 1\n" +
"rule 1:3 2\n" +
"mode 0:0\n" +
"0:'a'..'a', 'x'..'x'\n" +
"1:128169..128169, 128175..128175\n" +
"0->1 EPSILON 0,0,0\n" +
"0->3 EPSILON 0,0,0\n" +
"1->5 EPSILON 0,0,0\n" +
"3->7 EPSILON 0,0,0\n" +
"5->6 SET 1,0,0\n" +
"6->2 EPSILON 0,0,0\n" +
"7->8 SET 0,0,0\n" +
"8->4 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@Test public void testLexerNotLiteral() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"INT : ~'a' ;");
String expecting =
"max type 1\n" +
"0:TOKEN_START -1\n" +
"1:RULE_START 0\n" +
"2:RULE_STOP 0\n" +
"3:BASIC 0\n" +
"4:BASIC 0\n" +
"rule 0:1 1\n" +
"mode 0:0\n" +
"0:'a'..'a'\n" +
"0->1 EPSILON 0,0,0\n" +
"1->3 EPSILON 0,0,0\n" +
"3->4 NOT_SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@Test public void testLexerRange() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
@ -518,6 +625,222 @@ public class TestATNSerialization extends BaseJavaToolTest {
assertEquals(expecting, result);
}
@Test public void testLexerUnicodeUnescapedBMPNotSet() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ~('\u4E9C'|'\u4E9D')\n ;");
String expecting =
"max type 1\n" +
"0:TOKEN_START -1\n" +
"1:RULE_START 0\n" +
"2:RULE_STOP 0\n" +
"3:BASIC 0\n" +
"4:BASIC 0\n" +
"rule 0:1 1\n" +
"mode 0:0\n" +
"0:'\\u4E9C'..'\\u4E9D'\n" +
"0->1 EPSILON 0,0,0\n" +
"1->3 EPSILON 0,0,0\n" +
"3->4 NOT_SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@Test public void testLexerUnicodeUnescapedBMPSetWithRange() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ('\u4E9C'|'\u4E9D'|'\u6C5F'|'\u305F'..'\u307B')\n ;");
String expecting =
"max type 1\n" +
"0:TOKEN_START -1\n" +
"1:RULE_START 0\n" +
"2:RULE_STOP 0\n" +
"3:BASIC 0\n" +
"4:BASIC 0\n" +
"rule 0:1 1\n" +
"mode 0:0\n" +
"0:'\\u305F'..'\\u307B', '\\u4E9C'..'\\u4E9D', '\\u6C5F'..'\\u6C5F'\n" +
"0->1 EPSILON 0,0,0\n" +
"1->3 EPSILON 0,0,0\n" +
"3->4 SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@Test public void testLexerUnicodeUnescapedBMPNotSetWithRange() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ~('\u4E9C'|'\u4E9D'|'\u6C5F'|'\u305F'..'\u307B')\n ;");
String expecting =
"max type 1\n" +
"0:TOKEN_START -1\n" +
"1:RULE_START 0\n" +
"2:RULE_STOP 0\n" +
"3:BASIC 0\n" +
"4:BASIC 0\n" +
"rule 0:1 1\n" +
"mode 0:0\n" +
"0:'\\u305F'..'\\u307B', '\\u4E9C'..'\\u4E9D', '\\u6C5F'..'\\u6C5F'\n" +
"0->1 EPSILON 0,0,0\n" +
"1->3 EPSILON 0,0,0\n" +
"3->4 NOT_SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@Test public void testLexerUnicodeEscapedBMPNotSet() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ~('\\u4E9C'|'\\u4E9D')\n ;");
String expecting =
"max type 1\n" +
"0:TOKEN_START -1\n" +
"1:RULE_START 0\n" +
"2:RULE_STOP 0\n" +
"3:BASIC 0\n" +
"4:BASIC 0\n" +
"rule 0:1 1\n" +
"mode 0:0\n" +
"0:'\\u4E9C'..'\\u4E9D'\n" +
"0->1 EPSILON 0,0,0\n" +
"1->3 EPSILON 0,0,0\n" +
"3->4 NOT_SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@Test public void testLexerUnicodeEscapedBMPSetWithRange() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ('\\u4E9C'|'\\u4E9D'|'\\u6C5F'|'\\u305F'..'\\u307B')\n ;");
String expecting =
"max type 1\n" +
"0:TOKEN_START -1\n" +
"1:RULE_START 0\n" +
"2:RULE_STOP 0\n" +
"3:BASIC 0\n" +
"4:BASIC 0\n" +
"rule 0:1 1\n" +
"mode 0:0\n" +
"0:'\\u305F'..'\\u307B', '\\u4E9C'..'\\u4E9D', '\\u6C5F'..'\\u6C5F'\n" +
"0->1 EPSILON 0,0,0\n" +
"1->3 EPSILON 0,0,0\n" +
"3->4 SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@Test public void testLexerUnicodeEscapedBMPNotSetWithRange() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ~('\\u4E9C'|'\\u4E9D'|'\\u6C5F'|'\\u305F'..'\\u307B')\n ;");
String expecting =
"max type 1\n" +
"0:TOKEN_START -1\n" +
"1:RULE_START 0\n" +
"2:RULE_STOP 0\n" +
"3:BASIC 0\n" +
"4:BASIC 0\n" +
"rule 0:1 1\n" +
"mode 0:0\n" +
"0:'\\u305F'..'\\u307B', '\\u4E9C'..'\\u4E9D', '\\u6C5F'..'\\u6C5F'\n" +
"0->1 EPSILON 0,0,0\n" +
"1->3 EPSILON 0,0,0\n" +
"3->4 NOT_SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@Test public void testLexerUnicodeEscapedSMPNotSet() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ~('\\u{1F4A9}'|'\\u{1F4AA}')\n ;");
String expecting =
"max type 1\n" +
"0:TOKEN_START -1\n" +
"1:RULE_START 0\n" +
"2:RULE_STOP 0\n" +
"3:BASIC 0\n" +
"4:BASIC 0\n" +
"rule 0:1 1\n" +
"mode 0:0\n" +
"0:128169..128170\n" +
"0->1 EPSILON 0,0,0\n" +
"1->3 EPSILON 0,0,0\n" +
"3->4 NOT_SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@Test public void testLexerUnicodeEscapedSMPSetWithRange() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ('\\u{1F4A9}'|'\\u{1F4AA}'|'\\u{1F441}'|'\\u{1D40F}'..'\\u{1D413}')\n ;");
String expecting =
"max type 1\n" +
"0:TOKEN_START -1\n" +
"1:RULE_START 0\n" +
"2:RULE_STOP 0\n" +
"3:BASIC 0\n" +
"4:BASIC 0\n" +
"rule 0:1 1\n" +
"mode 0:0\n" +
"0:119823..119827, 128065..128065, 128169..128170\n" +
"0->1 EPSILON 0,0,0\n" +
"1->3 EPSILON 0,0,0\n" +
"3->4 SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@Test public void testLexerUnicodeEscapedSMPNotSetWithRange() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+
"ID : ~('\\u{1F4A9}'|'\\u{1F4AA}'|'\\u{1F441}'|'\\u{1D40F}'..'\\u{1D413}')\n ;");
String expecting =
"max type 1\n" +
"0:TOKEN_START -1\n" +
"1:RULE_START 0\n" +
"2:RULE_STOP 0\n" +
"3:BASIC 0\n" +
"4:BASIC 0\n" +
"rule 0:1 1\n" +
"mode 0:0\n" +
"0:119823..119827, 128065..128065, 128169..128170\n" +
"0->1 EPSILON 0,0,0\n" +
"1->3 EPSILON 0,0,0\n" +
"3->4 NOT_SET 0,0,0\n" +
"4->2 EPSILON 0,0,0\n" +
"0:0\n";
ATN atn = createATN(lg, true);
String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames()));
assertEquals(expecting, result);
}
@Test public void testLexerWildcardWithMode() throws Exception {
LexerGrammar lg = new LexerGrammar(
"lexer grammar L;\n"+

18
tool-testsuite/test/org/antlr/v4/test/tool/TestTokenTypeAssignment.java
View File

@ -141,6 +141,24 @@ public class TestTokenTypeAssignment extends BaseJavaToolTest {
assertEquals("'\\n'", literals.toArray()[0]);
}
@Test public void testParserCharLiteralWithBasicUnicodeEscape() throws Exception {
Grammar g = new Grammar(
"grammar t;\n"+
"a : '\\uABCD';\n");
Set<?> literals = g.stringLiteralToTypeMap.keySet();
// must store literals how they appear in the antlr grammar
assertEquals("'\\uABCD'", literals.toArray()[0]);
}
@Test public void testParserCharLiteralWithExtendedUnicodeEscape() throws Exception {
Grammar g = new Grammar(
"grammar t;\n"+
"a : '\\u{1ABCD}';\n");
Set<?> literals = g.stringLiteralToTypeMap.keySet();
// must store literals how they appear in the antlr grammar
assertEquals("'\\u{1ABCD}'", literals.toArray()[0]);
}
protected void checkSymbols(Grammar g,
String rulesStr,
String allValidTokensStr)

172
tool-testsuite/test/org/antlr/v4/test/tool/TestUnicodeGrammar.java Normal file
View File

@ -0,0 +1,172 @@
/*
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
* Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root.
*/
package org.antlr.v4.test.tool;
import java.io.ByteArrayInputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import org.antlr.v4.gui.Trees;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.LexerInterpreter;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.GrammarParserInterpreter;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
public class TestUnicodeGrammar extends BaseJavaToolTest {
@Test
public void unicodeBMPLiteralInGrammar() throws Exception {
String grammarText =
"grammar Unicode;\n" +
"r : 'hello' WORLD;\n" +
"WORLD : ('world' | '\\u4E16\\u754C' | '\\u1000\\u1019\\u1039\\u1018\\u102C' );\n" +
"WS : [ \\t\\r\\n]+ -> skip;\n";
String inputText = "hello \u4E16\u754C";
assertEquals(
"(r:1 " + inputText + ")",
parseTreeForGrammarWithInput(
grammarText,
"r",
inputText));
}
// TODO: This test cannot pass unless we change either the grammar
// parser to decode surrogate pair literals to code points (which
// would break existing clients) or to treat them as an
// alternative:
//
// '\\uD83C\\uDF0D' -> ('\\u{1F30E}' | '\\uD83C\\uDF0D')
//
// but I worry that might cause parse ambiguity if we're not careful.
//@Test
public void unicodeSurrogatePairLiteralInGrammar() throws Exception {
String grammarText =
"grammar Unicode;\n" +
"r : 'hello' WORLD;\n" +
"WORLD : ('\\uD83C\\uDF0D' | '\\uD83C\\uDF0E' | '\\uD83C\\uDF0F' );\n" +
"WS : [ \\t\\r\\n]+ -> skip;\n";
String inputText = new StringBuilder("hello ")
.appendCodePoint(0x1F30E)
.toString();
assertEquals(
"(r:1 " + inputText + ")",
parseTreeForGrammarWithInput(
grammarText,
"r",
inputText));
}
@Test
public void unicodeSMPLiteralInGrammar() throws Exception {
String grammarText =
"grammar Unicode;\n" +
"r : 'hello' WORLD;\n" +
"WORLD : ('\\u{1F30D}' | '\\u{1F30E}' | '\\u{1F30F}' );\n" +
"WS : [ \\t\\r\\n]+ -> skip;\n";
String inputText = new StringBuilder("hello ")
.appendCodePoint(0x1F30E)
.toString();
assertEquals(
"(r:1 " + inputText + ")",
parseTreeForGrammarWithInput(
grammarText,
"r",
inputText));
}
@Test
public void unicodeSMPRangeInGrammar() throws Exception {
String grammarText =
"grammar Unicode;\n" +
"r : 'hello' WORLD;\n" +
"WORLD : ('\\u{1F30D}'..'\\u{1F30F}' );\n" +
"WS : [ \\t\\r\\n]+ -> skip;\n";
String inputText = new StringBuilder("hello ")
.appendCodePoint(0x1F30E)
.toString();
assertEquals(
"(r:1 " + inputText + ")",
parseTreeForGrammarWithInput(
grammarText,
"r",
inputText));
}
@Test
public void matchingDanglingSurrogateInInput() throws Exception {
String grammarText =
"grammar Unicode;\n" +
"r : 'hello' WORLD;\n" +
"WORLD : ('\\uD83C' | '\\uD83D' | '\\uD83E' );\n" +
"WS : [ \\t\\r\\n]+ -> skip;\n";
String inputText = "hello \uD83C";
assertEquals(
"(r:1 " + inputText + ")",
parseTreeForGrammarWithInput(
grammarText,
"r",
inputText));
}
@Test
public void binaryGrammar() throws Exception {
String grammarText =
"grammar Binary;\n" +
"r : HEADER PACKET+ FOOTER;\n" +
"HEADER : '\\u0002\\u0000\\u0001\\u0007';\n" +
"PACKET : '\\u00D0' ('\\u00D1' | '\\u00D2' | '\\u00D3') +;\n" +
"FOOTER : '\\u00FF';\n";
byte[] toParse = new byte[] {
(byte)0x02, (byte)0x00, (byte)0x01, (byte)0x07,
(byte)0xD0, (byte)0xD2, (byte)0xD2, (byte)0xD3, (byte)0xD3, (byte)0xD3,
(byte)0xD0, (byte)0xD3, (byte)0xD3, (byte)0xD1,
(byte)0xFF
};
CharStream charStream;
try (ByteArrayInputStream is = new ByteArrayInputStream(toParse);
// Note we use ISO_8859_1 to treat all byte values as Unicode "characters" from
// U+0000 to U+00FF.
InputStreamReader isr = new InputStreamReader(is, StandardCharsets.ISO_8859_1)) {
charStream = new ANTLRInputStream(isr);
}
Grammar grammar = new Grammar(grammarText);
LexerInterpreter lexEngine = grammar.createLexerInterpreter(charStream);
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
GrammarParserInterpreter parser = grammar.createGrammarParserInterpreter(tokens);
ParseTree parseTree = parser.parse(grammar.rules.get("r").index);
InterpreterTreeTextProvider nodeTextProvider =
new InterpreterTreeTextProvider(grammar.getRuleNames());
String result = Trees.toStringTree(parseTree, nodeTextProvider);
assertEquals(
"(r:1 \u0002\u0000\u0001\u0007 \u00D0\u00D2\u00D2\u00D3\u00D3\u00D3 \u00D0\u00D3\u00D3\u00D1 \u00FF)",
result);
}
private static String parseTreeForGrammarWithInput(
String grammarText,
String rootRule,
String inputText) throws Exception {
Grammar grammar = new Grammar(grammarText);
LexerInterpreter lexEngine = grammar.createLexerInterpreter(
CharStreams.createWithString(inputText));
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
GrammarParserInterpreter parser = grammar.createGrammarParserInterpreter(tokens);
ParseTree parseTree = parser.parse(grammar.rules.get(rootRule).index);
InterpreterTreeTextProvider nodeTextProvider =
new InterpreterTreeTextProvider(grammar.getRuleNames());
return Trees.toStringTree(parseTree, nodeTextProvider);
}
}

16
tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg
View File

@ -288,7 +288,7 @@ public partial class <csIdentifier.(parser.name)> : <superClass; null="Parser">
public override string[] RuleNames { get { return ruleNames; } }
public override string SerializedAtn { get { return _serializedATN; } }
public override string SerializedAtn { get { return new string(_serializedATN); } }
static <csIdentifier.(parser.name)>() {
decisionToDFA = new DFA[_ATN.NumberOfDecisions];
@ -1023,7 +1023,7 @@ public partial class <csIdentifier.(lexer.name)> : <superClass; null="Lexer"> {
public override string[] ModeNames { get { return modeNames; } }
public override string SerializedAtn { get { return _serializedATN; } }
public override string SerializedAtn { get { return new string(_serializedATN); } }
static <csIdentifier.(lexer.name)>() {
decisionToDFA = new DFA[_ATN.NumberOfDecisions];
@ -1038,16 +1038,12 @@ public partial class <csIdentifier.(lexer.name)> : <superClass; null="Lexer"> {
SerializedATN(model) ::= <<
private static string _serializedATN = _serializeATN();
private static string _serializeATN()
{
StringBuilder sb = new StringBuilder();
sb.Append("<model.serialized; wrap={");<\n><\t>sb.Append("}>");
return sb.ToString();
}
private static char[] _serializedATN = {
<model.serialized; separator=", ", wrap>,
};
public static readonly ATN _ATN =
new ATNDeserializer().Deserialize(_serializedATN.ToCharArray());
new ATNDeserializer().Deserialize(_serializedATN);
>>

10
tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg
View File

@ -192,23 +192,23 @@ atn::ATN <lexer.name>::_atn;
std::vector\<uint16_t> <lexer.name>::_serializedATN;
std::vector\<std::string> <lexer.name>::_ruleNames = {
<lexer.ruleNames: {r | "<r>"}; separator = ", ", wrap, anchor>
<lexer.ruleNames: {r | u8"<r>"}; separator = ", ", wrap, anchor>
};
std::vector\<std::string> <lexer.name>::_channelNames = {
"DEFAULT_TOKEN_CHANNEL", "HIDDEN"<if (lexer.channels)>, <lexer.channels: {c | "<c>"}; separator = ", ", wrap, anchor><endif>
"DEFAULT_TOKEN_CHANNEL", "HIDDEN"<if (lexer.channels)>, <lexer.channels: {c | u8"<c>"}; separator = ", ", wrap, anchor><endif>
};
std::vector\<std::string> <lexer.name>::_modeNames = {
<lexer.modes: {m | "<m>"}; separator = ", ", wrap, anchor>
<lexer.modes: {m | u8"<m>"}; separator = ", ", wrap, anchor>
};
std::vector\<std::string> <lexer.name>::_literalNames = {
<lexer.literalNames: {t | <t>}; null = "\"\"", separator = ", ", wrap, anchor>
<lexer.literalNames: {t | u8<t>}; null = "\"\"", separator = ", ", wrap, anchor>
};
std::vector\<std::string> <lexer.name>::_symbolicNames = {
<lexer.symbolicNames: {t | <t>}; null = "\"\"", separator = ", ", wrap, anchor>
<lexer.symbolicNames: {t | u8<t>}; null = "\"\"", separator = ", ", wrap, anchor>
};
dfa::Vocabulary <lexer.name>::_vocabulary(_literalNames, _symbolicNames);

2
tool/src/org/antlr/v4/codegen/target/CSharpTarget.java
View File

@ -46,7 +46,7 @@ public class CSharpTarget extends Target {
formatted = String.format("\\x%X", v & 0xFFFF);
}
return formatted;
return "'" + formatted + "'";
}
@Override

74
tool/src/org/antlr/v4/misc/CharSupport.java
View File

@ -45,10 +45,9 @@ public class CharSupport {
}
/** Return a string representing the escaped char for code c. E.g., If c
* has value 0x100, you will get "\u0100". ASCII gets the usual
* char (non-hex) representation. Control characters are spit out
* as unicode. While this is specially set up for returning Java strings,
* it can be used by any language target that has the same syntax. :)
* has value 0x100, you will get "\\u0100". ASCII gets the usual
* char (non-hex) representation. Non-ASCII characters are spit out
* as \\uXXXX or \\u{XXXXXX} escapes.
*/
public static String getANTLRCharLiteralForChar(int c) {
if ( c< Lexer.MIN_CHAR_VALUE ) {
@ -67,11 +66,11 @@ public class CharSupport {
}
return '\''+Character.toString((char)c)+'\'';
}
// turn on the bit above max "\uFFFF" value so that we pad with zeros
// then only take last 4 digits
String hex = Integer.toHexString(c|0x10000).toUpperCase().substring(1,5);
String unicodeStr = "'\\u"+hex+"'";
return unicodeStr;
if (c <= 0xFFFF) {
return String.format("\\u%04X", c);
} else {
return String.format("\\u{%06X}", c);
}
}
/** Given a literal like (the 3 char sequence with single quotes) 'a',
@ -92,11 +91,25 @@ public class CharSupport {
if ( literal.charAt(i) == '\\' ) {
end = i+2;
if ( i+1 < n && literal.charAt(i+1) == 'u' ) {
for (end = i + 2; end < i + 6; end++) {
if ( end>n ) return null; // invalid escape sequence.
char charAt = literal.charAt(end);
if (!Character.isDigit(charAt) && !(charAt >= 'a' && charAt <= 'f') && !(charAt >= 'A' && charAt <= 'F')) {
return null; // invalid escape sequence.
if ( i+2 < n && literal.charAt(i+2) == '{' ) { // extended escape sequence
end = i + 3;
while (true) {
if ( end + 1 > n ) return null; // invalid escape sequence.
char charAt = literal.charAt(end++);
if (charAt == '}') {
break;
}
if (!Character.isDigit(charAt) && !(charAt >= 'a' && charAt <= 'f') && !(charAt >= 'A' && charAt <= 'F')) {
return null; // invalid escape sequence.
}
}
} else {
for (end = i + 2; end < i + 6; end++) {
if ( end>n ) return null; // invalid escape sequence.
char charAt = literal.charAt(end);
if (!Character.isDigit(charAt) && !(charAt >= 'a' && charAt <= 'f') && !(charAt >= 'A' && charAt <= 'F')) {
return null; // invalid escape sequence.
}
}
}
}
@ -107,13 +120,13 @@ public class CharSupport {
if ( c==-1 ) {
return null; // invalid escape sequence.
}
else buf.append((char)c);
else buf.appendCodePoint(c);
i = end;
}
return buf.toString();
}
/** Given char x or \t or \u1234 return the char value;
/** Given char x or \\t or \\u1234 return the char value;
* Unnecessary escapes like '\{' yield -1.
*/
public static int getCharValueFromCharInGrammarLiteral(String cstr) {
@ -130,9 +143,31 @@ public class CharSupport {
if ( charVal==0 ) return -1;
return charVal;
case 6:
// '\u1234'
// '\\u1234' or '\\u{12}'
if ( !cstr.startsWith("\\u") ) return -1;
String unicodeChars = cstr.substring(2, cstr.length());
int startOff;
int endOff;
if ( cstr.charAt(2) == '{' ) {
startOff = 3;
endOff = cstr.indexOf('}');
} else {
startOff = 2;
endOff = cstr.length();
}
return parseHexValue(cstr, startOff, endOff);
default:
if ( cstr.startsWith("\\u{") ) {
return parseHexValue(cstr, 3, cstr.indexOf('}'));
}
return -1;
}
}
private static int parseHexValue(String cstr, int startOff, int endOff) {
if (startOff < 0 || endOff < 0) {
return -1;
}
String unicodeChars = cstr.substring(startOff, endOff);
int result = -1;
try {
result = Integer.parseInt(unicodeChars, 16);
@ -140,9 +175,6 @@ public class CharSupport {
catch (NumberFormatException e) {
}
return result;
default:
return -1;
}
}
public static String capitalize(String s) {

29
tool/src/org/antlr/v4/parse/ANTLRLexer.g
View File

@ -615,8 +615,8 @@ SRC : 'src' WSCHARS+ file=ACTION_STRING_LITERAL WSCHARS+ line=INT
//
// ANTLR makes no disticintion between a single character literal and a
// multi-character string. All literals are single quote delimited and
// may contain unicode escape sequences of the form \uxxxx, where x
// is a valid hexadecimal number (as per Java basically).
// may contain unicode escape sequences of the form \uxxxx or \u{xxxxxx},
// where x is a valid hexadecimal number.
STRING_LITERAL
: '\'' ( ( ESC_SEQ | ~('\\'|'\''|'\r'|'\n') ) )*
( '\''
@ -652,6 +652,10 @@ ESC_SEQ
//
UNICODE_ESC
| // A Swift/Hack style Unicode escape sequence
//
UNICODE_EXTENDED_ESC
| // An illegal escape seqeunce
//
{
@ -720,6 +724,27 @@ UNICODE_ESC
}
;
fragment
UNICODE_EXTENDED_ESC
: 'u{' // Leadin for unicode extended escape sequence
HEX_DIGIT+ // One or more hexadecimal digits
'}' // Leadout for unicode extended escape sequence
// Now check the digit count and issue an error if we need to
{
int numDigits = getCharIndex()-state.tokenStartCharIndex-6;
if (numDigits > 6) {
Token t = new CommonToken(input, state.type, state.channel, state.tokenStartCharIndex, getCharIndex()-1);
t.setText(t.getText());
t.setLine(input.getLine());
t.setCharPositionInLine(input.getCharPositionInLine()-numDigits);
grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, t);
}
}
;
// ----------
// Whitespace
//