diff --git a/runtime/Cpp/demo/Mac/antlr4-cpp-demo/main.cpp b/runtime/Cpp/demo/Mac/antlr4-cpp-demo/main.cpp index d96976928..d5ab4263c 100644 --- a/runtime/Cpp/demo/Mac/antlr4-cpp-demo/main.cpp +++ b/runtime/Cpp/demo/Mac/antlr4-cpp-demo/main.cpp @@ -23,7 +23,7 @@ int main(int argc, const char * argv[]) { CommonTokenStream tokens(&lexer); TParser parser(&tokens); - std::shared_ptr tree = parser.main(); + Ref tree = parser.main(); std::cout << antlrcpp::ws2s(tree->toStringTree(&parser)) << std::endl; diff --git a/runtime/Cpp/demo/Mac/antlrcpp Tests/MiscClassTests.mm b/runtime/Cpp/demo/Mac/antlrcpp Tests/MiscClassTests.mm index f1758915e..d3e6aeb84 100644 --- a/runtime/Cpp/demo/Mac/antlrcpp Tests/MiscClassTests.mm +++ b/runtime/Cpp/demo/Mac/antlrcpp Tests/MiscClassTests.mm @@ -105,10 +105,10 @@ using namespace antlrcpp; delete a; delete b; delete c; delete d; } { - std::shared_ptr a(new A()); - std::shared_ptr b(new B()); - std::shared_ptr c(new C()); - std::shared_ptr d(new D()); + Ref a(new A()); + Ref b(new B()); + Ref c(new C()); + Ref d(new D()); XCTAssert(is(b)); XCTAssertFalse(is(a)); XCTAssert(is(c)); @@ -136,7 +136,7 @@ using namespace antlrcpp; hash = MurmurHash::finish(hash, data.size()); hashs.insert(hash); } - XCTAssertEqual(hashs.size(), 100000U, @"At least one duplicat hash found."); + XCTAssertEqual(hashs.size(), 100000U, @"At least one duplicate hash found."); hashs.clear(); for (size_t i = 0; i < 100000; ++i) { @@ -147,7 +147,7 @@ using namespace antlrcpp; hash = MurmurHash::finish(hash, data.size()); hashs.insert(hash); } - XCTAssertEqual(hashs.size(), 100000U, @"At least one duplicat hash found."); + XCTAssertEqual(hashs.size(), 100000U, @"At least one duplicate hash found."); // Another test with fixed input but varying seeds. // Note: the higher the seed the less LSDs are in the result (for small input data). @@ -160,7 +160,7 @@ using namespace antlrcpp; hash = MurmurHash::finish(hash, data.size()); hashs.insert(hash); } - XCTAssertEqual(hashs.size(), 100000U, @"At least one duplicat hash found."); + XCTAssertEqual(hashs.size(), 100000U, @"At least one duplicate hash found."); } - (void)testInterval { diff --git a/runtime/Cpp/demo/Mac/antlrcpp.xcodeproj/project.pbxproj b/runtime/Cpp/demo/Mac/antlrcpp.xcodeproj/project.pbxproj index cf9aa11d0..286061620 100644 --- a/runtime/Cpp/demo/Mac/antlrcpp.xcodeproj/project.pbxproj +++ b/runtime/Cpp/demo/Mac/antlrcpp.xcodeproj/project.pbxproj @@ -15,6 +15,14 @@ 2747A70D1CA691310030247B /* ConfigLookup.h in Headers */ = {isa = PBXBuildFile; fileRef = 2747A7091CA691310030247B /* ConfigLookup.h */; }; 2747A7131CA6C46C0030247B /* InputHandlingTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 2747A7121CA6C46C0030247B /* InputHandlingTests.mm */; }; 274FC6D91CA96B6C008D4374 /* MiscClassTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 274FC6D81CA96B6C008D4374 /* MiscClassTests.mm */; }; + 275411DE1CCE704300CAD5B0 /* LexerActionExecutor.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 275411DC1CCE704300CAD5B0 /* LexerActionExecutor.cpp */; }; + 275411DF1CCE704300CAD5B0 /* LexerActionExecutor.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 275411DC1CCE704300CAD5B0 /* LexerActionExecutor.cpp */; }; + 275411E01CCE704300CAD5B0 /* LexerActionExecutor.h in Headers */ = {isa = PBXBuildFile; fileRef = 275411DD1CCE704300CAD5B0 /* LexerActionExecutor.h */; }; + 275411E11CCE704300CAD5B0 /* LexerActionExecutor.h in Headers */ = {isa = PBXBuildFile; fileRef = 275411DD1CCE704300CAD5B0 /* LexerActionExecutor.h */; }; + 275411E41CCE733800CAD5B0 /* LexerIndexedCustomAction.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 275411E21CCE733800CAD5B0 /* LexerIndexedCustomAction.cpp */; }; + 275411E51CCE733800CAD5B0 /* LexerIndexedCustomAction.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 275411E21CCE733800CAD5B0 /* LexerIndexedCustomAction.cpp */; }; + 275411E61CCE733800CAD5B0 /* LexerIndexedCustomAction.h in Headers */ = {isa = PBXBuildFile; fileRef = 275411E31CCE733800CAD5B0 /* LexerIndexedCustomAction.h */; }; + 275411E71CCE733800CAD5B0 /* LexerIndexedCustomAction.h in Headers */ = {isa = PBXBuildFile; fileRef = 275411E31CCE733800CAD5B0 /* LexerIndexedCustomAction.h */; }; 275DB3E41CCD23C000D8C543 /* LexerChannelAction.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 275DB3D41CCD23C000D8C543 /* LexerChannelAction.cpp */; }; 275DB3E51CCD23C000D8C543 /* LexerChannelAction.h in Headers */ = {isa = PBXBuildFile; fileRef = 275DB3D51CCD23C000D8C543 /* LexerChannelAction.h */; }; 275DB3E61CCD23C000D8C543 /* LexerCustomAction.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 275DB3D61CCD23C000D8C543 /* LexerCustomAction.cpp */; }; @@ -49,6 +57,42 @@ 278A66FC1C95838E002D667E /* ANTLRErrorListener.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 278A66FA1C95838E002D667E /* ANTLRErrorListener.cpp */; }; 27A23EA31CC2A8D60036D8A3 /* TLexer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27A23EA11CC2A8D60036D8A3 /* TLexer.cpp */; }; 27A23EA41CC2A8D60036D8A3 /* TLexer.h in Headers */ = {isa = PBXBuildFile; fileRef = 27A23EA21CC2A8D60036D8A3 /* TLexer.h */; }; + 27C62E261CD269C90088721B /* ParseInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E241CD269C90088721B /* ParseInfo.cpp */; }; + 27C62E271CD269C90088721B /* ParseInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E241CD269C90088721B /* ParseInfo.cpp */; }; + 27C62E281CD269C90088721B /* ParseInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E251CD269C90088721B /* ParseInfo.h */; }; + 27C62E291CD269C90088721B /* ParseInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E251CD269C90088721B /* ParseInfo.h */; }; + 27C62E2C1CD26C780088721B /* ProfilingATNSimulator.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E2A1CD26C780088721B /* ProfilingATNSimulator.cpp */; }; + 27C62E2D1CD26C780088721B /* ProfilingATNSimulator.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E2A1CD26C780088721B /* ProfilingATNSimulator.cpp */; }; + 27C62E2E1CD26C780088721B /* ProfilingATNSimulator.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E2B1CD26C780088721B /* ProfilingATNSimulator.h */; }; + 27C62E2F1CD26C780088721B /* ProfilingATNSimulator.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E2B1CD26C780088721B /* ProfilingATNSimulator.h */; }; + 27C62E321CD26EEA0088721B /* DecisionInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E301CD26EEA0088721B /* DecisionInfo.cpp */; }; + 27C62E331CD26EEA0088721B /* DecisionInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E301CD26EEA0088721B /* DecisionInfo.cpp */; }; + 27C62E341CD26EEA0088721B /* DecisionInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E311CD26EEA0088721B /* DecisionInfo.h */; }; + 27C62E351CD26EEA0088721B /* DecisionInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E311CD26EEA0088721B /* DecisionInfo.h */; }; + 27C62E381CD271C60088721B /* ErrorInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E361CD271C60088721B /* ErrorInfo.cpp */; }; + 27C62E391CD271C60088721B /* ErrorInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E361CD271C60088721B /* ErrorInfo.cpp */; }; + 27C62E3A1CD271C60088721B /* ErrorInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E371CD271C60088721B /* ErrorInfo.h */; }; + 27C62E3B1CD271C60088721B /* ErrorInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E371CD271C60088721B /* ErrorInfo.h */; }; + 27C62E3E1CD272480088721B /* DecisionEventInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E3C1CD272480088721B /* DecisionEventInfo.cpp */; }; + 27C62E3F1CD272480088721B /* DecisionEventInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E3C1CD272480088721B /* DecisionEventInfo.cpp */; }; + 27C62E401CD272480088721B /* DecisionEventInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E3D1CD272480088721B /* DecisionEventInfo.h */; }; + 27C62E411CD272480088721B /* DecisionEventInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E3D1CD272480088721B /* DecisionEventInfo.h */; }; + 27C62E441CD273DE0088721B /* LookaheadEventInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E421CD273DE0088721B /* LookaheadEventInfo.cpp */; }; + 27C62E451CD273DE0088721B /* LookaheadEventInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E421CD273DE0088721B /* LookaheadEventInfo.cpp */; }; + 27C62E461CD273DE0088721B /* LookaheadEventInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E431CD273DE0088721B /* LookaheadEventInfo.h */; }; + 27C62E471CD273DE0088721B /* LookaheadEventInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E431CD273DE0088721B /* LookaheadEventInfo.h */; }; + 27C62E4A1CD274CA0088721B /* AmbiguityInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E481CD274CA0088721B /* AmbiguityInfo.cpp */; }; + 27C62E4B1CD274CA0088721B /* AmbiguityInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E481CD274CA0088721B /* AmbiguityInfo.cpp */; }; + 27C62E4C1CD274CA0088721B /* AmbiguityInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E491CD274CA0088721B /* AmbiguityInfo.h */; }; + 27C62E4D1CD274CA0088721B /* AmbiguityInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E491CD274CA0088721B /* AmbiguityInfo.h */; }; + 27C62E501CD275C50088721B /* ContextSensitivityInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E4E1CD275C50088721B /* ContextSensitivityInfo.cpp */; }; + 27C62E511CD275C50088721B /* ContextSensitivityInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E4E1CD275C50088721B /* ContextSensitivityInfo.cpp */; }; + 27C62E521CD275C50088721B /* ContextSensitivityInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E4F1CD275C50088721B /* ContextSensitivityInfo.h */; }; + 27C62E531CD275C50088721B /* ContextSensitivityInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E4F1CD275C50088721B /* ContextSensitivityInfo.h */; }; + 27C62E561CD2768A0088721B /* PredicateEvalInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E541CD2768A0088721B /* PredicateEvalInfo.cpp */; }; + 27C62E571CD2768A0088721B /* PredicateEvalInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C62E541CD2768A0088721B /* PredicateEvalInfo.cpp */; }; + 27C62E581CD2768A0088721B /* PredicateEvalInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E551CD2768A0088721B /* PredicateEvalInfo.h */; }; + 27C62E591CD2768A0088721B /* PredicateEvalInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C62E551CD2768A0088721B /* PredicateEvalInfo.h */; }; 27C666AC1C9584050021E494 /* ANTLRErrorListener.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C6665D1C9584050021E494 /* ANTLRErrorListener.h */; }; 27C666AD1C9584050021E494 /* ANTLRErrorListener.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C6665D1C9584050021E494 /* ANTLRErrorListener.h */; }; 27C666AE1C9584050021E494 /* ANTLRErrorStrategy.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C6665E1C9584050021E494 /* ANTLRErrorStrategy.cpp */; }; @@ -532,6 +576,12 @@ 27C6E1821C972FFC0079AF06 /* TParserBaseVisitor.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C6E1791C972FFC0079AF06 /* TParserBaseVisitor.cpp */; }; 27C6E1831C972FFC0079AF06 /* TParserListener.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C6E17B1C972FFC0079AF06 /* TParserListener.cpp */; }; 27C6E1841C972FFC0079AF06 /* TParserVisitor.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C6E17D1C972FFC0079AF06 /* TParserVisitor.cpp */; }; + 27C7D08A1CD10D1900F7E81F /* Vocabulary.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C7D0851CD10D1900F7E81F /* Vocabulary.h */; }; + 27C7D08B1CD10D1900F7E81F /* Vocabulary.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C7D0851CD10D1900F7E81F /* Vocabulary.h */; }; + 27C7D08C1CD10D1900F7E81F /* VocabularyImpl.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C7D0861CD10D1900F7E81F /* VocabularyImpl.cpp */; }; + 27C7D08D1CD10D1900F7E81F /* VocabularyImpl.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 27C7D0861CD10D1900F7E81F /* VocabularyImpl.cpp */; }; + 27C7D08E1CD10D1900F7E81F /* VocabularyImpl.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C7D0871CD10D1900F7E81F /* VocabularyImpl.h */; }; + 27C7D08F1CD10D1900F7E81F /* VocabularyImpl.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C7D0871CD10D1900F7E81F /* VocabularyImpl.h */; }; 37F1356D1B4AC02800E0CACF /* antlrcpp_Tests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 37F1356C1B4AC02800E0CACF /* antlrcpp_Tests.mm */; }; 37F1356E1B4AC02800E0CACF /* antlrcpp.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 37D727AA1867AF1E007B6D10 /* antlrcpp.dylib */; }; /* End PBXBuildFile section */ @@ -570,6 +620,10 @@ 2747A7091CA691310030247B /* ConfigLookup.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ConfigLookup.h; sourceTree = ""; wrapsLines = 0; }; 2747A7121CA6C46C0030247B /* InputHandlingTests.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = InputHandlingTests.mm; sourceTree = ""; wrapsLines = 0; }; 274FC6D81CA96B6C008D4374 /* MiscClassTests.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = MiscClassTests.mm; sourceTree = ""; wrapsLines = 0; }; + 275411DC1CCE704300CAD5B0 /* LexerActionExecutor.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LexerActionExecutor.cpp; sourceTree = ""; wrapsLines = 0; }; + 275411DD1CCE704300CAD5B0 /* LexerActionExecutor.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = LexerActionExecutor.h; sourceTree = ""; wrapsLines = 0; }; + 275411E21CCE733800CAD5B0 /* LexerIndexedCustomAction.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LexerIndexedCustomAction.cpp; sourceTree = ""; wrapsLines = 0; }; + 275411E31CCE733800CAD5B0 /* LexerIndexedCustomAction.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = LexerIndexedCustomAction.h; sourceTree = ""; }; 275DB3D41CCD23C000D8C543 /* LexerChannelAction.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LexerChannelAction.cpp; sourceTree = ""; }; 275DB3D51CCD23C000D8C543 /* LexerChannelAction.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = LexerChannelAction.h; sourceTree = ""; }; 275DB3D61CCD23C000D8C543 /* LexerCustomAction.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LexerCustomAction.cpp; sourceTree = ""; }; @@ -595,6 +649,24 @@ 278A66FA1C95838E002D667E /* ANTLRErrorListener.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ANTLRErrorListener.cpp; path = ../../runtime/ANTLRErrorListener.cpp; sourceTree = SOURCE_ROOT; }; 27A23EA11CC2A8D60036D8A3 /* TLexer.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = TLexer.cpp; path = ../generated/TLexer.cpp; sourceTree = ""; wrapsLines = 0; }; 27A23EA21CC2A8D60036D8A3 /* TLexer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = TLexer.h; path = ../generated/TLexer.h; sourceTree = ""; }; + 27C62E241CD269C90088721B /* ParseInfo.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ParseInfo.cpp; sourceTree = ""; }; + 27C62E251CD269C90088721B /* ParseInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ParseInfo.h; sourceTree = ""; }; + 27C62E2A1CD26C780088721B /* ProfilingATNSimulator.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ProfilingATNSimulator.cpp; sourceTree = ""; wrapsLines = 0; }; + 27C62E2B1CD26C780088721B /* ProfilingATNSimulator.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ProfilingATNSimulator.h; sourceTree = ""; wrapsLines = 0; }; + 27C62E301CD26EEA0088721B /* DecisionInfo.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = DecisionInfo.cpp; sourceTree = ""; wrapsLines = 0; }; + 27C62E311CD26EEA0088721B /* DecisionInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = DecisionInfo.h; sourceTree = ""; }; + 27C62E361CD271C60088721B /* ErrorInfo.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ErrorInfo.cpp; sourceTree = ""; wrapsLines = 0; }; + 27C62E371CD271C60088721B /* ErrorInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ErrorInfo.h; sourceTree = ""; wrapsLines = 0; }; + 27C62E3C1CD272480088721B /* DecisionEventInfo.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = DecisionEventInfo.cpp; sourceTree = ""; wrapsLines = 0; }; + 27C62E3D1CD272480088721B /* DecisionEventInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = DecisionEventInfo.h; sourceTree = ""; wrapsLines = 0; }; + 27C62E421CD273DE0088721B /* LookaheadEventInfo.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LookaheadEventInfo.cpp; sourceTree = ""; wrapsLines = 0; }; + 27C62E431CD273DE0088721B /* LookaheadEventInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = LookaheadEventInfo.h; sourceTree = ""; wrapsLines = 0; }; + 27C62E481CD274CA0088721B /* AmbiguityInfo.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = AmbiguityInfo.cpp; sourceTree = ""; wrapsLines = 0; }; + 27C62E491CD274CA0088721B /* AmbiguityInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AmbiguityInfo.h; sourceTree = ""; wrapsLines = 0; }; + 27C62E4E1CD275C50088721B /* ContextSensitivityInfo.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ContextSensitivityInfo.cpp; sourceTree = ""; wrapsLines = 0; }; + 27C62E4F1CD275C50088721B /* ContextSensitivityInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ContextSensitivityInfo.h; sourceTree = ""; wrapsLines = 0; }; + 27C62E541CD2768A0088721B /* PredicateEvalInfo.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = PredicateEvalInfo.cpp; sourceTree = ""; wrapsLines = 0; }; + 27C62E551CD2768A0088721B /* PredicateEvalInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = PredicateEvalInfo.h; sourceTree = ""; wrapsLines = 0; }; 27C6665D1C9584050021E494 /* ANTLRErrorListener.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ANTLRErrorListener.h; path = ../../runtime/ANTLRErrorListener.h; sourceTree = SOURCE_ROOT; wrapsLines = 0; }; 27C6665E1C9584050021E494 /* ANTLRErrorStrategy.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ANTLRErrorStrategy.cpp; path = ../../runtime/ANTLRErrorStrategy.cpp; sourceTree = SOURCE_ROOT; }; 27C6665F1C9584050021E494 /* ANTLRErrorStrategy.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ANTLRErrorStrategy.h; path = ../../runtime/ANTLRErrorStrategy.h; sourceTree = SOURCE_ROOT; }; @@ -617,7 +689,7 @@ 27C666701C9584050021E494 /* CommonTokenStream.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = CommonTokenStream.cpp; path = ../../runtime/CommonTokenStream.cpp; sourceTree = SOURCE_ROOT; wrapsLines = 0; }; 27C666711C9584050021E494 /* CommonTokenStream.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = CommonTokenStream.h; path = ../../runtime/CommonTokenStream.h; sourceTree = SOURCE_ROOT; }; 27C666721C9584050021E494 /* ConsoleErrorListener.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ConsoleErrorListener.cpp; path = ../../runtime/ConsoleErrorListener.cpp; sourceTree = SOURCE_ROOT; }; - 27C666731C9584050021E494 /* ConsoleErrorListener.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ConsoleErrorListener.h; path = ../../runtime/ConsoleErrorListener.h; sourceTree = SOURCE_ROOT; }; + 27C666731C9584050021E494 /* ConsoleErrorListener.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ConsoleErrorListener.h; path = ../../runtime/ConsoleErrorListener.h; sourceTree = SOURCE_ROOT; wrapsLines = 0; }; 27C666741C9584050021E494 /* DefaultErrorStrategy.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = DefaultErrorStrategy.cpp; path = ../../runtime/DefaultErrorStrategy.cpp; sourceTree = SOURCE_ROOT; wrapsLines = 0; }; 27C666751C9584050021E494 /* DefaultErrorStrategy.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = DefaultErrorStrategy.h; path = ../../runtime/DefaultErrorStrategy.h; sourceTree = SOURCE_ROOT; }; 27C666761C9584050021E494 /* DiagnosticErrorListener.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = DiagnosticErrorListener.cpp; path = ../../runtime/DiagnosticErrorListener.cpp; sourceTree = SOURCE_ROOT; wrapsLines = 0; }; @@ -654,7 +726,7 @@ 27C666961C9584050021E494 /* RecognitionException.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = RecognitionException.cpp; path = ../../runtime/RecognitionException.cpp; sourceTree = SOURCE_ROOT; wrapsLines = 0; }; 27C666971C9584050021E494 /* RecognitionException.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = RecognitionException.h; path = ../../runtime/RecognitionException.h; sourceTree = SOURCE_ROOT; wrapsLines = 0; }; 27C666981C9584050021E494 /* Recognizer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Recognizer.h; path = ../../runtime/Recognizer.h; sourceTree = SOURCE_ROOT; }; - 27C666991C9584050021E494 /* Recognizer.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = Recognizer.cpp; path = ../../runtime/Recognizer.cpp; sourceTree = SOURCE_ROOT; }; + 27C666991C9584050021E494 /* Recognizer.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = Recognizer.cpp; path = ../../runtime/Recognizer.cpp; sourceTree = SOURCE_ROOT; wrapsLines = 0; }; 27C6669A1C9584050021E494 /* RuleContext.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = RuleContext.cpp; path = ../../runtime/RuleContext.cpp; sourceTree = SOURCE_ROOT; wrapsLines = 0; }; 27C6669B1C9584050021E494 /* RuleContext.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = RuleContext.h; path = ../../runtime/RuleContext.h; sourceTree = SOURCE_ROOT; }; 27C6669C1C9584050021E494 /* Token.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = Token.cpp; path = ../../runtime/Token.cpp; sourceTree = SOURCE_ROOT; wrapsLines = 0; }; @@ -714,7 +786,7 @@ 27C6676D1C95846E0021E494 /* EpsilonTransition.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = EpsilonTransition.cpp; sourceTree = ""; wrapsLines = 0; }; 27C6676E1C95846E0021E494 /* EpsilonTransition.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = EpsilonTransition.h; sourceTree = ""; }; 27C6676F1C95846E0021E494 /* LexerATNConfig.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LexerATNConfig.cpp; sourceTree = ""; wrapsLines = 0; }; - 27C667701C95846E0021E494 /* LexerATNConfig.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = LexerATNConfig.h; sourceTree = ""; }; + 27C667701C95846E0021E494 /* LexerATNConfig.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = LexerATNConfig.h; sourceTree = ""; wrapsLines = 0; }; 27C667711C95846E0021E494 /* LexerATNSimulator.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LexerATNSimulator.cpp; sourceTree = ""; wrapsLines = 0; }; 27C667721C95846E0021E494 /* LexerATNSimulator.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = LexerATNSimulator.h; sourceTree = ""; wrapsLines = 0; }; 27C667731C95846E0021E494 /* LL1Analyzer.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LL1Analyzer.cpp; sourceTree = ""; wrapsLines = 0; }; @@ -849,6 +921,9 @@ 27C6E17D1C972FFC0079AF06 /* TParserVisitor.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = TParserVisitor.cpp; path = ../generated/TParserVisitor.cpp; sourceTree = ""; }; 27C6E1851C97322F0079AF06 /* TParserBaseVisitor.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = TParserBaseVisitor.h; path = ../generated/TParserBaseVisitor.h; sourceTree = ""; wrapsLines = 0; }; 27C6E1861C97322F0079AF06 /* TParserVisitor.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = TParserVisitor.h; path = ../generated/TParserVisitor.h; sourceTree = ""; }; + 27C7D0851CD10D1900F7E81F /* Vocabulary.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Vocabulary.h; sourceTree = ""; }; + 27C7D0861CD10D1900F7E81F /* VocabularyImpl.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = VocabularyImpl.cpp; sourceTree = ""; wrapsLines = 0; }; + 27C7D0871CD10D1900F7E81F /* VocabularyImpl.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = VocabularyImpl.h; sourceTree = ""; wrapsLines = 0; }; 37C147171B4D5A04008EDDDB /* libantlrcpp_static.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libantlrcpp_static.a; sourceTree = BUILT_PRODUCTS_DIR; }; 37D727AA1867AF1E007B6D10 /* antlrcpp.dylib */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.dylib"; includeInIndex = 0; path = antlrcpp.dylib; sourceTree = BUILT_PRODUCTS_DIR; }; 37F135681B4AC02800E0CACF /* antlrcpp Tests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = "antlrcpp Tests.xctest"; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -907,6 +982,8 @@ 27C667481C95846E0021E494 /* AbstractPredicateTransition.h */, 27C667491C95846E0021E494 /* ActionTransition.cpp */, 27C6674A1C95846E0021E494 /* ActionTransition.h */, + 27C62E481CD274CA0088721B /* AmbiguityInfo.cpp */, + 27C62E491CD274CA0088721B /* AmbiguityInfo.h */, 27C6674B1C95846E0021E494 /* ArrayPredictionContext.cpp */, 27C6674C1C95846E0021E494 /* ArrayPredictionContext.h */, 27C6674D1C95846E0021E494 /* ATN.cpp */, @@ -939,14 +1016,24 @@ 27C667681C95846E0021E494 /* BlockStartState.h */, 2747A7081CA691310030247B /* ConfigLookup.cpp */, 2747A7091CA691310030247B /* ConfigLookup.h */, + 27C62E4E1CD275C50088721B /* ContextSensitivityInfo.cpp */, + 27C62E4F1CD275C50088721B /* ContextSensitivityInfo.h */, + 27C62E3C1CD272480088721B /* DecisionEventInfo.cpp */, + 27C62E3D1CD272480088721B /* DecisionEventInfo.h */, + 27C62E301CD26EEA0088721B /* DecisionInfo.cpp */, + 27C62E311CD26EEA0088721B /* DecisionInfo.h */, 27C667691C95846E0021E494 /* DecisionState.cpp */, 27C6676A1C95846E0021E494 /* DecisionState.h */, 27C6676B1C95846E0021E494 /* EmptyPredictionContext.cpp */, 27C6676C1C95846E0021E494 /* EmptyPredictionContext.h */, 27C6676D1C95846E0021E494 /* EpsilonTransition.cpp */, 27C6676E1C95846E0021E494 /* EpsilonTransition.h */, + 27C62E361CD271C60088721B /* ErrorInfo.cpp */, + 27C62E371CD271C60088721B /* ErrorInfo.h */, 275ECC4A1CCCD95B00E79E2A /* LexerAction.cpp */, 275ECC4B1CCCD95B00E79E2A /* LexerAction.h */, + 275411DC1CCE704300CAD5B0 /* LexerActionExecutor.cpp */, + 275411DD1CCE704300CAD5B0 /* LexerActionExecutor.h */, 275ECC501CCCD9FB00E79E2A /* LexerActionType.cpp */, 275ECC511CCCD9FB00E79E2A /* LexerActionType.h */, 27C6676F1C95846E0021E494 /* LexerATNConfig.cpp */, @@ -957,6 +1044,8 @@ 275DB3D51CCD23C000D8C543 /* LexerChannelAction.h */, 275DB3D61CCD23C000D8C543 /* LexerCustomAction.cpp */, 275DB3D71CCD23C000D8C543 /* LexerCustomAction.h */, + 275411E21CCE733800CAD5B0 /* LexerIndexedCustomAction.cpp */, + 275411E31CCE733800CAD5B0 /* LexerIndexedCustomAction.h */, 275DB3D81CCD23C000D8C543 /* LexerModeAction.cpp */, 275DB3D91CCD23C000D8C543 /* LexerModeAction.h */, 275DB3DA1CCD23C000D8C543 /* LexerMoreAction.cpp */, @@ -971,12 +1060,16 @@ 275DB3E31CCD23C000D8C543 /* LexerTypeAction.h */, 27C667731C95846E0021E494 /* LL1Analyzer.cpp */, 27C667741C95846E0021E494 /* LL1Analyzer.h */, + 27C62E421CD273DE0088721B /* LookaheadEventInfo.cpp */, + 27C62E431CD273DE0088721B /* LookaheadEventInfo.h */, 27C667751C95846E0021E494 /* LoopEndState.cpp */, 27C667761C95846E0021E494 /* LoopEndState.h */, 27C667781C95846E0021E494 /* NotSetTransition.cpp */, 27C667791C95846E0021E494 /* NotSetTransition.h */, 27C6677A1C95846E0021E494 /* OrderedATNConfigSet.cpp */, 27C6677B1C95846E0021E494 /* OrderedATNConfigSet.h */, + 27C62E241CD269C90088721B /* ParseInfo.cpp */, + 27C62E251CD269C90088721B /* ParseInfo.h */, 27C6677C1C95846E0021E494 /* ParserATNSimulator.cpp */, 27C6677D1C95846E0021E494 /* ParserATNSimulator.h */, 27C6677E1C95846E0021E494 /* PlusBlockStartState.cpp */, @@ -985,12 +1078,16 @@ 27C667811C95846E0021E494 /* PlusLoopbackState.h */, 27C667821C95846E0021E494 /* PrecedencePredicateTransition.cpp */, 27C667831C95846E0021E494 /* PrecedencePredicateTransition.h */, + 27C62E541CD2768A0088721B /* PredicateEvalInfo.cpp */, + 27C62E551CD2768A0088721B /* PredicateEvalInfo.h */, 27C667841C95846E0021E494 /* PredicateTransition.cpp */, 27C667851C95846E0021E494 /* PredicateTransition.h */, 27C667861C95846E0021E494 /* PredictionContext.cpp */, 27C667871C95846E0021E494 /* PredictionContext.h */, 27C6678A1C95846E0021E494 /* PredictionMode.cpp */, 27C6678B1C95846E0021E494 /* PredictionMode.h */, + 27C62E2A1CD26C780088721B /* ProfilingATNSimulator.cpp */, + 27C62E2B1CD26C780088721B /* ProfilingATNSimulator.h */, 27C6678C1C95846E0021E494 /* RangeTransition.cpp */, 27C6678D1C95846E0021E494 /* RangeTransition.h */, 27C6678E1C95846E0021E494 /* RuleStartState.cpp */, @@ -1223,8 +1320,8 @@ 27C666951C9584050021E494 /* ProxyErrorListener.h */, 27C666961C9584050021E494 /* RecognitionException.cpp */, 27C666971C9584050021E494 /* RecognitionException.h */, - 27C666981C9584050021E494 /* Recognizer.h */, 27C666991C9584050021E494 /* Recognizer.cpp */, + 27C666981C9584050021E494 /* Recognizer.h */, 27C6669A1C9584050021E494 /* RuleContext.cpp */, 27C6669B1C9584050021E494 /* RuleContext.h */, 27C6669C1C9584050021E494 /* Token.cpp */, @@ -1239,8 +1336,11 @@ 27C666A51C9584050021E494 /* TokenStreamRewriter.h */, 27C666A61C9584050021E494 /* UnbufferedCharStream.cpp */, 27C666A71C9584050021E494 /* UnbufferedCharStream.h */, - 27C666A81C9584050021E494 /* UnbufferedTokenStream.h */, 27C666A91C9584050021E494 /* UnbufferedTokenStream.cpp */, + 27C666A81C9584050021E494 /* UnbufferedTokenStream.h */, + 27C7D0851CD10D1900F7E81F /* Vocabulary.h */, + 27C7D0861CD10D1900F7E81F /* VocabularyImpl.cpp */, + 27C7D0871CD10D1900F7E81F /* VocabularyImpl.h */, 27C666AA1C9584050021E494 /* WritableToken.cpp */, 27C666AB1C9584050021E494 /* WritableToken.h */, ); @@ -1301,6 +1401,7 @@ 27C668431C95846E0021E494 /* SemanticContext.h in Headers */, 27C667B51C95846E0021E494 /* ATN.h in Headers */, 27C668471C95846E0021E494 /* SetTransition.h in Headers */, + 27C62E411CD272480088721B /* DecisionEventInfo.h in Headers */, 27C667DD1C95846E0021E494 /* BasicBlockStartState.h in Headers */, 27C667111C9584050021E494 /* Parser.h in Headers */, 27C6684B1C95846E0021E494 /* SingletonPredictionContext.h in Headers */, @@ -1319,6 +1420,7 @@ 27C667D11C95846E0021E494 /* ATNState.h in Headers */, 27C6672B1C9584050021E494 /* Token.h in Headers */, 27C667151C9584050021E494 /* ParserInterpreter.h in Headers */, + 27C62E531CD275C50088721B /* ContextSensitivityInfo.h in Headers */, 27C667C51C95846E0021E494 /* ATNDeserializer.h in Headers */, 27C66A1D1C958AB30021E494 /* TextChunk.h in Headers */, 27C667CD1C95846E0021E494 /* ATNSimulator.h in Headers */, @@ -1337,15 +1439,18 @@ 27C667371C9584050021E494 /* TokenStream.h in Headers */, 27C6682F1C95846E0021E494 /* PredictionMode.h in Headers */, 27C669A61C9585B80021E494 /* ParseTreeVisitor.h in Headers */, + 27C7D08B1CD10D1900F7E81F /* Vocabulary.h in Headers */, 27C668741C9584B60021E494 /* DFASerializer.h in Headers */, 27C669961C9585B80021E494 /* ErrorNodeImpl.h in Headers */, 27C667C91C95846E0021E494 /* ATNSerializer.h in Headers */, 27C66A191C958AB30021E494 /* RuleTagToken.h in Headers */, 27C667411C9584050021E494 /* UnbufferedTokenStream.h in Headers */, + 27C62E291CD269C90088721B /* ParseInfo.h in Headers */, 27C668131C95846E0021E494 /* ParserATNSimulator.h in Headers */, 27C667051C9584050021E494 /* LexerNoViableAltException.h in Headers */, 27C666B51C9584050021E494 /* ANTLRFileStream.h in Headers */, 27C667B11C95846E0021E494 /* ArrayPredictionContext.h in Headers */, + 27C62E591CD2768A0088721B /* PredicateEvalInfo.h in Headers */, 2747A70D1CA691310030247B /* ConfigLookup.h in Headers */, 27C669AE1C9585B80021E494 /* RuleNode.h in Headers */, 27C667211C9584050021E494 /* RecognitionException.h in Headers */, @@ -1364,14 +1469,17 @@ 27C666D51C9584050021E494 /* CommonTokenStream.h in Headers */, 27C6681F1C95846E0021E494 /* PrecedencePredicateTransition.h in Headers */, 27C669BE1C9585B80021E494 /* Tree.h in Headers */, + 27C62E351CD26EEA0088721B /* DecisionInfo.h in Headers */, 27C667091C9584050021E494 /* ListTokenSource.h in Headers */, 27C669B21C9585B80021E494 /* SyntaxTree.h in Headers */, + 27C62E2F1CD26C780088721B /* ProfilingATNSimulator.h in Headers */, 27C666C91C9584050021E494 /* CharStream.h in Headers */, 27C6671D1C9584050021E494 /* ProxyErrorListener.h in Headers */, 275ECC4F1CCCD95B00E79E2A /* LexerAction.h in Headers */, 27C666E51C9584050021E494 /* Exceptions.h in Headers */, 27C668B51C9584FA0021E494 /* Interval.h in Headers */, 27C667E11C95846E0021E494 /* BasicState.h in Headers */, + 275411E71CCE733800CAD5B0 /* LexerIndexedCustomAction.h in Headers */, 27C666B91C9584050021E494 /* ANTLRInputStream.h in Headers */, 27C6680F1C95846E0021E494 /* OrderedATNConfigSet.h in Headers */, 27C666B11C9584050021E494 /* ANTLRErrorStrategy.h in Headers */, @@ -1383,11 +1491,13 @@ 27C6683B1C95846E0021E494 /* RuleStopState.h in Headers */, 27C668781C9584B60021E494 /* DFAState.h in Headers */, 27C66A131C958AB30021E494 /* ParseTreeMatch.h in Headers */, + 27C62E4D1CD274CA0088721B /* AmbiguityInfo.h in Headers */, 27C66A151C958AB30021E494 /* ParseTreePattern.h in Headers */, 27C667ED1C95846E0021E494 /* DecisionState.h in Headers */, 27C667FD1C95846E0021E494 /* LexerATNSimulator.h in Headers */, 27C668571C95846E0021E494 /* StarLoopEntryState.h in Headers */, 275ECC551CCCD9FB00E79E2A /* LexerActionType.h in Headers */, + 27C62E3B1CD271C60088721B /* ErrorInfo.h in Headers */, 27C6683F1C95846E0021E494 /* RuleTransition.h in Headers */, 27C667231C9584050021E494 /* Recognizer.h in Headers */, 27C667F11C95846E0021E494 /* EmptyPredictionContext.h in Headers */, @@ -1411,11 +1521,14 @@ 27C667011C9584050021E494 /* LexerInterpreter.h in Headers */, 27C668701C9584B60021E494 /* DFA.h in Headers */, 27C666D11C9584050021E494 /* CommonTokenFactory.h in Headers */, + 27C62E471CD273DE0088721B /* LookaheadEventInfo.h in Headers */, 27C6680B1C95846E0021E494 /* NotSetTransition.h in Headers */, 27C667E91C95846E0021E494 /* BlockStartState.h in Headers */, 27C666F91C9584050021E494 /* IRecognizer.h in Headers */, 276927251C9ED49100E4EBF8 /* antlrcpp-Prefix.h in Headers */, 27C669A21C9585B80021E494 /* ParseTreeProperty.h in Headers */, + 27C7D08F1CD10D1900F7E81F /* VocabularyImpl.h in Headers */, + 275411E11CCE704300CAD5B0 /* LexerActionExecutor.h in Headers */, 27C667A91C95846E0021E494 /* AbstractPredicateTransition.h in Headers */, 27C669171C9585230021E494 /* guid.h in Headers */, 27C66A111C958AB30021E494 /* Chunk.h in Headers */, @@ -1439,10 +1552,13 @@ 27C667DC1C95846E0021E494 /* BasicBlockStartState.h in Headers */, 27C666F81C9584050021E494 /* IRecognizer.h in Headers */, 27C667E01C95846E0021E494 /* BasicState.h in Headers */, + 27C62E281CD269C90088721B /* ParseInfo.h in Headers */, 27C666C01C9584050021E494 /* BaseErrorListener.h in Headers */, + 27C62E2E1CD26C780088721B /* ProfilingATNSimulator.h in Headers */, 27C668421C95846E0021E494 /* SemanticContext.h in Headers */, 27C6682E1C95846E0021E494 /* PredictionMode.h in Headers */, 27C667261C9584050021E494 /* RuleContext.h in Headers */, + 27C62E4C1CD274CA0088721B /* AmbiguityInfo.h in Headers */, 27C6672A1C9584050021E494 /* Token.h in Headers */, 27C666B01C9584050021E494 /* ANTLRErrorStrategy.h in Headers */, 27C667201C9584050021E494 /* RecognitionException.h in Headers */, @@ -1450,6 +1566,7 @@ 27C6670C1C9584050021E494 /* NoViableAltException.h in Headers */, 27C6683A1C95846E0021E494 /* RuleStopState.h in Headers */, 27C666BC1C9584050021E494 /* BailErrorStrategy.h in Headers */, + 27C62E401CD272480088721B /* DecisionEventInfo.h in Headers */, 27C66A1C1C958AB30021E494 /* TextChunk.h in Headers */, 27C667EC1C95846E0021E494 /* DecisionState.h in Headers */, 27C667001C9584050021E494 /* LexerInterpreter.h in Headers */, @@ -1467,10 +1584,12 @@ 27C6684E1C95846E0021E494 /* StarBlockStartState.h in Headers */, 27C668221C95846E0021E494 /* PredicateTransition.h in Headers */, 27C6698D1C9585B80021E494 /* AbstractParseTreeVisitor.h in Headers */, + 275411E61CCE733800CAD5B0 /* LexerIndexedCustomAction.h in Headers */, 27C668621C95846E0021E494 /* WildcardTransition.h in Headers */, 27C66A181C958AB30021E494 /* RuleTagToken.h in Headers */, 27C668B81C9584FA0021E494 /* IntervalSet.h in Headers */, 27C668261C95846E0021E494 /* PredictionContext.h in Headers */, + 275411E01CCE704300CAD5B0 /* LexerActionExecutor.h in Headers */, 27C669B11C9585B80021E494 /* SyntaxTree.h in Headers */, 275DB3F31CCD23C000D8C543 /* LexerTypeAction.h in Headers */, 27C666D01C9584050021E494 /* CommonTokenFactory.h in Headers */, @@ -1495,16 +1614,19 @@ 27C6687B1C9584B60021E494 /* LexerDFASerializer.h in Headers */, 27C668001C95846E0021E494 /* LL1Analyzer.h in Headers */, 27C667141C9584050021E494 /* ParserInterpreter.h in Headers */, + 27C7D08E1CD10D1900F7E81F /* VocabularyImpl.h in Headers */, 27C6673A1C9584050021E494 /* TokenStreamRewriter.h in Headers */, 27C6684A1C95846E0021E494 /* SingletonPredictionContext.h in Headers */, 27C667041C9584050021E494 /* LexerNoViableAltException.h in Headers */, 27C669121C9585230021E494 /* Strings.h in Headers */, 275DB3E51CCD23C000D8C543 /* LexerChannelAction.h in Headers */, 27C667E41C95846E0021E494 /* BlockEndState.h in Headers */, + 27C7D08A1CD10D1900F7E81F /* Vocabulary.h in Headers */, 27C666B41C9584050021E494 /* ANTLRFileStream.h in Headers */, 27C6685E1C95846E0021E494 /* Transition.h in Headers */, 275ECC541CCCD9FB00E79E2A /* LexerActionType.h in Headers */, 27C666CC1C9584050021E494 /* CommonToken.h in Headers */, + 27C62E581CD2768A0088721B /* PredicateEvalInfo.h in Headers */, 27C6699D1C9585B80021E494 /* ParseTreeListener.h in Headers */, 27C666F61C9584050021E494 /* IntStream.h in Headers */, 27C669951C9585B80021E494 /* ErrorNodeImpl.h in Headers */, @@ -1531,9 +1653,11 @@ 27C666C81C9584050021E494 /* CharStream.h in Headers */, 27C668041C95846E0021E494 /* LoopEndState.h in Headers */, 27C6685A1C95846E0021E494 /* TokensStartState.h in Headers */, + 27C62E3A1CD271C60088721B /* ErrorInfo.h in Headers */, 27C668561C95846E0021E494 /* StarLoopEntryState.h in Headers */, 27C667B41C95846E0021E494 /* ATN.h in Headers */, 275DB3ED1CCD23C000D8C543 /* LexerPopModeAction.h in Headers */, + 27C62E521CD275C50088721B /* ContextSensitivityInfo.h in Headers */, 27C669AD1C9585B80021E494 /* RuleNode.h in Headers */, 27C668771C9584B60021E494 /* DFAState.h in Headers */, 27C6680A1C95846E0021E494 /* NotSetTransition.h in Headers */, @@ -1546,6 +1670,7 @@ 27C669BD1C9585B80021E494 /* Tree.h in Headers */, 27C668161C95846E0021E494 /* PlusBlockStartState.h in Headers */, 27C667D01C95846E0021E494 /* ATNState.h in Headers */, + 27C62E461CD273DE0088721B /* LookaheadEventInfo.h in Headers */, 27C667FC1C95846E0021E494 /* LexerATNSimulator.h in Headers */, 27C669A11C9585B80021E494 /* ParseTreeProperty.h in Headers */, 27C667F01C95846E0021E494 /* EmptyPredictionContext.h in Headers */, @@ -1565,6 +1690,7 @@ 27C667B81C95846E0021E494 /* ATNConfig.h in Headers */, 27C667B01C95846E0021E494 /* ArrayPredictionContext.h in Headers */, 27C669021C9585230021E494 /* BitSet.h in Headers */, + 27C62E341CD26EEA0088721B /* DecisionInfo.h in Headers */, 27C66A1E1C958AB30021E494 /* TokenTagToken.h in Headers */, ); runOnlyForDeploymentPostprocessing = 0; @@ -1732,12 +1858,14 @@ 27C668B71C9584FA0021E494 /* IntervalSet.cpp in Sources */, 27C668251C95846E0021E494 /* PredictionContext.cpp in Sources */, 27C668491C95846E0021E494 /* SingletonPredictionContext.cpp in Sources */, + 27C62E2D1CD26C780088721B /* ProfilingATNSimulator.cpp in Sources */, 27C667BF1C95846E0021E494 /* ATNDeserializationOptions.cpp in Sources */, 27C667CF1C95846E0021E494 /* ATNState.cpp in Sources */, 27C667AF1C95846E0021E494 /* ArrayPredictionContext.cpp in Sources */, 27C669981C9585B80021E494 /* ParseTree.cpp in Sources */, 27C6698C1C9585B80021E494 /* AbstractParseTreeVisitor.cpp in Sources */, 27C669C01C9585B80021E494 /* Trees.cpp in Sources */, + 27C62E271CD269C90088721B /* ParseInfo.cpp in Sources */, 278A66FC1C95838E002D667E /* ANTLRErrorListener.cpp in Sources */, 27C668391C95846E0021E494 /* RuleStopState.cpp in Sources */, 27C669AC1C9585B80021E494 /* RuleNode.cpp in Sources */, @@ -1746,6 +1874,7 @@ 27C668551C95846E0021E494 /* StarLoopEntryState.cpp in Sources */, 27C668761C9584B60021E494 /* DFAState.cpp in Sources */, 27C667B71C95846E0021E494 /* ATNConfig.cpp in Sources */, + 27C62E391CD271C60088721B /* ErrorInfo.cpp in Sources */, 27C668611C95846E0021E494 /* WildcardTransition.cpp in Sources */, 27C667DF1C95846E0021E494 /* BasicState.cpp in Sources */, 27C6670B1C9584050021E494 /* NoViableAltException.cpp in Sources */, @@ -1760,10 +1889,13 @@ 27C668B31C9584FA0021E494 /* Interval.cpp in Sources */, 27C6682D1C95846E0021E494 /* PredictionMode.cpp in Sources */, 27C666D31C9584050021E494 /* CommonTokenStream.cpp in Sources */, + 27C62E3F1CD272480088721B /* DecisionEventInfo.cpp in Sources */, 27C667311C9584050021E494 /* TokenSource.cpp in Sources */, 27C668CB1C9584FA0021E494 /* MurmurHash.cpp in Sources */, 27C669A81C9585B80021E494 /* ParseTreeWalker.cpp in Sources */, + 27C62E511CD275C50088721B /* ContextSensitivityInfo.cpp in Sources */, 27C666ED1C9584050021E494 /* InputMismatchException.cpp in Sources */, + 275411E51CCE733800CAD5B0 /* LexerIndexedCustomAction.cpp in Sources */, 27C668031C95846E0021E494 /* LoopEndState.cpp in Sources */, 27C666B31C9584050021E494 /* ANTLRFileStream.cpp in Sources */, 27C669111C9585230021E494 /* Strings.cpp in Sources */, @@ -1779,11 +1911,14 @@ 27C669151C9585230021E494 /* guid.cpp in Sources */, 27C669941C9585B80021E494 /* ErrorNodeImpl.cpp in Sources */, 27C6687A1C9584B60021E494 /* LexerDFASerializer.cpp in Sources */, + 27C7D08D1CD10D1900F7E81F /* VocabularyImpl.cpp in Sources */, 27C6685D1C95846E0021E494 /* Transition.cpp in Sources */, + 27C62E4B1CD274CA0088721B /* AmbiguityInfo.cpp in Sources */, 27C667351C9584050021E494 /* TokenStream.cpp in Sources */, 27C666F51C9584050021E494 /* IntStream.cpp in Sources */, 27C669B01C9585B80021E494 /* SyntaxTree.cpp in Sources */, 27C667071C9584050021E494 /* ListTokenSource.cpp in Sources */, + 27C62E331CD26EEA0088721B /* DecisionInfo.cpp in Sources */, 27C6681D1C95846E0021E494 /* PrecedencePredicateTransition.cpp in Sources */, 27C66A0F1C958AB30021E494 /* TokenTagToken.cpp in Sources */, 276901AA1CAD7E67005CEC6A /* Recognizer.cpp in Sources */, @@ -1801,6 +1936,7 @@ 27C666D71C9584050021E494 /* ConsoleErrorListener.cpp in Sources */, 27C666FB1C9584050021E494 /* Lexer.cpp in Sources */, 27C668351C95846E0021E494 /* RuleStartState.cpp in Sources */, + 27C62E451CD273DE0088721B /* LookaheadEventInfo.cpp in Sources */, 27C66A0B1C958AB30021E494 /* TagChunk.cpp in Sources */, 272415E51CB54C09007E056A /* UnbufferedTokenStream.cpp in Sources */, 27C666FF1C9584050021E494 /* LexerInterpreter.cpp in Sources */, @@ -1808,6 +1944,7 @@ 27C6673D1C9584050021E494 /* UnbufferedCharStream.cpp in Sources */, 27C667D31C95846E0021E494 /* ATNType.cpp in Sources */, 27C669B41C9585B80021E494 /* TerminalNode.cpp in Sources */, + 27C62E571CD2768A0088721B /* PredicateEvalInfo.cpp in Sources */, 27C666C31C9584050021E494 /* BufferedTokenStream.cpp in Sources */, 27C666B71C9584050021E494 /* ANTLRInputStream.cpp in Sources */, 27C668211C95846E0021E494 /* PredicateTransition.cpp in Sources */, @@ -1833,6 +1970,7 @@ 27C66A0D1C958AB30021E494 /* TextChunk.cpp in Sources */, 27C669BC1C9585B80021E494 /* Tree.cpp in Sources */, 27C6670F1C9584050021E494 /* Parser.cpp in Sources */, + 275411DF1CCE704300CAD5B0 /* LexerActionExecutor.cpp in Sources */, 27C669A01C9585B80021E494 /* ParseTreeProperty.cpp in Sources */, 27C668451C95846E0021E494 /* SetTransition.cpp in Sources */, 27C667BB1C95846E0021E494 /* ATNConfigSet.cpp in Sources */, @@ -1858,6 +1996,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 27C62E261CD269C90088721B /* ParseInfo.cpp in Sources */, 27C668241C95846E0021E494 /* PredictionContext.cpp in Sources */, 27C666E61C9584050021E494 /* FailedPredicateException.cpp in Sources */, 275DB3EE1CCD23C000D8C543 /* LexerPushModeAction.cpp in Sources */, @@ -1873,17 +2012,22 @@ 27C6670A1C9584050021E494 /* NoViableAltException.cpp in Sources */, 27C667161C9584050021E494 /* ParserRuleContext.cpp in Sources */, 27C669BF1C9585B80021E494 /* Trees.cpp in Sources */, + 27C62E381CD271C60088721B /* ErrorInfo.cpp in Sources */, 27C666B61C9584050021E494 /* ANTLRInputStream.cpp in Sources */, + 27C62E561CD2768A0088721B /* PredicateEvalInfo.cpp in Sources */, 27C666F41C9584050021E494 /* IntStream.cpp in Sources */, 27C667D61C95846E0021E494 /* AtomTransition.cpp in Sources */, 27C667E21C95846E0021E494 /* BlockEndState.cpp in Sources */, + 27C62E4A1CD274CA0088721B /* AmbiguityInfo.cpp in Sources */, 27C669BB1C9585B80021E494 /* Tree.cpp in Sources */, 27C6673C1C9584050021E494 /* UnbufferedCharStream.cpp in Sources */, 27C6698B1C9585B80021E494 /* AbstractParseTreeVisitor.cpp in Sources */, + 27C62E321CD26EEA0088721B /* DecisionInfo.cpp in Sources */, 27C666D61C9584050021E494 /* ConsoleErrorListener.cpp in Sources */, 27C668381C95846E0021E494 /* RuleStopState.cpp in Sources */, 27C6680C1C95846E0021E494 /* OrderedATNConfigSet.cpp in Sources */, 275ECC4C1CCCD95B00E79E2A /* LexerAction.cpp in Sources */, + 27C62E441CD273DE0088721B /* LookaheadEventInfo.cpp in Sources */, 275DB3F21CCD23C000D8C543 /* LexerTypeAction.cpp in Sources */, 27C667BA1C95846E0021E494 /* ATNConfigSet.cpp in Sources */, 275DB3EC1CCD23C000D8C543 /* LexerPopModeAction.cpp in Sources */, @@ -1923,6 +2067,7 @@ 27C669AF1C9585B80021E494 /* SyntaxTree.cpp in Sources */, 27C667DA1C95846E0021E494 /* BasicBlockStartState.cpp in Sources */, 27C666AE1C9584050021E494 /* ANTLRErrorStrategy.cpp in Sources */, + 275411DE1CCE704300CAD5B0 /* LexerActionExecutor.cpp in Sources */, 27C667EA1C95846E0021E494 /* DecisionState.cpp in Sources */, 27C666FE1C9584050021E494 /* LexerInterpreter.cpp in Sources */, 27C666CE1C9584050021E494 /* CommonTokenFactory.cpp in Sources */, @@ -1938,6 +2083,8 @@ 27C667AE1C95846E0021E494 /* ArrayPredictionContext.cpp in Sources */, 27C669041C9585230021E494 /* CPPUtils.cpp in Sources */, 272415E41CB54C08007E056A /* UnbufferedTokenStream.cpp in Sources */, + 275411E41CCE733800CAD5B0 /* LexerIndexedCustomAction.cpp in Sources */, + 27C62E3E1CD272480088721B /* DecisionEventInfo.cpp in Sources */, 27C667021C9584050021E494 /* LexerNoViableAltException.cpp in Sources */, 27C667EE1C95846E0021E494 /* EmptyPredictionContext.cpp in Sources */, 27C666C61C9584050021E494 /* CharStream.cpp in Sources */, @@ -1949,7 +2096,9 @@ 27C667E61C95846E0021E494 /* BlockStartState.cpp in Sources */, 27C668DE1C9584FA0021E494 /* TestRig.cpp in Sources */, 27C666D21C9584050021E494 /* CommonTokenStream.cpp in Sources */, + 27C62E501CD275C50088721B /* ContextSensitivityInfo.cpp in Sources */, 27C669A71C9585B80021E494 /* ParseTreeWalker.cpp in Sources */, + 27C62E2C1CD26C780088721B /* ProfilingATNSimulator.cpp in Sources */, 27C669B71C9585B80021E494 /* TerminalNodeImpl.cpp in Sources */, 27C667C21C95846E0021E494 /* ATNDeserializer.cpp in Sources */, 27C668301C95846E0021E494 /* RangeTransition.cpp in Sources */, @@ -1960,6 +2109,7 @@ 27C666DA1C9584050021E494 /* DefaultErrorStrategy.cpp in Sources */, 27C667FE1C95846E0021E494 /* LL1Analyzer.cpp in Sources */, 27C6699B1C9585B80021E494 /* ParseTreeListener.cpp in Sources */, + 27C7D08C1CD10D1900F7E81F /* VocabularyImpl.cpp in Sources */, 27C66A081C958AB30021E494 /* RuleTagToken.cpp in Sources */, 27C667061C9584050021E494 /* ListTokenSource.cpp in Sources */, 27C6699F1C9585B80021E494 /* ParseTreeProperty.cpp in Sources */, diff --git a/runtime/Cpp/runtime/ANTLRErrorListener.h b/runtime/Cpp/runtime/ANTLRErrorListener.h index f3cbe3a2a..2888fc946 100755 --- a/runtime/Cpp/runtime/ANTLRErrorListener.h +++ b/runtime/Cpp/runtime/ANTLRErrorListener.h @@ -77,7 +77,7 @@ namespace runtime { /// the reporting of an error. It is null in the case where /// the parser was able to recover in line without exiting the /// surrounding rule. - virtual void syntaxError(IRecognizer *recognizer, Token::Ref offendingSymbol, size_t line, int charPositionInLine, + virtual void syntaxError(IRecognizer *recognizer, Ref offendingSymbol, size_t line, int charPositionInLine, const std::wstring &msg, std::exception_ptr e) = 0; /// @@ -108,7 +108,7 @@ namespace runtime { /// the ATN configuration set where the ambiguity was /// determined virtual void reportAmbiguity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, bool exact, - const antlrcpp::BitSet &ambigAlts, std::shared_ptr configs) = 0; + const antlrcpp::BitSet &ambigAlts, Ref configs) = 0; /// /// This method is called when an SLL conflict occurs and the parser is about @@ -131,7 +131,7 @@ namespace runtime { /// the ATN configuration set where the SLL conflict was /// detected virtual void reportAttemptingFullContext(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, - const antlrcpp::BitSet &conflictingAlts, std::shared_ptr configs) = 0; + const antlrcpp::BitSet &conflictingAlts, Ref configs) = 0; /// /// This method is called by the parser when a full-context prediction has a @@ -161,7 +161,7 @@ namespace runtime { /// the ATN configuration set where the unambiguous prediction /// was determined virtual void reportContextSensitivity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, - int prediction, std::shared_ptr configs) = 0; + int prediction, Ref configs) = 0; }; } // namespace runtime diff --git a/runtime/Cpp/runtime/ANTLRErrorStrategy.h b/runtime/Cpp/runtime/ANTLRErrorStrategy.h index 4c1c736da..6d1e9f52c 100755 --- a/runtime/Cpp/runtime/ANTLRErrorStrategy.h +++ b/runtime/Cpp/runtime/ANTLRErrorStrategy.h @@ -79,7 +79,7 @@ namespace runtime { /// the parser instance /// if the error strategy was not able to /// recover from the unexpected input symbol - virtual Token::Ref recoverInline(Parser *recognizer) = 0; + virtual Ref recoverInline(Parser *recognizer) = 0; /// /// This method is called to recover from exception {@code e}. This method is diff --git a/runtime/Cpp/runtime/BailErrorStrategy.cpp b/runtime/Cpp/runtime/BailErrorStrategy.cpp index f64b7acb2..32e313e99 100755 --- a/runtime/Cpp/runtime/BailErrorStrategy.cpp +++ b/runtime/Cpp/runtime/BailErrorStrategy.cpp @@ -40,7 +40,7 @@ using namespace org::antlr::v4::runtime; void BailErrorStrategy::recover(Parser *recognizer, const RecognitionException &e) { std::exception_ptr exception = std::make_exception_ptr(e); - ParserRuleContext::Ref context = recognizer->getContext(); + Ref context = recognizer->getContext(); do { context->exception = exception; if (context->getParent().expired()) @@ -60,11 +60,11 @@ void BailErrorStrategy::recover(Parser *recognizer, const RecognitionException & #endif } -Token::Ref BailErrorStrategy::recoverInline(Parser *recognizer) { +Ref BailErrorStrategy::recoverInline(Parser *recognizer) { InputMismatchException e(recognizer); std::exception_ptr exception = std::make_exception_ptr(e); - ParserRuleContext::Ref context = recognizer->getContext(); + Ref context = recognizer->getContext(); do { context->exception = exception; if (context->getParent().expired()) diff --git a/runtime/Cpp/runtime/BailErrorStrategy.h b/runtime/Cpp/runtime/BailErrorStrategy.h index 5ee47aad9..120fa81cf 100755 --- a/runtime/Cpp/runtime/BailErrorStrategy.h +++ b/runtime/Cpp/runtime/BailErrorStrategy.h @@ -53,7 +53,7 @@ namespace runtime { /// Make sure we don't attempt to recover inline; if the parser /// successfully recovers, it won't throw an exception. /// - virtual Token::Ref recoverInline(Parser *recognizer) override; + virtual Ref recoverInline(Parser *recognizer) override; /// /// Make sure we don't attempt to recover from problems in subrules. diff --git a/runtime/Cpp/runtime/BaseErrorListener.cpp b/runtime/Cpp/runtime/BaseErrorListener.cpp index bc08bc641..1ee1322d8 100755 --- a/runtime/Cpp/runtime/BaseErrorListener.cpp +++ b/runtime/Cpp/runtime/BaseErrorListener.cpp @@ -34,18 +34,18 @@ using namespace org::antlr::v4::runtime; -void BaseErrorListener::syntaxError(IRecognizer *recognizer, Token::Ref offendingSymbol, size_t line, int charPositionInLine, +void BaseErrorListener::syntaxError(IRecognizer *recognizer, Ref offendingSymbol, size_t line, int charPositionInLine, const std::wstring &msg, std::exception_ptr e) { } void BaseErrorListener::reportAmbiguity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, - bool exact, const antlrcpp::BitSet &ambigAlts, std::shared_ptr configs) { + bool exact, const antlrcpp::BitSet &ambigAlts, Ref configs) { } void BaseErrorListener::reportAttemptingFullContext(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, - size_t stopIndex, const antlrcpp::BitSet &conflictingAlts, std::shared_ptr configs) { + size_t stopIndex, const antlrcpp::BitSet &conflictingAlts, Ref configs) { } void BaseErrorListener::reportContextSensitivity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, - int prediction, std::shared_ptr configs) { + int prediction, Ref configs) { } diff --git a/runtime/Cpp/runtime/BaseErrorListener.h b/runtime/Cpp/runtime/BaseErrorListener.h index 2c5e80cbd..f08639033 100755 --- a/runtime/Cpp/runtime/BaseErrorListener.h +++ b/runtime/Cpp/runtime/BaseErrorListener.h @@ -44,17 +44,17 @@ namespace runtime { class BaseErrorListener : public ANTLRErrorListener { - virtual void syntaxError(IRecognizer *recognizer, Token::Ref offendingSymbol, size_t line, int charPositionInLine, + virtual void syntaxError(IRecognizer *recognizer, Ref offendingSymbol, size_t line, int charPositionInLine, const std::wstring &msg, std::exception_ptr e) override; virtual void reportAmbiguity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, bool exact, - const antlrcpp::BitSet &ambigAlts, std::shared_ptr configs) override; + const antlrcpp::BitSet &ambigAlts, Ref configs) override; virtual void reportAttemptingFullContext(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, - const antlrcpp::BitSet &conflictingAlts, std::shared_ptr configs) override; + const antlrcpp::BitSet &conflictingAlts, Ref configs) override; virtual void reportContextSensitivity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, - int prediction, std::shared_ptr configs) override; + int prediction, Ref configs) override; }; } // namespace runtime diff --git a/runtime/Cpp/runtime/BufferedTokenStream.cpp b/runtime/Cpp/runtime/BufferedTokenStream.cpp index c74831be1..f6af6608c 100755 --- a/runtime/Cpp/runtime/BufferedTokenStream.cpp +++ b/runtime/Cpp/runtime/BufferedTokenStream.cpp @@ -103,7 +103,7 @@ size_t BufferedTokenStream::fetch(size_t n) { } for (size_t i = 0; i < n; i++) { - Token::Ref t = _tokenSource->nextToken(); + Ref t = _tokenSource->nextToken(); if (is(t)) { (std::dynamic_pointer_cast(t))->setTokenIndex((int)_tokens.size()); } @@ -117,7 +117,7 @@ size_t BufferedTokenStream::fetch(size_t n) { return n; } -Token::Ref BufferedTokenStream::get(size_t i) const { +Ref BufferedTokenStream::get(size_t i) const { if (i >= _tokens.size()) { throw IndexOutOfBoundsException(std::string("token index ") + std::to_string(i) + @@ -127,8 +127,8 @@ Token::Ref BufferedTokenStream::get(size_t i) const { return _tokens[i]; } -std::vector BufferedTokenStream::get(size_t start, size_t stop) { - std::vector subset; +std::vector> BufferedTokenStream::get(size_t start, size_t stop) { + std::vector> subset; lazyInit(); @@ -140,7 +140,7 @@ std::vector BufferedTokenStream::get(size_t start, size_t stop) { stop = _tokens.size() - 1; } for (size_t i = start; i <= stop; i++) { - Token::Ref t = _tokens[i]; + Ref t = _tokens[i]; if (t->getType() == EOF) { break; } @@ -153,14 +153,14 @@ ssize_t BufferedTokenStream::LA(ssize_t i) { return LT(i)->getType(); } -Token::Ref BufferedTokenStream::LB(size_t k) { +Ref BufferedTokenStream::LB(size_t k) { if (k > _p) { return nullptr; } return _tokens[(size_t)(_p - k)]; } -Token::Ref BufferedTokenStream::LT(ssize_t k) { +Ref BufferedTokenStream::LT(ssize_t k) { lazyInit(); if (k == 0) { return nullptr; @@ -201,15 +201,15 @@ void BufferedTokenStream::setTokenSource(TokenSource *tokenSource) { _needSetup = true; } -std::vector BufferedTokenStream::getTokens() { +std::vector> BufferedTokenStream::getTokens() { return _tokens; } -std::vector BufferedTokenStream::getTokens(int start, int stop) { +std::vector> BufferedTokenStream::getTokens(int start, int stop) { return getTokens(start, stop, std::vector()); } -std::vector BufferedTokenStream::getTokens(int start, int stop, const std::vector &types) { +std::vector> BufferedTokenStream::getTokens(int start, int stop, const std::vector &types) { lazyInit(); if (start < 0 || stop >= (int)_tokens.size() || stop < 0 || start >= (int)_tokens.size()) { throw IndexOutOfBoundsException(std::string("start ") + @@ -220,7 +220,7 @@ std::vector BufferedTokenStream::getTokens(int start, int stop, cons std::to_string(_tokens.size() - 1)); } - std::vector filteredTokens; + std::vector> filteredTokens; if (start > stop) { return filteredTokens; @@ -228,7 +228,7 @@ std::vector BufferedTokenStream::getTokens(int start, int stop, cons // list = tokens[start:stop]:{T t, t.getType() in types} for (size_t i = (size_t)start; i <= (size_t)stop; i++) { - Token::Ref tok = _tokens[i]; + Ref tok = _tokens[i]; if (types.empty() || std::find(types.begin(), types.end(), tok->getType()) != types.end()) { filteredTokens.push_back(tok); @@ -237,7 +237,7 @@ std::vector BufferedTokenStream::getTokens(int start, int stop, cons return filteredTokens; } -std::vector BufferedTokenStream::getTokens(int start, int stop, int ttype) { +std::vector> BufferedTokenStream::getTokens(int start, int stop, int ttype) { std::vector s; s.push_back(ttype); return getTokens(start, stop, s); @@ -249,7 +249,7 @@ ssize_t BufferedTokenStream::nextTokenOnChannel(size_t i, int channel) { return -1; } - Token::Ref token = _tokens[i]; + Ref token = _tokens[i]; while (token->getChannel() != channel) { if (token->getType() == EOF) { return -1; @@ -272,7 +272,7 @@ ssize_t BufferedTokenStream::previousTokenOnChannel(ssize_t i, int channel) cons return -1; } -std::vector BufferedTokenStream::getHiddenTokensToRight(size_t tokenIndex, int channel) { +std::vector> BufferedTokenStream::getHiddenTokensToRight(size_t tokenIndex, int channel) { lazyInit(); if (tokenIndex >= _tokens.size()) { throw IndexOutOfBoundsException(std::to_string(tokenIndex) + " not in 0.." + std::to_string(_tokens.size() - 1)); @@ -291,11 +291,11 @@ std::vector BufferedTokenStream::getHiddenTokensToRight(size_t token return filterForChannel(from, (size_t)to, channel); } -std::vector BufferedTokenStream::getHiddenTokensToRight(size_t tokenIndex) { +std::vector> BufferedTokenStream::getHiddenTokensToRight(size_t tokenIndex) { return getHiddenTokensToRight(tokenIndex, -1); } -std::vector BufferedTokenStream::getHiddenTokensToLeft(size_t tokenIndex, int channel) { +std::vector> BufferedTokenStream::getHiddenTokensToLeft(size_t tokenIndex, int channel) { lazyInit(); if (tokenIndex >= _tokens.size()) { throw IndexOutOfBoundsException(std::to_string(tokenIndex) + " not in 0.." + std::to_string(_tokens.size() - 1)); @@ -303,7 +303,7 @@ std::vector BufferedTokenStream::getHiddenTokensToLeft(size_t tokenI ssize_t prevOnChannel = previousTokenOnChannel((ssize_t)tokenIndex - 1, Lexer::DEFAULT_TOKEN_CHANNEL); if (prevOnChannel == (ssize_t)tokenIndex - 1) { - return std::vector(); + return { }; } // if none onchannel to left, prevOnChannel=-1 then from=0 size_t from = (size_t)(prevOnChannel + 1); @@ -312,14 +312,14 @@ std::vector BufferedTokenStream::getHiddenTokensToLeft(size_t tokenI return filterForChannel(from, to, channel); } -std::vector BufferedTokenStream::getHiddenTokensToLeft(size_t tokenIndex) { +std::vector> BufferedTokenStream::getHiddenTokensToLeft(size_t tokenIndex) { return getHiddenTokensToLeft(tokenIndex, -1); } -std::vector BufferedTokenStream::filterForChannel(size_t from, size_t to, int channel) { - std::vector hidden; +std::vector> BufferedTokenStream::filterForChannel(size_t from, size_t to, int channel) { + std::vector> hidden; for (size_t i = from; i <= to; i++) { - Token::Ref t = _tokens[i]; + Ref t = _tokens[i]; if (channel == -1) { if (t->getChannel() != Lexer::DEFAULT_TOKEN_CHANNEL) { hidden.push_back(t); @@ -361,7 +361,7 @@ std::wstring BufferedTokenStream::getText(const misc::Interval &interval) { std::wstringstream ss; for (size_t i = (size_t)start; i <= (size_t)stop; i++) { - Token::Ref t = _tokens[i]; + Ref t = _tokens[i]; if (t->getType() == EOF) { break; } @@ -374,7 +374,7 @@ std::wstring BufferedTokenStream::getText(RuleContext *ctx) { return getText(ctx->getSourceInterval()); } -std::wstring BufferedTokenStream::getText(Token::Ref start, Token::Ref stop) { +std::wstring BufferedTokenStream::getText(Ref start, Ref stop) { if (start != nullptr && stop != nullptr) { return getText(misc::Interval(start->getTokenIndex(), stop->getTokenIndex())); } diff --git a/runtime/Cpp/runtime/BufferedTokenStream.h b/runtime/Cpp/runtime/BufferedTokenStream.h index 0917110be..09c12b04f 100755 --- a/runtime/Cpp/runtime/BufferedTokenStream.h +++ b/runtime/Cpp/runtime/BufferedTokenStream.h @@ -63,57 +63,57 @@ namespace runtime { virtual size_t size() override; virtual void consume() override; - virtual Token::Ref get(size_t i) const override; + virtual Ref get(size_t i) const override; /// Get all tokens from start..stop inclusively. - virtual std::vector get(size_t start, size_t stop); + virtual std::vector> get(size_t start, size_t stop); virtual ssize_t LA(ssize_t i) override; - virtual Token::Ref LT(ssize_t k) override; + virtual Ref LT(ssize_t k) override; /// Reset this token stream by setting its token source. virtual void setTokenSource(TokenSource *tokenSource); - virtual std::vector getTokens(); - virtual std::vector getTokens(int start, int stop); + virtual std::vector> getTokens(); + virtual std::vector> getTokens(int start, int stop); /// /// Given a start and stop index, return a List of all tokens in /// the token type BitSet. Return null if no tokens were found. This /// method looks at both on and off channel tokens. /// - virtual std::vector getTokens(int start, int stop, const std::vector &types); - virtual std::vector getTokens(int start, int stop, int ttype); + virtual std::vector> getTokens(int start, int stop, const std::vector &types); + virtual std::vector> getTokens(int start, int stop, int ttype); /// Collect all tokens on specified channel to the right of /// the current token up until we see a token on DEFAULT_TOKEN_CHANNEL or /// EOF. If channel is -1, find any non default channel token. - virtual std::vector getHiddenTokensToRight(size_t tokenIndex, int channel); + virtual std::vector> getHiddenTokensToRight(size_t tokenIndex, int channel); /// /// Collect all hidden tokens (any off-default channel) to the right of /// the current token up until we see a token on DEFAULT_TOKEN_CHANNEL /// of EOF. /// - virtual std::vector getHiddenTokensToRight(size_t tokenIndex); + virtual std::vector> getHiddenTokensToRight(size_t tokenIndex); /// /// Collect all tokens on specified channel to the left of /// the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. /// If channel is -1, find any non default channel token. /// - virtual std::vector getHiddenTokensToLeft(size_t tokenIndex, int channel); + virtual std::vector> getHiddenTokensToLeft(size_t tokenIndex, int channel); /// /// Collect all hidden tokens (any off-default channel) to the left of /// the current token up until we see a token on DEFAULT_TOKEN_CHANNEL. /// - virtual std::vector getHiddenTokensToLeft(size_t tokenIndex); + virtual std::vector> getHiddenTokensToLeft(size_t tokenIndex); virtual std::string getSourceName() const override; virtual std::wstring getText() override; virtual std::wstring getText(const misc::Interval &interval) override; virtual std::wstring getText(RuleContext *ctx) override; - virtual std::wstring getText(Token::Ref start, Token::Ref stop) override; + virtual std::wstring getText(Ref start, Ref stop) override; /// /// Get all tokens from lexer until EOF @@ -126,7 +126,7 @@ namespace runtime { /// chunks of it later. This list captures everything so we can access /// complete input text. // ml: we own the tokens produced by the token factory. - std::vector _tokens; + std::vector> _tokens; /// /// The index into of the current token (next token to @@ -160,7 +160,7 @@ namespace runtime { /// The actual number of elements added to the buffer. virtual size_t fetch(size_t n); - virtual Token::Ref LB(size_t k); + virtual Ref LB(size_t k); /// Allowed derived classes to modify the behavior of operations which change /// the current stream position by adjusting the target token index of a seek @@ -187,7 +187,7 @@ namespace runtime { /// on channel between i and 0. virtual ssize_t previousTokenOnChannel(ssize_t i, int channel) const; - virtual std::vector filterForChannel(size_t from, size_t to, int channel); + virtual std::vector> filterForChannel(size_t from, size_t to, int channel); private: bool _needSetup; void InitializeInstanceFields(); diff --git a/runtime/Cpp/runtime/CommonTokenFactory.cpp b/runtime/Cpp/runtime/CommonTokenFactory.cpp index 85826fd80..d61f51618 100755 --- a/runtime/Cpp/runtime/CommonTokenFactory.cpp +++ b/runtime/Cpp/runtime/CommonTokenFactory.cpp @@ -37,7 +37,7 @@ using namespace org::antlr::v4::runtime; -const std::shared_ptr> CommonTokenFactory::DEFAULT = std::make_shared(); +const Ref> CommonTokenFactory::DEFAULT = std::make_shared(); CommonTokenFactory::CommonTokenFactory(bool copyText) : copyText(copyText) { } @@ -45,10 +45,10 @@ CommonTokenFactory::CommonTokenFactory(bool copyText) : copyText(copyText) { CommonTokenFactory::CommonTokenFactory() : CommonTokenFactory(false) { } -std::shared_ptr CommonTokenFactory::create(std::pair source, int type, +Ref CommonTokenFactory::create(std::pair source, int type, const std::wstring &text, int channel, int start, int stop, int line, int charPositionInLine) { - std::shared_ptr t = std::make_shared(source, type, channel, start, stop); + Ref t = std::make_shared(source, type, channel, start, stop); t->setLine(line); t->setCharPositionInLine(charPositionInLine); if (text != L"") { @@ -60,6 +60,6 @@ std::shared_ptr CommonTokenFactory::create(std::pair CommonTokenFactory::create(int type, const std::wstring &text) { +Ref CommonTokenFactory::create(int type, const std::wstring &text) { return std::make_shared(type, text); } diff --git a/runtime/Cpp/runtime/CommonTokenFactory.h b/runtime/Cpp/runtime/CommonTokenFactory.h index 2b0ef6025..a8e846f01 100755 --- a/runtime/Cpp/runtime/CommonTokenFactory.h +++ b/runtime/Cpp/runtime/CommonTokenFactory.h @@ -40,7 +40,7 @@ namespace runtime { class CommonTokenFactory : public TokenFactory { public: - static const std::shared_ptr> DEFAULT; + static const Ref> DEFAULT; /// /// Copy text for token out of input char stream. Useful when input @@ -56,10 +56,10 @@ namespace runtime { CommonTokenFactory(); - virtual std::shared_ptr create(std::pair source, int type, + virtual Ref create(std::pair source, int type, const std::wstring &text, int channel, int start, int stop, int line, int charPositionInLine) override; - virtual std::shared_ptr create(int type, const std::wstring &text) override; + virtual Ref create(int type, const std::wstring &text) override; }; } // namespace runtime diff --git a/runtime/Cpp/runtime/CommonTokenStream.cpp b/runtime/Cpp/runtime/CommonTokenStream.cpp index c963d4362..d38ef3f45 100755 --- a/runtime/Cpp/runtime/CommonTokenStream.cpp +++ b/runtime/Cpp/runtime/CommonTokenStream.cpp @@ -47,9 +47,9 @@ ssize_t CommonTokenStream::adjustSeekIndex(size_t i) { return nextTokenOnChannel(i, channel); } -Token::Ref CommonTokenStream::LB(size_t k) { +Ref CommonTokenStream::LB(size_t k) { if (k == 0 || k > _p) { - return Token::Ref(); + return Ref(); } ssize_t i = (ssize_t)_p; @@ -67,7 +67,7 @@ Token::Ref CommonTokenStream::LB(size_t k) { return _tokens[i]; } -Token::Ref CommonTokenStream::LT(ssize_t k) { +Ref CommonTokenStream::LT(ssize_t k) { lazyInit(); if (k == 0) { return nullptr; @@ -93,7 +93,7 @@ int CommonTokenStream::getNumberOfOnChannelTokens() { int n = 0; fill(); for (size_t i = 0; i < _tokens.size(); i++) { - Token::Ref t = _tokens[i]; + Ref t = _tokens[i]; if (t->getChannel() == channel) { n++; } diff --git a/runtime/Cpp/runtime/CommonTokenStream.h b/runtime/Cpp/runtime/CommonTokenStream.h index 7095745b9..57b1fa797 100755 --- a/runtime/Cpp/runtime/CommonTokenStream.h +++ b/runtime/Cpp/runtime/CommonTokenStream.h @@ -68,10 +68,10 @@ namespace runtime { protected: virtual ssize_t adjustSeekIndex(size_t i) override; - virtual Token::Ref LB(size_t k) override; + virtual Ref LB(size_t k) override; public: - virtual Token::Ref LT(ssize_t k) override; + virtual Ref LT(ssize_t k) override; /// Count EOF just once. virtual int getNumberOfOnChannelTokens(); diff --git a/runtime/Cpp/runtime/ConsoleErrorListener.cpp b/runtime/Cpp/runtime/ConsoleErrorListener.cpp index 702beb8b2..9503b3a11 100755 --- a/runtime/Cpp/runtime/ConsoleErrorListener.cpp +++ b/runtime/Cpp/runtime/ConsoleErrorListener.cpp @@ -35,7 +35,7 @@ using namespace org::antlr::v4::runtime; ConsoleErrorListener ConsoleErrorListener::INSTANCE; -void ConsoleErrorListener::syntaxError(IRecognizer *recognizer, Token::Ref offendingSymbol, +void ConsoleErrorListener::syntaxError(IRecognizer *recognizer, Ref offendingSymbol, size_t line, int charPositionInLine, const std::wstring &msg, std::exception_ptr e) { std::wcerr << L"line " << line << L":" << charPositionInLine << L" " << msg << std::endl; } diff --git a/runtime/Cpp/runtime/ConsoleErrorListener.h b/runtime/Cpp/runtime/ConsoleErrorListener.h index c3f530c8c..04ce65198 100755 --- a/runtime/Cpp/runtime/ConsoleErrorListener.h +++ b/runtime/Cpp/runtime/ConsoleErrorListener.h @@ -42,7 +42,8 @@ namespace runtime { public: static ConsoleErrorListener INSTANCE; - virtual void syntaxError(IRecognizer *recognizer, Token::Ref offendingSymbol, size_t line, int charPositionInLine, const std::wstring &msg, std::exception_ptr e) override; + virtual void syntaxError(IRecognizer *recognizer, Ref offendingSymbol, size_t line, int charPositionInLine, + const std::wstring &msg, std::exception_ptr e) override; }; } // namespace runtime diff --git a/runtime/Cpp/runtime/DefaultErrorStrategy.cpp b/runtime/Cpp/runtime/DefaultErrorStrategy.cpp index b93139fd3..62ad45a6d 100755 --- a/runtime/Cpp/runtime/DefaultErrorStrategy.cpp +++ b/runtime/Cpp/runtime/DefaultErrorStrategy.cpp @@ -191,7 +191,7 @@ void DefaultErrorStrategy::reportUnwantedToken(Parser *recognizer) { beginErrorCondition(recognizer); - Token::Ref t = recognizer->getCurrentToken(); + Ref t = recognizer->getCurrentToken(); std::wstring tokenName = getTokenErrorDisplay(t); misc::IntervalSet expecting = getExpectedTokens(recognizer); @@ -206,16 +206,16 @@ void DefaultErrorStrategy::reportMissingToken(Parser *recognizer) { beginErrorCondition(recognizer); - Token::Ref t = recognizer->getCurrentToken(); + Ref t = recognizer->getCurrentToken(); misc::IntervalSet expecting = getExpectedTokens(recognizer); std::wstring msg = std::wstring(L"missing ") + expecting.toString(recognizer->getTokenNames()) + std::wstring(L" at ") + getTokenErrorDisplay(t); recognizer->notifyErrorListeners(t, msg, nullptr); } -Token::Ref DefaultErrorStrategy::recoverInline(Parser *recognizer) { +Ref DefaultErrorStrategy::recoverInline(Parser *recognizer) { // SINGLE TOKEN DELETION - Token::Ref matchedSymbol = singleTokenDeletion(recognizer); + Ref matchedSymbol = singleTokenDeletion(recognizer); if (matchedSymbol) { // we have deleted the extra token. // now, move past ttype token as if all were ok @@ -249,22 +249,22 @@ bool DefaultErrorStrategy::singleTokenInsertion(Parser *recognizer) { return false; } -Token::Ref DefaultErrorStrategy::singleTokenDeletion(Parser *recognizer) { +Ref DefaultErrorStrategy::singleTokenDeletion(Parser *recognizer) { ssize_t nextTokenType = recognizer->getInputStream()->LA(2); misc::IntervalSet expecting = getExpectedTokens(recognizer); if (expecting.contains((int)nextTokenType)) { reportUnwantedToken(recognizer); recognizer->consume(); // simply delete extra token // we want to return the token we're actually matching - Token::Ref matchedSymbol = recognizer->getCurrentToken(); + Ref matchedSymbol = recognizer->getCurrentToken(); reportMatch(recognizer); // we know current token is correct return matchedSymbol; } return nullptr; } -Token::Ref DefaultErrorStrategy::getMissingSymbol(Parser *recognizer) { - Token::Ref currentSymbol = recognizer->getCurrentToken(); +Ref DefaultErrorStrategy::getMissingSymbol(Parser *recognizer) { + Ref currentSymbol = recognizer->getCurrentToken(); misc::IntervalSet expecting = getExpectedTokens(recognizer); ssize_t expectedTokenType = expecting.getMinElement(); // get any element std::wstring tokenText; @@ -273,8 +273,8 @@ Token::Ref DefaultErrorStrategy::getMissingSymbol(Parser *recognizer) { } else { tokenText = std::wstring(L"getTokenNames()[(size_t)expectedTokenType] + std::wstring(L">"); } - Token::Ref current = currentSymbol; - Token::Ref lookback = recognizer->getTokenStream()->LT(-1); + Ref current = currentSymbol; + Ref lookback = recognizer->getTokenStream()->LT(-1); if (current->getType() == EOF && lookback != nullptr) { current = lookback; } @@ -287,9 +287,9 @@ misc::IntervalSet DefaultErrorStrategy::getExpectedTokens(Parser *recognizer) { return recognizer->getExpectedTokens(); } -std::wstring DefaultErrorStrategy::getTokenErrorDisplay(Token::Ref t) { +std::wstring DefaultErrorStrategy::getTokenErrorDisplay(Ref t) { if (t == nullptr) { - return L""; + return L""; } std::wstring s = getSymbolText(t); if (s == L"") { @@ -302,11 +302,11 @@ std::wstring DefaultErrorStrategy::getTokenErrorDisplay(Token::Ref t) { return escapeWSAndQuote(s); } -std::wstring DefaultErrorStrategy::getSymbolText(Token::Ref symbol) { +std::wstring DefaultErrorStrategy::getSymbolText(Ref symbol) { return symbol->getText(); } -int DefaultErrorStrategy::getSymbolType(Token::Ref symbol) { +int DefaultErrorStrategy::getSymbolType(Ref symbol) { return symbol->getType(); } @@ -320,7 +320,7 @@ std::wstring DefaultErrorStrategy::escapeWSAndQuote(std::wstring &s) { misc::IntervalSet DefaultErrorStrategy::getErrorRecoverySet(Parser *recognizer) { const atn::ATN &atn = recognizer->getInterpreter()->atn; - RuleContext::Ref ctx = recognizer->getContext(); + Ref ctx = recognizer->getContext(); misc::IntervalSet recoverSet; while (ctx->invokingState >= 0) { // compute what follows who invoked us diff --git a/runtime/Cpp/runtime/DefaultErrorStrategy.h b/runtime/Cpp/runtime/DefaultErrorStrategy.h index 398ff984a..66aa1bfaf 100755 --- a/runtime/Cpp/runtime/DefaultErrorStrategy.h +++ b/runtime/Cpp/runtime/DefaultErrorStrategy.h @@ -305,7 +305,7 @@ namespace runtime { /// in rule {@code atom}. It can assume that you forgot the {@code ')'}. /// public: - virtual Token::Ref recoverInline(Parser *recognizer) override; + virtual Ref recoverInline(Parser *recognizer) override; /// /// This method implements the single-token insertion inline error recovery @@ -344,7 +344,7 @@ namespace runtime { /// the successfully matched instance if single-token /// deletion successfully recovers from the mismatched input, otherwise /// {@code null} - virtual Token::Ref singleTokenDeletion(Parser *recognizer); + virtual Ref singleTokenDeletion(Parser *recognizer); /// /// Conjure up a missing token during error recovery. @@ -366,7 +366,7 @@ namespace runtime { /// If you change what tokens must be created by the lexer, /// override this method to create the appropriate tokens. /// - virtual Token::Ref getMissingSymbol(Parser *recognizer); + virtual Ref getMissingSymbol(Parser *recognizer); virtual misc::IntervalSet getExpectedTokens(Parser *recognizer); @@ -379,11 +379,11 @@ namespace runtime { /// your token objects because you don't have to go modify your lexer /// so that it creates a new class. /// - virtual std::wstring getTokenErrorDisplay(Token::Ref t); + virtual std::wstring getTokenErrorDisplay(Ref t); - virtual std::wstring getSymbolText(Token::Ref symbol); + virtual std::wstring getSymbolText(Ref symbol); - virtual int getSymbolType(Token::Ref symbol); + virtual int getSymbolType(Ref symbol); virtual std::wstring escapeWSAndQuote(std::wstring &s); diff --git a/runtime/Cpp/runtime/DiagnosticErrorListener.cpp b/runtime/Cpp/runtime/DiagnosticErrorListener.cpp index 380d49b02..98f250e72 100755 --- a/runtime/Cpp/runtime/DiagnosticErrorListener.cpp +++ b/runtime/Cpp/runtime/DiagnosticErrorListener.cpp @@ -47,7 +47,7 @@ DiagnosticErrorListener::DiagnosticErrorListener(bool exactOnly) : exactOnly(exa } void DiagnosticErrorListener::reportAmbiguity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, - bool exact, const antlrcpp::BitSet &ambigAlts, std::shared_ptr configs) { + bool exact, const antlrcpp::BitSet &ambigAlts, Ref configs) { if (exactOnly && !exact) { return; } @@ -62,7 +62,7 @@ void DiagnosticErrorListener::reportAmbiguity(Parser *recognizer, const dfa::DFA } void DiagnosticErrorListener::reportAttemptingFullContext(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, - size_t stopIndex, const antlrcpp::BitSet &conflictingAlts, std::shared_ptr configs) { + size_t stopIndex, const antlrcpp::BitSet &conflictingAlts, Ref configs) { std::wstring decision = getDecisionDescription(recognizer, dfa); std::wstring text = recognizer->getTokenStream()->getText(misc::Interval((int)startIndex, (int)stopIndex)); std::wstring message = L"reportAttemptingFullContext d = " + decision + L", input = '" + text + L"'"; @@ -70,7 +70,7 @@ void DiagnosticErrorListener::reportAttemptingFullContext(Parser *recognizer, co } void DiagnosticErrorListener::reportContextSensitivity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, - size_t stopIndex, int prediction, std::shared_ptr configs) { + size_t stopIndex, int prediction, Ref configs) { std::wstring decision = getDecisionDescription(recognizer, dfa); std::wstring text = recognizer->getTokenStream()->getText(misc::Interval((int)startIndex, (int)stopIndex)); std::wstring message = L"reportContextSensitivity d = " + decision + L", input = '" + text + L"'"; @@ -95,7 +95,7 @@ std::wstring DiagnosticErrorListener::getDecisionDescription(Parser *recognizer, } antlrcpp::BitSet DiagnosticErrorListener::getConflictingAlts(const antlrcpp::BitSet &reportedAlts, - std::shared_ptr configs) { + Ref configs) { if (reportedAlts.count() > 0) { // Not exactly like the original Java code, but this listener is only used // in the TestRig (where it never provides a good alt set), so it's probably ok so. return reportedAlts; @@ -103,7 +103,7 @@ antlrcpp::BitSet DiagnosticErrorListener::getConflictingAlts(const antlrcpp::Bit antlrcpp::BitSet result; for (size_t i = 0; i < configs->size(); i++) { - atn::ATNConfig::Ref config = configs->get(i); + Ref config = configs->get(i); result.set((size_t)config->alt); } diff --git a/runtime/Cpp/runtime/DiagnosticErrorListener.h b/runtime/Cpp/runtime/DiagnosticErrorListener.h index eee646d00..042b790a9 100755 --- a/runtime/Cpp/runtime/DiagnosticErrorListener.h +++ b/runtime/Cpp/runtime/DiagnosticErrorListener.h @@ -82,13 +82,13 @@ namespace runtime { DiagnosticErrorListener(bool exactOnly); virtual void reportAmbiguity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, bool exact, - const antlrcpp::BitSet &ambigAlts, std::shared_ptr configs) override; + const antlrcpp::BitSet &ambigAlts, Ref configs) override; virtual void reportAttemptingFullContext(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, - const antlrcpp::BitSet &conflictingAlts, std::shared_ptr configs) override; + const antlrcpp::BitSet &conflictingAlts, Ref configs) override; virtual void reportContextSensitivity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, - int prediction, std::shared_ptr configs) override; + int prediction, Ref configs) override; protected: virtual std::wstring getDecisionDescription(Parser *recognizer, const dfa::DFA &dfa); @@ -103,7 +103,7 @@ namespace runtime { /// The conflicting or ambiguous configuration set. /// Returns {@code reportedAlts} if it is not {@code null}, otherwise /// returns the set of alternatives represented in {@code configs}. - virtual antlrcpp::BitSet getConflictingAlts(const antlrcpp::BitSet &reportedAlts, std::shared_ptr configs); + virtual antlrcpp::BitSet getConflictingAlts(const antlrcpp::BitSet &reportedAlts, Ref configs); }; } // namespace runtime diff --git a/runtime/Cpp/runtime/Lexer.cpp b/runtime/Cpp/runtime/Lexer.cpp index f325413fc..3298294ed 100755 --- a/runtime/Cpp/runtime/Lexer.cpp +++ b/runtime/Cpp/runtime/Lexer.cpp @@ -66,7 +66,7 @@ void Lexer::reset() { getInterpreter()->reset(); } -Token::Ref Lexer::nextToken() { +Ref Lexer::nextToken() { // Mark start location in char stream so unbuffered streams are // guaranteed at least have text of current token ssize_t tokenStartMarker = _input->mark(); @@ -150,7 +150,7 @@ int Lexer::popMode() { } -std::shared_ptr> Lexer::getTokenFactory() { +Ref> Lexer::getTokenFactory() { return _factory; } @@ -167,27 +167,22 @@ CharStream* Lexer::getInputStream() { return _input; } -void Lexer::emit(Token::Ref token) { +void Lexer::emit(Ref token) { _token = token; } -Token::Ref Lexer::emit() { - Token::Ref t = std::dynamic_pointer_cast(_factory->create({ this, _input }, _type, _text, _channel, +Ref Lexer::emit() { + Ref t = std::dynamic_pointer_cast(_factory->create({ this, _input }, _type, _text, _channel, _tokenStartCharIndex, getCharIndex() - 1, _tokenStartLine, _tokenStartCharPositionInLine)); emit(t); return t; } -Token::Ref Lexer::emitEOF() { +Ref Lexer::emitEOF() { int cpos = getCharPositionInLine(); - // The character position for EOF is one beyond the position of - // the previous token's last character - if (_token != nullptr) { - int n = _token->getStopIndex() - _token->getStartIndex() + 1; - cpos = _token->getCharPositionInLine() + n; - } - Token::Ref eof = std::dynamic_pointer_cast(_factory->create({ this, _input }, EOF, L"", Token::DEFAULT_CHANNEL, - (int)_input->index(), (int)_input->index() - 1, (int)getLine(), cpos)); + size_t line = getLine(); + Ref eof = std::dynamic_pointer_cast(_factory->create({ this, _input }, EOF, L"", Token::DEFAULT_CHANNEL, + (int)_input->index(), (int)_input->index() - 1, (int)line, cpos)); emit(eof); return eof; } @@ -223,11 +218,11 @@ void Lexer::setText(const std::wstring &text) { this->_text = text; } -Token::Ref Lexer::getToken() { +Ref Lexer::getToken() { return _token; } -void Lexer::setToken(Token::Ref token) { +void Lexer::setToken(Ref token) { _token = token; } @@ -247,9 +242,9 @@ int Lexer::getChannel() { return _channel; } -std::vector Lexer::getAllTokens() { - std::vector tokens; - Token::Ref t = nextToken(); +std::vector> Lexer::getAllTokens() { + std::vector> tokens; + Ref t = nextToken(); while (t->getType() != EOF) { tokens.push_back(t); t = nextToken(); diff --git a/runtime/Cpp/runtime/Lexer.h b/runtime/Cpp/runtime/Lexer.h index e35e50c8e..c0d3ee203 100755 --- a/runtime/Cpp/runtime/Lexer.h +++ b/runtime/Cpp/runtime/Lexer.h @@ -47,8 +47,6 @@ namespace runtime { /// of speed. class Lexer : public Recognizer, public TokenSource { public: - typedef std::shared_ptr Ref; - static const int DEFAULT_MODE = 0; static const int MORE = -2; static const int SKIP = -3; @@ -62,7 +60,7 @@ namespace runtime { protected: /// /// How to create token objects - std::shared_ptr> _factory; + Ref> _factory; public: /// The goal of all lexer rules/methods is to create a token object. @@ -72,7 +70,7 @@ namespace runtime { /// emissions, then set this to the last token to be matched or /// something nonnull so that the auto token emit mechanism will not /// emit another token. - Token::Ref _token; + Ref _token; /// /// What character index in the stream did the current token start at? @@ -116,7 +114,7 @@ namespace runtime { virtual void reset(); /// Return a token from this source; i.e., match a token on the char stream. - virtual Token::Ref nextToken() override; + virtual Ref nextToken() override; /// Instruct the lexer to skip creating a token for current lexer rule /// and look for another token. nextToken() knows to keep looking when @@ -124,13 +122,9 @@ namespace runtime { /// if token == null at end of any token rule, it creates one for you /// and emits it. virtual void skip(); - virtual void more(); - virtual void mode(int m); - virtual void pushMode(int m); - virtual int popMode(); template @@ -138,7 +132,7 @@ namespace runtime { this->_factory = factory; } - virtual std::shared_ptr> getTokenFactory() override; + virtual Ref> getTokenFactory() override; /// /// Set the char stream and reset the lexer @@ -154,7 +148,7 @@ namespace runtime { /// and getToken (to push tokens into a list and pull from that list /// rather than a single variable as this implementation does). /// - virtual void emit(Token::Ref token); + virtual void emit(Ref token); /// /// The standard method called to automatically emit a token at the @@ -163,9 +157,9 @@ namespace runtime { /// use that to set the token's text. Override this method to emit /// custom Token objects or provide a new factory. /// - virtual Token::Ref emit(); + virtual Ref emit(); - virtual Token::Ref emitEOF(); + virtual Ref emitEOF(); virtual size_t getLine() const override; @@ -193,9 +187,9 @@ namespace runtime { /// /// Override if emitting multiple tokens. - virtual Token::Ref getToken(); + virtual Ref getToken(); - virtual void setToken(Token::Ref token); + virtual void setToken(Ref token); virtual void setType(int ttype); @@ -209,7 +203,7 @@ namespace runtime { /// Return a list of all Token objects in input char stream. /// Forces load of all tokens. Does not include EOF token. - virtual std::vector getAllTokens(); + virtual std::vector> getAllTokens(); virtual void recover(const LexerNoViableAltException &e); diff --git a/runtime/Cpp/runtime/LexerInterpreter.h b/runtime/Cpp/runtime/LexerInterpreter.h index aac8420e0..d4f167ff5 100755 --- a/runtime/Cpp/runtime/LexerInterpreter.h +++ b/runtime/Cpp/runtime/LexerInterpreter.h @@ -60,7 +60,7 @@ namespace runtime { const std::vector &_modeNames; std::vector _decisionToDFA; - std::shared_ptr _sharedContextCache; + Ref _sharedContextCache; }; } // namespace runtime diff --git a/runtime/Cpp/runtime/LexerNoViableAltException.cpp b/runtime/Cpp/runtime/LexerNoViableAltException.cpp index 02c9b50a5..1896e7993 100755 --- a/runtime/Cpp/runtime/LexerNoViableAltException.cpp +++ b/runtime/Cpp/runtime/LexerNoViableAltException.cpp @@ -39,7 +39,7 @@ using namespace org::antlr::v4::runtime; LexerNoViableAltException::LexerNoViableAltException(Lexer *lexer, CharStream *input, size_t startIndex, - std::shared_ptr deadEndConfigs) + Ref deadEndConfigs) : RecognitionException(lexer, input, nullptr, nullptr), _startIndex(startIndex), _deadEndConfigs(deadEndConfigs) { } @@ -47,7 +47,7 @@ size_t LexerNoViableAltException::getStartIndex() { return _startIndex; } -std::shared_ptr LexerNoViableAltException::getDeadEndConfigs() { +Ref LexerNoViableAltException::getDeadEndConfigs() { return _deadEndConfigs; } diff --git a/runtime/Cpp/runtime/LexerNoViableAltException.h b/runtime/Cpp/runtime/LexerNoViableAltException.h index ea1a4fa0e..6941e372b 100755 --- a/runtime/Cpp/runtime/LexerNoViableAltException.h +++ b/runtime/Cpp/runtime/LexerNoViableAltException.h @@ -42,10 +42,10 @@ namespace runtime { class LexerNoViableAltException : public RecognitionException { public: LexerNoViableAltException(Lexer *lexer, CharStream *input, size_t startIndex, - std::shared_ptr deadEndConfigs); + Ref deadEndConfigs); virtual size_t getStartIndex(); - virtual std::shared_ptr getDeadEndConfigs(); + virtual Ref getDeadEndConfigs(); virtual std::wstring toString(); private: @@ -53,7 +53,7 @@ namespace runtime { const size_t _startIndex; /// Which configurations did we try at input.index() that couldn't match input.LA(1)? - std::shared_ptr _deadEndConfigs; + Ref _deadEndConfigs; }; diff --git a/runtime/Cpp/runtime/ListTokenSource.cpp b/runtime/Cpp/runtime/ListTokenSource.cpp index 8dcb32560..a09358559 100755 --- a/runtime/Cpp/runtime/ListTokenSource.cpp +++ b/runtime/Cpp/runtime/ListTokenSource.cpp @@ -45,7 +45,7 @@ int ListTokenSource::getCharPositionInLine() { } else if (tokens.size() > 0) { // have to calculate the result from the line/column of the previous // token, along with the text of the token. - Token::Ref lastToken = tokens.back(); + Ref lastToken = tokens.back(); std::wstring tokenText = lastToken->getText(); if (tokenText != L"") { int lastNewLine = (int)tokenText.rfind(L'\n'); @@ -62,7 +62,7 @@ int ListTokenSource::getCharPositionInLine() { return 0; } -Token::Ref ListTokenSource::nextToken() { +Ref ListTokenSource::nextToken() { if (i >= tokens.size()) { if (eofToken == nullptr) { int start = -1; @@ -81,7 +81,7 @@ Token::Ref ListTokenSource::nextToken() { return eofToken; } - Token::Ref t = tokens[i]; + Ref t = tokens[i]; if (i == tokens.size() - 1 && t->getType() == EOF) { eofToken = t; } @@ -98,7 +98,7 @@ size_t ListTokenSource::getLine() const { } else if (tokens.size() > 0) { // have to calculate the result from the line/column of the previous // token, along with the text of the token. - Token::Ref lastToken = tokens.back(); + Ref lastToken = tokens.back(); int line = lastToken->getLine(); std::wstring tokenText = lastToken->getText(); @@ -145,7 +145,7 @@ std::string ListTokenSource::getSourceName() { return "List"; } -std::shared_ptr> ListTokenSource::getTokenFactory() { +Ref> ListTokenSource::getTokenFactory() { return _factory; } diff --git a/runtime/Cpp/runtime/ListTokenSource.h b/runtime/Cpp/runtime/ListTokenSource.h index 416fc7ee2..68aa95479 100755 --- a/runtime/Cpp/runtime/ListTokenSource.h +++ b/runtime/Cpp/runtime/ListTokenSource.h @@ -49,7 +49,7 @@ namespace runtime { /// class ListTokenSource : public TokenSource { protected: - const std::vector tokens; + const std::vector> tokens; /// /// The name of the input source. If this value is {@code null}, a call to @@ -71,14 +71,14 @@ namespace runtime { /// /// This field caches the EOF token for the token source. /// - Token::Ref eofToken; + Ref eofToken; /// /// This is the backing field for and /// . /// private: - std::shared_ptr> _factory = CommonTokenFactory::DEFAULT; + Ref> _factory = CommonTokenFactory::DEFAULT; /// /// Constructs a new instance from the specified @@ -115,7 +115,7 @@ namespace runtime { } virtual int getCharPositionInLine() override; - virtual Token::Ref nextToken() override; + virtual Ref nextToken() override; virtual size_t getLine() const override; virtual CharStream* getInputStream() override; virtual std::string getSourceName() override; @@ -125,7 +125,7 @@ namespace runtime { this->_factory = factory; } - virtual std::shared_ptr> getTokenFactory() override; + virtual Ref> getTokenFactory() override; private: void InitializeInstanceFields(); diff --git a/runtime/Cpp/runtime/NoViableAltException.cpp b/runtime/Cpp/runtime/NoViableAltException.cpp index f6172e8a8..fa121a6aa 100755 --- a/runtime/Cpp/runtime/NoViableAltException.cpp +++ b/runtime/Cpp/runtime/NoViableAltException.cpp @@ -40,15 +40,15 @@ NoViableAltException::NoViableAltException(Parser *recognizer) recognizer->getCurrentToken(), nullptr, recognizer->getContext()) { } -NoViableAltException::NoViableAltException(Parser *recognizer, TokenStream *input, Token::Ref startToken, - Token::Ref offendingToken, std::shared_ptr deadEndConfigs, ParserRuleContext::Ref ctx) +NoViableAltException::NoViableAltException(Parser *recognizer, TokenStream *input, Ref startToken, + Ref offendingToken, Ref deadEndConfigs, Ref ctx) : RecognitionException(recognizer, input, ctx, offendingToken), _deadEndConfigs(deadEndConfigs), _startToken(startToken) { } -Token::Ref NoViableAltException::getStartToken() const { +Ref NoViableAltException::getStartToken() const { return _startToken; } -std::shared_ptr NoViableAltException::getDeadEndConfigs() const { +Ref NoViableAltException::getDeadEndConfigs() const { return _deadEndConfigs; } diff --git a/runtime/Cpp/runtime/NoViableAltException.h b/runtime/Cpp/runtime/NoViableAltException.h index ef3f2be6e..052475594 100755 --- a/runtime/Cpp/runtime/NoViableAltException.h +++ b/runtime/Cpp/runtime/NoViableAltException.h @@ -47,11 +47,11 @@ namespace runtime { class NoViableAltException : public RecognitionException { public: NoViableAltException(Parser *recognizer); // LL(1) error - NoViableAltException(Parser *recognizer, TokenStream *input, Token::Ref startToken, Token::Ref offendingToken, - std::shared_ptr deadEndConfigs, ParserRuleContext::Ref ctx); + NoViableAltException(Parser *recognizer, TokenStream *input, Ref startToken, Ref offendingToken, + Ref deadEndConfigs, Ref ctx); - virtual Token::Ref getStartToken() const; - virtual std::shared_ptr getDeadEndConfigs() const; + virtual Ref getStartToken() const; + virtual Ref getDeadEndConfigs() const; private: /// Which configurations did we try at input.index() that couldn't match input.LT(1)? @@ -61,7 +61,7 @@ namespace runtime { /// not be buffering tokens so get a reference to it. (At the /// time the error occurred, of course the stream needs to keep a /// buffer all of the tokens but later we might not have access to those.) - Token::Ref _startToken; + Ref _startToken; }; diff --git a/runtime/Cpp/runtime/Parser.cpp b/runtime/Cpp/runtime/Parser.cpp index ccf8cd540..76f587b9c 100755 --- a/runtime/Cpp/runtime/Parser.cpp +++ b/runtime/Cpp/runtime/Parser.cpp @@ -47,6 +47,9 @@ #include "ANTLRErrorListener.h" #include "ParseTreePattern.h" +#include "ProfilingATNSimulator.h" +#include "ParseInfo.h" + #include "Parser.h" using namespace org::antlr::v4::runtime; @@ -57,43 +60,43 @@ std::map Parser::bypassAltsAtnCache; Parser::TraceListener::TraceListener(Parser *outerInstance) : outerInstance(outerInstance) { } -void Parser::TraceListener::enterEveryRule(ParserRuleContext::Ref ctx) { +void Parser::TraceListener::enterEveryRule(Ref ctx) { std::cout << "enter " << antlrcpp::ws2s(outerInstance->getRuleNames()[(size_t)ctx->getRuleIndex()]) << ", LT(1)=" << antlrcpp::ws2s(outerInstance->_input->LT(1)->getText()) << std::endl; } -void Parser::TraceListener::visitTerminal(std::shared_ptr node) { +void Parser::TraceListener::visitTerminal(Ref node) { std::cout << "consume " << node->getSymbol() << " rule " << antlrcpp::ws2s(outerInstance->getRuleNames()[(size_t)outerInstance->getContext()->getRuleIndex()]) << std::endl; } -void Parser::TraceListener::visitErrorNode(std::shared_ptr node) { +void Parser::TraceListener::visitErrorNode(Ref node) { } -void Parser::TraceListener::exitEveryRule(ParserRuleContext::Ref ctx) { +void Parser::TraceListener::exitEveryRule(Ref ctx) { std::cout << "exit " << antlrcpp::ws2s(outerInstance->getRuleNames()[(size_t)ctx->getRuleIndex()]) << ", LT(1)=" << antlrcpp::ws2s(outerInstance->_input->LT(1)->getText()) << std::endl; } -const std::shared_ptr Parser::TrimToSizeListener::INSTANCE = +const Ref Parser::TrimToSizeListener::INSTANCE = std::make_shared(); -void Parser::TrimToSizeListener::enterEveryRule(ParserRuleContext::Ref ctx) { +void Parser::TrimToSizeListener::enterEveryRule(Ref ctx) { } -void Parser::TrimToSizeListener::visitTerminal(std::shared_ptr node) { +void Parser::TrimToSizeListener::visitTerminal(Ref node) { } -void Parser::TrimToSizeListener::visitErrorNode(std::shared_ptr node) { +void Parser::TrimToSizeListener::visitErrorNode(Ref node) { } -void Parser::TrimToSizeListener::exitEveryRule(ParserRuleContext::Ref ctx) { +void Parser::TrimToSizeListener::exitEveryRule(Ref ctx) { ctx->children.shrink_to_fit(); } @@ -121,9 +124,12 @@ void Parser::reset() { } } -Token::Ref Parser::match(int ttype) { - Token::Ref t = getCurrentToken(); +Ref Parser::match(int ttype) { + Ref t = getCurrentToken(); if (t->getType() == ttype) { + if (ttype == EOF) { + _matchedEOF = true; + } _errHandler->reportMatch(this); consume(); } else { @@ -137,8 +143,8 @@ Token::Ref Parser::match(int ttype) { return t; } -Token::Ref Parser::matchWildcard() { - Token::Ref t = getCurrentToken(); +Ref Parser::matchWildcard() { + Ref t = getCurrentToken(); if (t->getType() > 0) { _errHandler->reportMatch(this); consume(); @@ -177,11 +183,11 @@ bool Parser::getTrimParseTree() { return std::find(getParseListeners().begin(), getParseListeners().end(), TrimToSizeListener::INSTANCE) != getParseListeners().end(); } -std::vector> Parser::getParseListeners() { +std::vector> Parser::getParseListeners() { return _parseListeners; } -void Parser::addParseListener(std::shared_ptr listener) { +void Parser::addParseListener(Ref listener) { if (!listener) { throw NullPointerException("listener"); } @@ -189,7 +195,7 @@ void Parser::addParseListener(std::shared_ptr listener) this->_parseListeners.push_back(listener); } -void Parser::removeParseListener(std::shared_ptr listener) { +void Parser::removeParseListener(Ref listener) { if (!_parseListeners.empty()) { auto it = std::find(_parseListeners.begin(), _parseListeners.end(), listener); if (it != _parseListeners.end()) { @@ -221,7 +227,7 @@ int Parser::getNumberOfSyntaxErrors() { return _syntaxErrors; } -std::shared_ptr> Parser::getTokenFactory() { +Ref> Parser::getTokenFactory() { return _input->getTokenSource()->getTokenFactory(); } @@ -265,11 +271,11 @@ tree::pattern::ParseTreePattern Parser::compileParseTreePattern(const std::wstri return m.compile(pattern, patternRuleIndex); } -std::shared_ptr Parser::getErrorHandler() { +Ref Parser::getErrorHandler() { return _errHandler; } -void Parser::setErrorHandler(std::shared_ptr handler) { +void Parser::setErrorHandler(Ref handler) { _errHandler = handler; } @@ -291,7 +297,7 @@ void Parser::setTokenStream(TokenStream *input) { _input = input; } -Token::Ref Parser::getCurrentToken() { +Ref Parser::getCurrentToken() { return _input->LT(1); } @@ -299,7 +305,7 @@ void Parser::notifyErrorListeners(const std::wstring &msg) { notifyErrorListeners(getCurrentToken(), msg, nullptr); } -void Parser::notifyErrorListeners(Token::Ref offendingToken, const std::wstring &msg, std::exception_ptr e) { +void Parser::notifyErrorListeners(Ref offendingToken, const std::wstring &msg, std::exception_ptr e) { _syntaxErrors++; int line = -1; int charPositionInLine = -1; @@ -310,22 +316,22 @@ void Parser::notifyErrorListeners(Token::Ref offendingToken, const std::wstring listener.syntaxError(this, offendingToken, (size_t)line, charPositionInLine, msg, e); } -Token::Ref Parser::consume() { - Token::Ref o = getCurrentToken(); +Ref Parser::consume() { + Ref o = getCurrentToken(); if (o->getType() != EOF) { getInputStream()->consume(); } bool hasListener = _parseListeners.size() > 0 && !_parseListeners.empty(); if (_buildParseTrees || hasListener) { if (_errHandler->inErrorRecoveryMode(this)) { - std::shared_ptr node = _ctx->addErrorNode(o); + Ref node = _ctx->addErrorNode(o); if (_parseListeners.size() > 0) { for (auto listener : _parseListeners) { listener->visitErrorNode(node); } } } else { - std::shared_ptr node = _ctx->addChild(o); + Ref node = _ctx->addChild(o); if (_parseListeners.size() > 0) { for (auto listener : _parseListeners) { listener->visitTerminal(node); @@ -341,11 +347,11 @@ void Parser::addContextToParseTree() { if (_ctx->parent.expired()) return; - ParserRuleContext::Ref parent = std::dynamic_pointer_cast(_ctx->parent.lock()); + Ref parent = std::dynamic_pointer_cast(_ctx->parent.lock()); parent->addChild(_ctx); } -void Parser::enterRule(ParserRuleContext::Ref localctx, int state, int ruleIndex) { +void Parser::enterRule(Ref localctx, int state, int ruleIndex) { setState(state); _ctx = localctx; _ctx->start = _input->LT(1); @@ -358,6 +364,13 @@ void Parser::enterRule(ParserRuleContext::Ref localctx, int state, int ruleIndex } void Parser::exitRule() { + if (_matchedEOF) { + // if we have matched EOF, it cannot consume past EOF so we use LT(1) here + _ctx->stop = _input->LT(1); // LT(1) will be end of file + } else { + _ctx->stop = _input->LT(-1); // stop node is what we just matched + } + _ctx->stop = _input->LT(-1); // trigger event on ctx, before it reverts to parent if (_parseListeners.size() > 0) { @@ -367,12 +380,14 @@ void Parser::exitRule() { _ctx = std::dynamic_pointer_cast(_ctx->parent.lock()); } -void Parser::enterOuterAlt(ParserRuleContext::Ref localctx, int altNum) { +void Parser::enterOuterAlt(Ref localctx, int altNum) { + localctx->setAltNumber(altNum); + // if we have new localctx, make sure we replace existing ctx // that is previous child of parse tree if (_buildParseTrees && _ctx != localctx) { if (!_ctx->parent.expired()) { - ParserRuleContext::Ref parent = std::dynamic_pointer_cast(_ctx->parent.lock()); + Ref parent = std::dynamic_pointer_cast(_ctx->parent.lock()); parent->removeLastChild(); parent->addChild(localctx); } @@ -380,11 +395,19 @@ void Parser::enterOuterAlt(ParserRuleContext::Ref localctx, int altNum) { _ctx = localctx; } -void Parser::enterRecursionRule(ParserRuleContext::Ref localctx, int ruleIndex) { +int Parser::getPrecedence() const { + if (_precedenceStack.empty()) { + return -1; + } + + return _precedenceStack.back(); +} + +void Parser::enterRecursionRule(Ref localctx, int ruleIndex) { enterRecursionRule(localctx, getATN().ruleToStartState[(size_t)ruleIndex]->stateNumber, ruleIndex, 0); } -void Parser::enterRecursionRule(ParserRuleContext::Ref localctx, int state, int ruleIndex, int precedence) { +void Parser::enterRecursionRule(Ref localctx, int state, int ruleIndex, int precedence) { setState(state); _precedenceStack.push_back(precedence); _ctx = localctx; @@ -394,8 +417,8 @@ void Parser::enterRecursionRule(ParserRuleContext::Ref localctx, int state, int } } -void Parser::pushNewRecursionContext(ParserRuleContext::Ref localctx, int state, int ruleIndex) { - ParserRuleContext::Ref previous = _ctx; +void Parser::pushNewRecursionContext(Ref localctx, int state, int ruleIndex) { + Ref previous = _ctx; previous->parent = localctx; previous->invokingState = state; previous->stop = _input->LT(-1); @@ -411,10 +434,10 @@ void Parser::pushNewRecursionContext(ParserRuleContext::Ref localctx, int state, } } -void Parser::unrollRecursionContexts(ParserRuleContext::Ref parentctx) { +void Parser::unrollRecursionContexts(Ref parentctx) { _precedenceStack.pop_back(); _ctx->stop = _input->LT(-1); - ParserRuleContext::Ref retctx = _ctx; // save current ctx (return value) + Ref retctx = _ctx; // save current ctx (return value) // unroll so ctx is as it was before call to recursive method if (_parseListeners.size() > 0) { @@ -435,8 +458,8 @@ void Parser::unrollRecursionContexts(ParserRuleContext::Ref parentctx) { } } -ParserRuleContext::Ref Parser::getInvokingContext(int ruleIndex) { - ParserRuleContext::Ref p = _ctx; +Ref Parser::getInvokingContext(int ruleIndex) { + Ref p = _ctx; while (p) { if (p->getRuleIndex() == ruleIndex) { return p; @@ -445,18 +468,18 @@ ParserRuleContext::Ref Parser::getInvokingContext(int ruleIndex) { break; p = std::dynamic_pointer_cast(p->parent.lock()); } - return ParserRuleContext::Ref(); + return Ref(); } -ParserRuleContext::Ref Parser::getContext() { +Ref Parser::getContext() { return _ctx; } -void Parser::setContext(ParserRuleContext::Ref ctx) { +void Parser::setContext(Ref ctx) { _ctx = ctx; } -bool Parser::precpred(RuleContext::Ref localctx, int precedence) { +bool Parser::precpred(Ref localctx, int precedence) { return precedence >= _precedenceStack.back(); } @@ -467,7 +490,7 @@ bool Parser::inContext(const std::wstring &context) { bool Parser::isExpectedToken(int symbol) { const atn::ATN &atn = getInterpreter()->atn; - ParserRuleContext::Ref ctx = _ctx; + Ref ctx = _ctx; atn::ATNState *s = atn.states[(size_t)getState()]; misc::IntervalSet following = atn.nextTokens(s); @@ -497,6 +520,10 @@ bool Parser::isExpectedToken(int symbol) { return false; } +bool Parser::isMatchedEOF() const { + return _matchedEOF; +} + misc::IntervalSet Parser::getExpectedTokens() { return getATN().getExpectedTokens(getState(), getContext()); } @@ -516,7 +543,7 @@ ssize_t Parser::getRuleIndex(const std::wstring &ruleName) { return iterator->second; } -ParserRuleContext::Ref Parser::getRuleContext() { +Ref Parser::getRuleContext() { return _ctx; } @@ -524,7 +551,7 @@ std::vector Parser::getRuleInvocationStack() { return getRuleInvocationStack(_ctx); } -std::vector Parser::getRuleInvocationStack(RuleContext::Ref p) { +std::vector Parser::getRuleInvocationStack(Ref p) { std::vector ruleNames = getRuleNames(); std::vector stack = std::vector(); while (p) { @@ -544,13 +571,13 @@ std::vector Parser::getRuleInvocationStack(RuleContext::Ref p) { std::vector Parser::getDFAStrings() { atn::ParserATNSimulator *simulator = getInterpreter(); - if (!simulator->_decisionToDFA.empty()) { + if (!simulator->decisionToDFA.empty()) { std::lock_guard lck(mtx); std::vector s; - for (size_t d = 0; d < simulator->_decisionToDFA.size(); d++) { - dfa::DFA &dfa = simulator->_decisionToDFA[d]; - s.push_back(dfa.toString(getTokenNames())); + for (size_t d = 0; d < simulator->decisionToDFA.size(); d++) { + dfa::DFA &dfa = simulator->decisionToDFA[d]; + s.push_back(dfa.toString(getVocabulary())); } return s; } @@ -559,18 +586,18 @@ std::vector Parser::getDFAStrings() { void Parser::dumpDFA() { atn::ParserATNSimulator *simulator = getInterpreter(); - if (!simulator->_decisionToDFA.empty()) { + if (!simulator->decisionToDFA.empty()) { std::lock_guard lck(mtx); bool seenOne = false; - for (size_t d = 0; d < simulator->_decisionToDFA.size(); d++) { - dfa::DFA &dfa = simulator->_decisionToDFA[d]; + for (size_t d = 0; d < simulator->decisionToDFA.size(); d++) { + dfa::DFA &dfa = simulator->decisionToDFA[d]; if (!dfa.states.empty()) { if (seenOne) { std::cout << std::endl; } std::cout << L"Decision " << dfa.decision << L":" << std::endl; dfa.toString(getTokenNames()); - std::wcout << dfa.toString(getTokenNames()); + std::wcout << dfa.toString(getVocabulary()); seenOne = true; } } @@ -581,6 +608,29 @@ std::string Parser::getSourceName() { return _input->getSourceName(); } +Ref Parser::getParseInfo() const { + atn::ProfilingATNSimulator *interp = getInterpreter(); + if (interp != nullptr) { + return std::make_shared(interp); + } + return nullptr; +} + +void Parser::setProfile(bool profile) { + atn::ParserATNSimulator *interp = getInterpreter(); + atn::PredictionMode saveMode = interp->getPredictionMode(); + if (profile) { + if (!is(interp)) { + setInterpreter(new atn::ProfilingATNSimulator(this)); /* mem-check: replacing existing interpreter which gets deleted. */ + } + } else if (is(interp)) { + /* mem-check: replacing existing interpreter which gets deleted. */ + atn::ParserATNSimulator *sim = new atn::ParserATNSimulator(this, getATN(), interp->decisionToDFA, interp->getSharedContextCache()); + setInterpreter(sim); + } + getInterpreter()->setPredictionMode(saveMode); +} + void Parser::setTrace(bool trace) { if (!trace) { if (_tracer) @@ -594,12 +644,17 @@ void Parser::setTrace(bool trace) { } } +bool Parser::isTrace() const { + return _tracer != nullptr; +} + void Parser::InitializeInstanceFields() { _errHandler = std::make_shared(); _precedenceStack.clear(); _precedenceStack.push_back(0); _buildParseTrees = true; _syntaxErrors = 0; + _matchedEOF = false; _input = nullptr; } diff --git a/runtime/Cpp/runtime/Parser.h b/runtime/Cpp/runtime/Parser.h index d55fd1adf..6124db1c0 100755 --- a/runtime/Cpp/runtime/Parser.h +++ b/runtime/Cpp/runtime/Parser.h @@ -51,10 +51,10 @@ namespace runtime { TraceListener(Parser *outerInstance); virtual ~TraceListener() {}; - virtual void enterEveryRule(ParserRuleContext::Ref ctx) override; - virtual void visitTerminal(std::shared_ptr node) override; - virtual void visitErrorNode(std::shared_ptr node) override; - virtual void exitEveryRule(ParserRuleContext::Ref ctx) override; + virtual void enterEveryRule(Ref ctx) override; + virtual void visitTerminal(Ref node) override; + virtual void visitErrorNode(Ref node) override; + virtual void exitEveryRule(Ref ctx) override; private: Parser *const outerInstance; @@ -62,12 +62,12 @@ namespace runtime { class TrimToSizeListener : public tree::ParseTreeListener { public: - static const std::shared_ptr INSTANCE; + static const Ref INSTANCE; - virtual void enterEveryRule(ParserRuleContext::Ref ctx) override; - virtual void visitTerminal(std::shared_ptr node) override; - virtual void visitErrorNode(std::shared_ptr node) override; - virtual void exitEveryRule(ParserRuleContext::Ref ctx) override; + virtual void enterEveryRule(Ref ctx) override; + virtual void visitTerminal(Ref node) override; + virtual void visitErrorNode(Ref node) override; + virtual void exitEveryRule(Ref ctx) override; }; Parser(TokenStream *input); @@ -94,7 +94,7 @@ namespace runtime { /// if the current input symbol did not match /// {@code ttype} and the error strategy could not recover from the /// mismatched symbol - virtual Token::Ref match(int ttype); + virtual Ref match(int ttype); /// /// Match current input symbol as a wildcard. If the symbol type matches @@ -112,7 +112,7 @@ namespace runtime { /// if the current input symbol did not match /// a wildcard and the error strategy could not recover from the mismatched /// symbol - virtual Token::Ref matchWildcard(); + virtual Ref matchWildcard(); /// /// Track the objects during the parse and hook @@ -151,7 +151,7 @@ namespace runtime { /// using the default during the parse process. virtual bool getTrimParseTree(); - virtual std::vector> getParseListeners(); + virtual std::vector> getParseListeners(); /// /// Registers {@code listener} to receive events during the parsing process. @@ -181,7 +181,7 @@ namespace runtime { /// the listener to add /// /// if {@code} listener is {@code null} - virtual void addParseListener(std::shared_ptr listener); + virtual void addParseListener(Ref listener); /// /// Remove {@code listener} from the list of parse listeners. @@ -192,7 +192,7 @@ namespace runtime { /// /// the listener to remove - virtual void removeParseListener(std::shared_ptr listener); + virtual void removeParseListener(Ref listener); /// /// Remove all parse listeners. @@ -219,7 +219,7 @@ namespace runtime { /// virtual int getNumberOfSyntaxErrors(); - virtual std::shared_ptr> getTokenFactory() override; + virtual Ref> getTokenFactory() override; /// /// Tell our token source and error strategy about a new way to create tokens. @@ -252,8 +252,8 @@ namespace runtime { virtual tree::pattern::ParseTreePattern compileParseTreePattern(const std::wstring &pattern, int patternRuleIndex, Lexer *lexer); - virtual std::shared_ptr getErrorHandler(); - virtual void setErrorHandler(std::shared_ptr handler); + virtual Ref getErrorHandler(); + virtual void setErrorHandler(Ref handler); virtual IntStream* getInputStream() override; void setInputStream(IntStream *input) override; @@ -267,11 +267,11 @@ namespace runtime { /// Match needs to return the current input symbol, which gets put /// into the label for the associated token ref; e.g., x=ID. /// - virtual Token::Ref getCurrentToken(); + virtual Ref getCurrentToken(); void notifyErrorListeners(const std::wstring &msg); - virtual void notifyErrorListeners(Token::Ref offendingToken, const std::wstring &msg, std::exception_ptr e); + virtual void notifyErrorListeners(Ref offendingToken, const std::wstring &msg, std::exception_ptr e); /// /// Consume and return the . @@ -294,38 +294,40 @@ namespace runtime { /// is called on any parse /// listeners. /// - virtual Token::Ref consume(); + virtual Ref consume(); /// /// Always called by generated parsers upon entry to a rule. Access field /// get the current context. /// - virtual void enterRule(ParserRuleContext::Ref localctx, int state, int ruleIndex); + virtual void enterRule(Ref localctx, int state, int ruleIndex); virtual void exitRule(); - virtual void enterOuterAlt(ParserRuleContext::Ref localctx, int altNum); + virtual void enterOuterAlt(Ref localctx, int altNum); + + /** + * Get the precedence level for the top-most precedence rule. + * + * @return The precedence level for the top-most precedence rule, or -1 if + * the parser context is not nested within a precedence rule. + */ + int getPrecedence() const; /// @deprecated Use /// instead. - virtual void enterRecursionRule(ParserRuleContext::Ref localctx, int ruleIndex); - virtual void enterRecursionRule(ParserRuleContext::Ref localctx, int state, int ruleIndex, int precedence); - - /// - /// Like but for recursive rules. - /// - virtual void pushNewRecursionContext(ParserRuleContext::Ref localctx, int state, int ruleIndex); - - virtual void unrollRecursionContexts(ParserRuleContext::Ref parentctx); - - virtual ParserRuleContext::Ref getInvokingContext(int ruleIndex); - - virtual ParserRuleContext::Ref getContext(); - - virtual void setContext(ParserRuleContext::Ref ctx); - - virtual bool precpred(RuleContext::Ref localctx, int precedence) override; + virtual void enterRecursionRule(Ref localctx, int ruleIndex); + virtual void enterRecursionRule(Ref localctx, int state, int ruleIndex, int precedence); + /** Like {@link #enterRule} but for recursive rules. + * Make the current context the child of the incoming localctx. + */ + virtual void pushNewRecursionContext(Ref localctx, int state, int ruleIndex); + virtual void unrollRecursionContexts(Ref parentctx); + virtual Ref getInvokingContext(int ruleIndex); + virtual Ref getContext(); + virtual void setContext(Ref ctx); + virtual bool precpred(Ref localctx, int precedence) override; virtual bool inContext(const std::wstring &context); /// @@ -343,6 +345,8 @@ namespace runtime { /// the ATN, otherwise {@code false}. virtual bool isExpectedToken(int symbol); + bool isMatchedEOF() const; + /// /// Computes the set of input symbols which could follow the current parser /// state and context, as given by and , @@ -357,7 +361,7 @@ namespace runtime { /// Get a rule's index (i.e., {@code RULE_ruleName} field) or -1 if not found. virtual ssize_t getRuleIndex(const std::wstring &ruleName); - virtual ParserRuleContext::Ref getRuleContext(); + virtual Ref getRuleContext(); /// /// Return List<String> of the rule names in your parser instance @@ -369,7 +373,7 @@ namespace runtime { /// virtual std::vector getRuleInvocationStack(); - virtual std::vector getRuleInvocationStack(RuleContext::Ref p); + virtual std::vector getRuleInvocationStack(Ref p); /// /// For debugging and other purposes. @@ -381,20 +385,35 @@ namespace runtime { virtual std::string getSourceName(); + virtual Ref getParseInfo() const override; + + /** + * @since 4.3 + */ + void setProfile(bool profile); + /// /// During a parse is sometimes useful to listen in on the rule entry and exit /// events as well as token matches. This is for quick and dirty debugging. /// virtual void setTrace(bool trace); + /** + * Gets whether a {@link TraceListener} is registered as a parse listener + * for the parser. + * + * @see #setTrace(boolean) + */ + bool isTrace() const; + protected: /// The ParserRuleContext object for the currently executing rule. /// This is always non-null during the parsing process. - ParserRuleContext::Ref _ctx; + Ref _ctx; /// The error handling strategy for the parser. The default is DefaultErrorStrategy. /// See also getErrorHandler. - std::shared_ptr _errHandler; + Ref _errHandler; /// /// The input stream. @@ -404,6 +423,7 @@ namespace runtime { TokenStream *_input; std::vector _precedenceStack; + //Mutex to manage synchronized access for multithreading in the parser std::mutex mtx; @@ -420,7 +440,7 @@ namespace runtime { /// events during the parse. /// /// - std::vector> _parseListeners; + std::vector> _parseListeners; /// /// The number of syntax errors reported during parsing. This value is @@ -428,6 +448,9 @@ namespace runtime { /// int _syntaxErrors; + /** Indicates parser has match()ed EOF token. See {@link #exitRule()}. */ + bool _matchedEOF; + virtual void addContextToParseTree(); private: @@ -442,7 +465,7 @@ namespace runtime { /// later call to setTrace(false). The listener itself is /// implemented as a parser listener so this field is not directly used by /// other parser methods. - std::shared_ptr _tracer; + Ref _tracer; void InitializeInstanceFields(); }; diff --git a/runtime/Cpp/runtime/ParserInterpreter.cpp b/runtime/Cpp/runtime/ParserInterpreter.cpp index a55737787..5c6b25cfc 100755 --- a/runtime/Cpp/runtime/ParserInterpreter.cpp +++ b/runtime/Cpp/runtime/ParserInterpreter.cpp @@ -105,10 +105,10 @@ std::wstring ParserInterpreter::getGrammarFileName() const { return _grammarFileName; } -ParserRuleContext::Ref ParserInterpreter::parse(int startRuleIndex) { +Ref ParserInterpreter::parse(int startRuleIndex) { atn::RuleStartState *startRuleStartState = _atn.ruleToStartState[(size_t)startRuleIndex]; - std::shared_ptr rootContext = + Ref rootContext = std::make_shared(std::weak_ptr(), atn::ATNState::INVALID_STATE_NUMBER, startRuleIndex); if (startRuleStartState->isLeftRecursiveRule) { @@ -137,7 +137,7 @@ ParserRuleContext::Ref ParserInterpreter::parse(int startRuleIndex) { } } -void ParserInterpreter::enterRecursionRule(ParserRuleContext::Ref localctx, int state, int ruleIndex, int precedence) { +void ParserInterpreter::enterRecursionRule(Ref localctx, int state, int ruleIndex, int precedence) { _parentContextStack.push({ _ctx, localctx->invokingState }); Parser::enterRecursionRule(localctx, state, ruleIndex, precedence); } @@ -158,7 +158,7 @@ void ParserInterpreter::visitState(atn::ATNState *p) { switch (transition->getSerializationType()) { case atn::Transition::EPSILON: if (_pushRecursionContextStates[(size_t)p->stateNumber] == 1 && is(transition->target)) { - std::shared_ptr ruleContext = std::make_shared(_parentContextStack.top().first, + Ref ruleContext = std::make_shared(_parentContextStack.top().first, _parentContextStack.top().second, _ctx->getRuleIndex()); pushNewRecursionContext(ruleContext, _atn.ruleToStartState[(size_t)p->ruleIndex]->stateNumber, (int)ruleContext->getRuleIndex()); @@ -186,7 +186,7 @@ void ParserInterpreter::visitState(atn::ATNState *p) { { atn::RuleStartState *ruleStartState = (atn::RuleStartState*)(transition->target); int ruleIndex = ruleStartState->ruleIndex; - std::shared_ptr ruleContext = std::make_shared(_ctx, p->stateNumber, ruleIndex); + Ref ruleContext = std::make_shared(_ctx, p->stateNumber, ruleIndex); if (ruleStartState->isLeftRecursiveRule) { enterRecursionRule(ruleContext, ruleStartState->stateNumber, ruleIndex, ((atn::RuleTransition*)(transition))->precedence); } else { @@ -229,7 +229,7 @@ void ParserInterpreter::visitState(atn::ATNState *p) { void ParserInterpreter::visitRuleStopState(atn::ATNState *p) { atn::RuleStartState *ruleStartState = _atn.ruleToStartState[(size_t)p->ruleIndex]; if (ruleStartState->isLeftRecursiveRule) { - std::pair parentContext = _parentContextStack.top(); + std::pair, int> parentContext = _parentContextStack.top(); _parentContextStack.pop(); unrollRecursionContexts(parentContext.first); diff --git a/runtime/Cpp/runtime/ParserInterpreter.h b/runtime/Cpp/runtime/ParserInterpreter.h index 35be17c6e..3aafaa4eb 100755 --- a/runtime/Cpp/runtime/ParserInterpreter.h +++ b/runtime/Cpp/runtime/ParserInterpreter.h @@ -67,9 +67,9 @@ namespace runtime { virtual std::wstring getGrammarFileName() const override; /// Begin parsing at startRuleIndex - virtual ParserRuleContext::Ref parse(int startRuleIndex); + virtual Ref parse(int startRuleIndex); - virtual void enterRecursionRule(ParserRuleContext::Ref localctx, int state, int ruleIndex, int precedence) override; + virtual void enterRecursionRule(Ref localctx, int state, int ruleIndex, int precedence) override; protected: const std::wstring _grammarFileName; @@ -80,9 +80,9 @@ namespace runtime { antlrcpp::BitSet _pushRecursionContextStates; std::vector _decisionToDFA; // not shared like it is for generated parsers - std::shared_ptr _sharedContextCache; + Ref _sharedContextCache; - std::stack> _parentContextStack; + std::stack, int>> _parentContextStack; virtual atn::ATNState *getATNState(); virtual void visitState(atn::ATNState *p); diff --git a/runtime/Cpp/runtime/ParserRuleContext.cpp b/runtime/Cpp/runtime/ParserRuleContext.cpp index 570685ea6..b8cdc5519 100755 --- a/runtime/Cpp/runtime/ParserRuleContext.cpp +++ b/runtime/Cpp/runtime/ParserRuleContext.cpp @@ -43,7 +43,7 @@ using namespace antlrcpp; ParserRuleContext::ParserRuleContext() { } -void ParserRuleContext::copyFrom(std::shared_ptr ctx) { +void ParserRuleContext::copyFrom(Ref ctx) { // from RuleContext this->parent = ctx->parent; this->invokingState = ctx->invokingState; @@ -56,18 +56,18 @@ ParserRuleContext::ParserRuleContext(std::weak_ptr parent, in : RuleContext(parent, invokingStateNumber) { } -void ParserRuleContext::enterRule(std::shared_ptr listener) { +void ParserRuleContext::enterRule(Ref listener) { } -void ParserRuleContext::exitRule(std::shared_ptr listener) { +void ParserRuleContext::exitRule(Ref listener) { } -std::shared_ptr ParserRuleContext::addChild(std::shared_ptr t) { +Ref ParserRuleContext::addChild(Ref t) { children.push_back(t); return t; } -RuleContext::Ref ParserRuleContext::addChild(RuleContext::Ref ruleInvocation) { +Ref ParserRuleContext::addChild(Ref ruleInvocation) { children.push_back(ruleInvocation); return ruleInvocation; } @@ -78,25 +78,25 @@ void ParserRuleContext::removeLastChild() { } } -std::shared_ptr ParserRuleContext::addChild(Token::Ref matchedToken) { - std::shared_ptr t = std::make_shared(matchedToken); +Ref ParserRuleContext::addChild(Ref matchedToken) { + Ref t = std::make_shared(matchedToken); addChild(t); t->parent = shared_from_this(); return t; } -std::shared_ptr ParserRuleContext::addErrorNode(Token::Ref badToken) { - std::shared_ptr t = std::make_shared(badToken); +Ref ParserRuleContext::addErrorNode(Ref badToken) { + Ref t = std::make_shared(badToken); addChild(t); t->parent = shared_from_this(); return t; } -std::shared_ptr ParserRuleContext::getChildReference(std::size_t i) { +Ref ParserRuleContext::getChildReference(std::size_t i) { return children[i]; } -std::shared_ptr ParserRuleContext::getToken(int ttype, std::size_t i) { +Ref ParserRuleContext::getToken(int ttype, std::size_t i) { if (i >= children.size()) { return nullptr; } @@ -104,8 +104,8 @@ std::shared_ptr ParserRuleContext::getToken(int ttype, std:: size_t j = 0; // what token with ttype have we found? for (auto o : children) { if (is(o)) { - std::shared_ptr tnode = std::dynamic_pointer_cast(o); - Token::Ref symbol = tnode->getSymbol(); + Ref tnode = std::dynamic_pointer_cast(o); + Ref symbol = tnode->getSymbol(); if (symbol->getType() == ttype) { if (j++ == i) { return tnode; @@ -117,12 +117,12 @@ std::shared_ptr ParserRuleContext::getToken(int ttype, std:: return nullptr; } -std::vector> ParserRuleContext::getTokens(int ttype) { - std::vector> tokens; +std::vector> ParserRuleContext::getTokens(int ttype) { + std::vector> tokens; for (auto &o : children) { if (is(o)) { - std::shared_ptr tnode = std::dynamic_pointer_cast(o); - Token::Ref symbol = tnode->getSymbol(); + Ref tnode = std::dynamic_pointer_cast(o); + Ref symbol = tnode->getSymbol(); if (symbol->getType() == ttype) { tokens.push_back(tnode); } @@ -132,24 +132,26 @@ std::vector> ParserRuleContext::getTokens(in return tokens; } - - std::size_t ParserRuleContext::getChildCount() { return children.size(); } misc::Interval ParserRuleContext::getSourceInterval() { - if (start == nullptr || stop == nullptr) { + if (start == nullptr) { return misc::Interval::INVALID; } + + if (stop == nullptr || stop->getTokenIndex() < start->getTokenIndex()) { + return misc::Interval(start->getTokenIndex(), start->getTokenIndex() - 1); // empty + } return misc::Interval(start->getTokenIndex(), stop->getTokenIndex()); } -Token::Ref ParserRuleContext::getStart() { +Ref ParserRuleContext::getStart() { return start; } -Token::Ref ParserRuleContext::getStop() { +Ref ParserRuleContext::getStop() { return stop; } diff --git a/runtime/Cpp/runtime/ParserRuleContext.h b/runtime/Cpp/runtime/ParserRuleContext.h index 511ea4860..c59fa1a8c 100755 --- a/runtime/Cpp/runtime/ParserRuleContext.h +++ b/runtime/Cpp/runtime/ParserRuleContext.h @@ -31,7 +31,6 @@ #pragma once -#include "Token.h" #include "RuleContext.h" #include "CPPUtils.h" @@ -45,9 +44,8 @@ namespace runtime { /// /// Contains all of the information about the current rule not stored in the /// RuleContext. It handles parse tree children list, Any ATN state - /// tracing, and the default values available for rule indications: - /// start, stop, rule index, current alt number, current - /// ATN state. + /// tracing, and the default values available for rule invocatons: + /// start, stop, rule index, current alt number. /// /// Subclasses made for each rule and grammar track the parameters, /// return values, locals, and labels specific to that rule. These @@ -66,14 +64,12 @@ namespace runtime { /// class ParserRuleContext : public RuleContext { public: - typedef std::shared_ptr Ref; - /// If we are debugging or building a parse tree for a visitor, /// we need to track all of the tokens and rule invocations associated /// with this rule's context. This is empty for parsing w/o tree constr. /// operation because we don't the need to track the details about /// how we parse this rule. - std::vector> children; + std::vector> children; /// /// For debugging/tracing purposes, we want to track all of the nodes in @@ -96,7 +92,7 @@ namespace runtime { /// // public List states; - Token::Ref start, stop; + Ref start, stop; /// The exception that forced this rule to return. If the rule successfully /// completed, this is "null exception pointer". @@ -105,38 +101,39 @@ namespace runtime { ParserRuleContext(); virtual ~ParserRuleContext() {} - /// - /// COPY a ctx (I'm deliberately not copy constructor) - virtual void copyFrom(std::shared_ptr ctx); + /** COPY a ctx (I'm deliberately not using copy constructor) to avoid + * confusion with creating node with parent. Does not copy children. + */ + virtual void copyFrom(Ref ctx); ParserRuleContext(std::weak_ptr parent, int invokingStateNumber); // Double dispatch methods for listeners - virtual void enterRule(std::shared_ptr listener); - virtual void exitRule(std::shared_ptr listener); + virtual void enterRule(Ref listener); + virtual void exitRule(Ref listener); /// Does not set parent link; other add methods do that. - virtual std::shared_ptr addChild(std::shared_ptr t); - virtual RuleContext::Ref addChild(RuleContext::Ref ruleInvocation); + virtual Ref addChild(Ref t); + virtual Ref addChild(Ref ruleInvocation); /// Used by enterOuterAlt to toss out a RuleContext previously added as /// we entered a rule. If we have # label, we will need to remove /// generic ruleContext object. virtual void removeLastChild(); - virtual std::shared_ptr addChild(Token::Ref matchedToken); + virtual Ref addChild(Ref matchedToken); - virtual std::shared_ptr addErrorNode(Token::Ref badToken); + virtual Ref addErrorNode(Ref badToken); std::weak_ptr getParent() { return std::dynamic_pointer_cast(getParentReference().lock()); }; - virtual std::shared_ptr getToken(int ttype, std::size_t i); + virtual Ref getToken(int ttype, std::size_t i); - virtual std::vector> getTokens(int ttype); + virtual std::vector> getTokens(int ttype); template - std::shared_ptr getRuleContext(size_t i) { + Ref getRuleContext(size_t i) { if (children.empty()) { return nullptr; } @@ -153,8 +150,8 @@ namespace runtime { } template - std::vector> getRuleContexts() { - std::vector> contexts; + std::vector> getRuleContexts() { + std::vector> contexts; for (auto &child : children) { if (antlrcpp::is(child)) { contexts.push_back(std::dynamic_pointer_cast(child)); @@ -167,15 +164,26 @@ namespace runtime { virtual std::size_t getChildCount() override; virtual misc::Interval getSourceInterval() override; - virtual Token::Ref getStart(); - virtual Token::Ref getStop(); + /** + * Get the initial token in this context. + * Note that the range from start to stop is inclusive, so for rules that do not consume anything + * (for example, zero length or error productions) this token may exceed stop. + */ + virtual Ref getStart(); + + /** + * Get the final token in this context. + * Note that the range from start to stop is inclusive, so for rules that do not consume anything + * (for example, zero length or error productions) this token may precede start. + */ + virtual Ref getStop(); /// /// Used for rule context info debugging during parse-time, not so much for ATN debugging virtual std::wstring toInfoString(Parser *recognizer); protected: - virtual std::shared_ptr getChildReference(size_t i) override; + virtual Ref getChildReference(size_t i) override; }; } // namespace runtime diff --git a/runtime/Cpp/runtime/ProxyErrorListener.cpp b/runtime/Cpp/runtime/ProxyErrorListener.cpp index ca35d6e9b..96ad5b540 100755 --- a/runtime/Cpp/runtime/ProxyErrorListener.cpp +++ b/runtime/Cpp/runtime/ProxyErrorListener.cpp @@ -49,7 +49,7 @@ void ProxyErrorListener::removeErrorListeners() { _delegates.clear(); } -void ProxyErrorListener::syntaxError(IRecognizer *recognizer, Token::Ref offendingSymbol, size_t line, +void ProxyErrorListener::syntaxError(IRecognizer *recognizer, Ref offendingSymbol, size_t line, int charPositionInLine, const std::wstring &msg, std::exception_ptr e) { for (auto listener : _delegates) { @@ -58,21 +58,21 @@ void ProxyErrorListener::syntaxError(IRecognizer *recognizer, Token::Ref offendi } void ProxyErrorListener::reportAmbiguity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, - bool exact, const antlrcpp::BitSet &ambigAlts, std::shared_ptr configs) { + bool exact, const antlrcpp::BitSet &ambigAlts, Ref configs) { for (auto listener : _delegates) { listener->reportAmbiguity(recognizer, dfa, startIndex, stopIndex, exact, ambigAlts, configs); } } void ProxyErrorListener::reportAttemptingFullContext(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, - size_t stopIndex, const antlrcpp::BitSet &conflictingAlts, std::shared_ptr configs) { + size_t stopIndex, const antlrcpp::BitSet &conflictingAlts, Ref configs) { for (auto listener : _delegates) { listener->reportAttemptingFullContext(recognizer, dfa, startIndex, stopIndex, conflictingAlts, configs); } } void ProxyErrorListener::reportContextSensitivity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, - int prediction, std::shared_ptr configs) { + int prediction, Ref configs) { for (auto listener : _delegates) { listener->reportContextSensitivity(recognizer, dfa, startIndex, stopIndex, prediction, configs); } diff --git a/runtime/Cpp/runtime/ProxyErrorListener.h b/runtime/Cpp/runtime/ProxyErrorListener.h index 5b1347bdf..deeb67d39 100755 --- a/runtime/Cpp/runtime/ProxyErrorListener.h +++ b/runtime/Cpp/runtime/ProxyErrorListener.h @@ -51,17 +51,17 @@ namespace runtime { void removeErrorListener(ANTLRErrorListener *listener); void removeErrorListeners(); - void syntaxError(IRecognizer *recognizer, Token::Ref offendingSymbol, size_t line, int charPositionInLine, + void syntaxError(IRecognizer *recognizer, Ref offendingSymbol, size_t line, int charPositionInLine, const std::wstring &msg, std::exception_ptr e) override; virtual void reportAmbiguity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, bool exact, - const antlrcpp::BitSet &ambigAlts, std::shared_ptr configs) override; + const antlrcpp::BitSet &ambigAlts, Ref configs) override; virtual void reportAttemptingFullContext(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, - const antlrcpp::BitSet &conflictingAlts, std::shared_ptr configs) override; + const antlrcpp::BitSet &conflictingAlts, Ref configs) override; virtual void reportContextSensitivity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, - int prediction, std::shared_ptr configs) override; + int prediction, Ref configs) override; }; } // namespace runtime diff --git a/runtime/Cpp/runtime/RecognitionException.cpp b/runtime/Cpp/runtime/RecognitionException.cpp index a38cf278e..1d73d63fe 100755 --- a/runtime/Cpp/runtime/RecognitionException.cpp +++ b/runtime/Cpp/runtime/RecognitionException.cpp @@ -33,18 +33,19 @@ #include "Recognizer.h" #include "Strings.h" #include "ParserRuleContext.h" +#include "IntervalSet.h" #include "RecognitionException.h" using namespace org::antlr::v4::runtime; RecognitionException::RecognitionException(IRecognizer *recognizer, IntStream *input, - ParserRuleContext::Ref ctx, Token::Ref offendingToken) + Ref ctx, Ref offendingToken) : RecognitionException("", recognizer, input, ctx, offendingToken) { } RecognitionException::RecognitionException(const std::string &message, IRecognizer *recognizer, IntStream *input, - ParserRuleContext::Ref ctx, Token::Ref offendingToken) + Ref ctx, Ref offendingToken) : RuntimeException(message), _recognizer(recognizer), _input(input), _offendingToken(offendingToken), _ctx(ctx) { InitializeInstanceFields(); if (recognizer != nullptr) { @@ -67,7 +68,7 @@ misc::IntervalSet RecognitionException::getExpectedTokens() const { return misc::IntervalSet::EMPTY_SET; } -RuleContext::Ref RecognitionException::getCtx() const { +Ref RecognitionException::getCtx() const { return _ctx; } @@ -75,7 +76,7 @@ IntStream* RecognitionException::getInputStream() const { return _input; } -Token::Ref RecognitionException::getOffendingToken() const { +Ref RecognitionException::getOffendingToken() const { return _offendingToken; } diff --git a/runtime/Cpp/runtime/RecognitionException.h b/runtime/Cpp/runtime/RecognitionException.h index 8ab430968..65e1bda2c 100755 --- a/runtime/Cpp/runtime/RecognitionException.h +++ b/runtime/Cpp/runtime/RecognitionException.h @@ -32,12 +32,6 @@ #pragma once #include "Exceptions.h" -#include "IntervalSet.h" -#include "IRecognizer.h" -#include "IntStream.h" -#include "RuleContext.h" -#include "ParserRuleContext.h" -#include "Token.h" namespace org { namespace antlr { @@ -54,20 +48,20 @@ namespace runtime { /// The Recognizer where this exception originated. IRecognizer *_recognizer; IntStream *_input; - ParserRuleContext::Ref _ctx; + Ref _ctx; /// The current Token when an error occurred. Since not all streams /// support accessing symbols by index, we have to track the Token /// instance itself. - Token::Ref _offendingToken; + Ref _offendingToken; int _offendingState; public: - RecognitionException(IRecognizer *recognizer, IntStream *input, ParserRuleContext::Ref ctx, - Token::Ref offendingToken = Token::Ref()); + RecognitionException(IRecognizer *recognizer, IntStream *input, Ref ctx, + Ref offendingToken = Ref()); RecognitionException(const std::string &message, IRecognizer *recognizer, IntStream *input, - ParserRuleContext::Ref ctx, Token::Ref offendingToken = Token::Ref()); + Ref ctx, Ref offendingToken = Ref()); /// Get the ATN state number the parser was in at the time the error /// occurred. For NoViableAltException and @@ -99,7 +93,7 @@ namespace runtime { /// /// The at the time this exception was thrown. /// If the context is not available, this method returns {@code null}. - virtual RuleContext::Ref getCtx() const; + virtual Ref getCtx() const; /// /// Gets the input stream which is the symbol source for the recognizer where @@ -112,7 +106,7 @@ namespace runtime { /// available. virtual IntStream* getInputStream() const; - virtual Token::Ref getOffendingToken() const; + virtual Ref getOffendingToken() const; /// /// Gets the where this exception occurred. diff --git a/runtime/Cpp/runtime/Recognizer.cpp b/runtime/Cpp/runtime/Recognizer.cpp index a2de2f5bd..36e9a8160 100755 --- a/runtime/Cpp/runtime/Recognizer.cpp +++ b/runtime/Cpp/runtime/Recognizer.cpp @@ -34,13 +34,18 @@ #include "CPPUtils.h" #include "Strings.h" #include "Token.h" +#include "ATN.h" +#include "ATNSimulator.h" #include "CPPUtils.h" +#include "Vocabulary.h" +#include "VocabularyImpl.h" + #include "Recognizer.h" using namespace org::antlr::v4::runtime; -std::map, std::map> Recognizer::_tokenTypeMapCache; +std::map, std::map> Recognizer::_tokenTypeMapCache; std::map, std::map> Recognizer::_ruleIndexMapCache; Recognizer::Recognizer() { @@ -48,21 +53,32 @@ Recognizer::Recognizer() { _proxListener.addErrorListener(&ConsoleErrorListener::INSTANCE); } +Ref Recognizer::getVocabulary() const { + return dfa::VocabularyImpl::fromTokenNames(getTokenNames()); +} + std::map Recognizer::getTokenTypeMap() { - const std::vector& tokenNames = getTokenNames(); - if (tokenNames.empty()) { - throw L"The current recognizer does not provide a list of token names."; - } + Ref vocabulary = getVocabulary(); std::lock_guard lck(mtx); std::map result; - auto iterator = _tokenTypeMapCache.find(tokenNames); + auto iterator = _tokenTypeMapCache.find(vocabulary); if (iterator != _tokenTypeMapCache.end()) { result = iterator->second; } else { - result = antlrcpp::toMap(tokenNames); + for (size_t i = 0; i < getATN().maxTokenType; ++i) { + std::wstring literalName = vocabulary->getLiteralName(i); + if (!literalName.empty()) { + result[literalName] = i; + } + + std::wstring symbolicName = vocabulary->getSymbolicName(i); + if (!symbolicName.empty()) { + result[symbolicName] = i; + } + } result[L"EOF"] = EOF; - _tokenTypeMapCache[tokenNames] = result; + _tokenTypeMapCache[vocabulary] = result; } return result; @@ -95,6 +111,17 @@ size_t Recognizer::getTokenType(const std::wstring &tokenName) { return iterator->second; } +Ref Recognizer::getParseInfo() const { + return nullptr; +} + +void Recognizer::setInterpreter(atn::ATNSimulator *interpreter) { + // Usually the interpreter is set by the descendant (lexer or parser (simulator), but can also be exchanged + // by the profiling ATN simulator. + delete _interpreter; + _interpreter = interpreter; +} + std::wstring Recognizer::getErrorHeader(RecognitionException *e) { // We're having issues with cross header dependencies, these two classes will need to be // rewritten to remove that. @@ -106,7 +133,7 @@ std::wstring Recognizer::getErrorHeader(RecognitionException *e) { std::wstring Recognizer::getTokenErrorDisplay(Token *t) { if (t == nullptr) { - return L""; + return L""; } std::wstring s = t->getText(); if (s == L"") { @@ -140,15 +167,15 @@ ProxyErrorListener& Recognizer::getErrorListenerDispatch() { return _proxListener; } -bool Recognizer::sempred(RuleContext::Ref localctx, int ruleIndex, int actionIndex) { +bool Recognizer::sempred(Ref localctx, int ruleIndex, int actionIndex) { return true; } -bool Recognizer::precpred(RuleContext::Ref localctx, int precedence) { +bool Recognizer::precpred(Ref localctx, int precedence) { return true; } -void Recognizer::action(RuleContext::Ref localctx, int ruleIndex, int actionIndex) { +void Recognizer::action(Ref localctx, int ruleIndex, int actionIndex) { } int Recognizer::getState() { diff --git a/runtime/Cpp/runtime/Recognizer.h b/runtime/Cpp/runtime/Recognizer.h index a2d61ca47..66e3b3277 100755 --- a/runtime/Cpp/runtime/Recognizer.h +++ b/runtime/Cpp/runtime/Recognizer.h @@ -43,31 +43,23 @@ namespace runtime { public: Recognizer(); - private: - static std::map, std::map> _tokenTypeMapCache; - static std::map, std::map> _ruleIndexMapCache; - - ProxyErrorListener _proxListener; // Manages a collection of listeners. - - // Mutex to manage synchronized access for multithreading. - std::mutex mtx; - - protected: - atn::ATNSimulator *_interpreter; // Set and deleted in descendants. - - private: - int _stateNumber; - - /// - /// Used to print out token names like ID during debugging and - /// error reporting. The generated parsers implement a method - /// that overrides this to point to their generated token names list. - /// - public: + /** Used to print out token names like ID during debugging and + * error reporting. The generated parsers implement a method + * that overrides this to point to their String[] tokenNames. + * + * @deprecated Use {@link #getVocabulary()} instead. + */ virtual const std::vector& getTokenNames() const = 0; - virtual const std::vector& getRuleNames() const = 0; + /** + * Get the vocabulary used by the recognizer. + * + * @return A {@link Vocabulary} instance providing information about the + * vocabulary used by the grammar. + */ + Ref getVocabulary() const; + /// /// Get a map from token names to token types. ///

@@ -108,19 +100,37 @@ namespace runtime { return dynamic_cast(_interpreter); } - ///

- /// What is the error header, normally line/character position information? + /** If profiling during the parse/lex, this will return DecisionInfo records + * for each decision in recognizer in a ParseInfo object. + * + * @since 4.3 + */ + virtual Ref getParseInfo() const; + + /** + * Set the ATN interpreter used by the recognizer for prediction. + * + * @param interpreter The ATN interpreter used by the recognizer for + * prediction. + */ + void setInterpreter(atn::ATNSimulator *interpreter); + + /// What is the error header, normally line/character position information? virtual std::wstring getErrorHeader(RecognitionException *e); - /// - /// How should a token be displayed in an error message? The default - /// is to display just the text, but during development you might - /// want to have a lot of information spit out. Override in that case - /// to use t.toString() (which, for CommonToken, dumps everything about - /// the token). This is better than forcing you to override a method in - /// your token objects because you don't have to go modify your lexer - /// so that it creates a new Java type. - /// + /** How should a token be displayed in an error message? The default + * is to display just the text, but during development you might + * want to have a lot of information spit out. Override in that case + * to use t.toString() (which, for CommonToken, dumps everything about + * the token). This is better than forcing you to override a method in + * your token objects because you don't have to go modify your lexer + * so that it creates a new Java type. + * + * @deprecated This method is not called by the ANTLR 4 Runtime. Specific + * implementations of {@link ANTLRErrorStrategy} may provide a similar + * feature when necessary. For example, see + * {@link DefaultErrorStrategy#getTokenErrorDisplay}. + */ virtual std::wstring getTokenErrorDisplay(Token *t); /// if {@code listener} is {@code null}. @@ -134,11 +144,11 @@ namespace runtime { // subclass needs to override these if there are sempreds or actions // that the ATN interp needs to execute - virtual bool sempred(RuleContext::Ref localctx, int ruleIndex, int actionIndex); + virtual bool sempred(Ref localctx, int ruleIndex, int actionIndex); - virtual bool precpred(RuleContext::Ref localctx, int precedence); + virtual bool precpred(Ref localctx, int precedence); - virtual void action(RuleContext::Ref localctx, int ruleIndex, int actionIndex); + virtual void action(Ref localctx, int ruleIndex, int actionIndex); int getState(); @@ -156,12 +166,25 @@ namespace runtime { virtual void setInputStream(IntStream *input) = 0; - virtual std::shared_ptr> getTokenFactory() = 0; + virtual Ref> getTokenFactory() = 0; template void setTokenFactory(TokenFactory *input); + protected: + atn::ATNSimulator *_interpreter; // Set and deleted in descendants (or the profiler). + private: + static std::map, std::map> _tokenTypeMapCache; + static std::map, std::map> _ruleIndexMapCache; + + ProxyErrorListener _proxListener; // Manages a collection of listeners. + + // Mutex to manage synchronized access for multithreading. + std::mutex mtx; + + int _stateNumber; + void InitializeInstanceFields(); }; diff --git a/runtime/Cpp/runtime/RuleContext.cpp b/runtime/Cpp/runtime/RuleContext.cpp index ddfd90322..fccad4622 100755 --- a/runtime/Cpp/runtime/RuleContext.cpp +++ b/runtime/Cpp/runtime/RuleContext.cpp @@ -32,12 +32,13 @@ #include "Trees.h" #include "Interval.h" #include "Parser.h" +#include "ATN.h" #include "RuleContext.h" using namespace org::antlr::v4::runtime; -const RuleContext::Ref RuleContext::EMPTY = std::make_shared(); +const Ref RuleContext::EMPTY = std::make_shared(); RuleContext::RuleContext() { InitializeInstanceFields(); @@ -51,7 +52,7 @@ RuleContext::RuleContext(std::weak_ptr parent, int invokingState) { int RuleContext::depth() { int n = 1; - RuleContext::Ref p = shared_from_this(); + Ref p = shared_from_this(); while (true) { if (p->parent.expired()) break; @@ -69,7 +70,7 @@ misc::Interval RuleContext::getSourceInterval() { return misc::Interval::INVALID; } -RuleContext::Ref RuleContext::getRuleContext() { +Ref RuleContext::getRuleContext() { return shared_from_this(); } @@ -97,43 +98,22 @@ ssize_t RuleContext::getRuleIndex() const { return -1; } -std::shared_ptr RuleContext::getChildReference(size_t i) { - return std::shared_ptr(); +Ref RuleContext::getChildReference(size_t i) { + return Ref(); } +int RuleContext::getAltNumber() const { + return atn::ATN::INVALID_ALT_NUMBER; +} + +void RuleContext::setAltNumber(int altNumber) { +} + std::size_t RuleContext::getChildCount() { return 0; } -void RuleContext::save(Parser *parser, const std::wstring &fileName) { - std::vector ruleNames; - if (parser != nullptr) { - ruleNames = parser->getRuleNames(); - } - save(ruleNames, fileName); -} - -void RuleContext::save(Parser *parser, const std::wstring &fileName, const std::wstring &fontName, int fontSize) { - std::vector ruleNames; - if (parser != nullptr) { - ruleNames = parser->getRuleNames(); - } - save(ruleNames, fileName, fontName, fontSize); -} - -void RuleContext::save(std::vector &ruleNames, const std::wstring &fileName) { -#ifdef TODO - tree::Trees::writePS(this, ruleNames, fileName); -#endif -} - -void RuleContext::save(std::vector &ruleNames, const std::wstring &fileName, const std::wstring &fontName, int fontSize) { -#ifdef TODO - tree::Trees::writePS(this, ruleNames, fileName, fontName, fontSize); -#endif -} - std::wstring RuleContext::toStringTree(Parser *recog) { return tree::Trees::toStringTree(shared_from_this(), recog); } @@ -148,14 +128,14 @@ std::wstring RuleContext::toStringTree() { std::wstring RuleContext::toString(const std::vector &ruleNames) { - return toString(ruleNames, RuleContext::Ref()); + return toString(ruleNames, Ref()); } -std::wstring RuleContext::toString(const std::vector &ruleNames, RuleContext::Ref stop) { +std::wstring RuleContext::toString(const std::vector &ruleNames, Ref stop) { std::wstringstream ss; - RuleContext::Ref parent = shared_from_this(); + Ref parent = shared_from_this(); ss << L"["; while (parent != stop) { if (ruleNames.empty()) { @@ -190,7 +170,7 @@ std::wstring RuleContext::toString(Recognizer *recog) { return toString(recog, ParserRuleContext::EMPTY); } -std::wstring RuleContext::toString(Recognizer *recog, RuleContext::Ref stop) { +std::wstring RuleContext::toString(Recognizer *recog, Ref stop) { if (recog == nullptr) return toString({}, stop); return toString(recog->getRuleNames(), stop); diff --git a/runtime/Cpp/runtime/RuleContext.h b/runtime/Cpp/runtime/RuleContext.h index 5af69363e..1be67d0c7 100755 --- a/runtime/Cpp/runtime/RuleContext.h +++ b/runtime/Cpp/runtime/RuleContext.h @@ -38,41 +38,66 @@ namespace antlr { namespace v4 { namespace runtime { - /// - /// A rule context is a record of a single rule invocation. It knows - /// which context invoked it, if any. If there is no parent context, then - /// naturally the invoking state is not valid. The parent link - /// provides a chain upwards from the current rule invocation to the root - /// of the invocation tree, forming a stack. We actually carry no - /// information about the rule associated with this context (except - /// when parsing). We keep only the state number of the invoking state from - /// the ATN submachine that invoked this. Contrast this with the s - /// pointer inside ParserRuleContext that tracks the current state - /// being "executed" for the current rule. - /// - /// The parent contexts are useful for computing lookahead sets and - /// getting error information. - /// - /// These objects are used during parsing and prediction. - /// For the special case of parsers, we use the subclass - /// ParserRuleContext. - /// - /// + /** A rule context is a record of a single rule invocation. + * + * We form a stack of these context objects using the parent + * pointer. A parent pointer of null indicates that the current + * context is the bottom of the stack. The ParserRuleContext subclass + * as a children list so that we can turn this data structure into a + * tree. + * + * The root node always has a null pointer and invokingState of -1. + * + * Upon entry to parsing, the first invoked rule function creates a + * context object (asubclass specialized for that rule such as + * SContext) and makes it the root of a parse tree, recorded by field + * Parser._ctx. + * + * public final SContext s() throws RecognitionException { + * SContext _localctx = new SContext(_ctx, getState()); <-- create new node + * enterRule(_localctx, 0, RULE_s); <-- push it + * ... + * exitRule(); <-- pop back to _localctx + * return _localctx; + * } + * + * A subsequent rule invocation of r from the start rule s pushes a + * new context object for r whose parent points at s and use invoking + * state is the state with r emanating as edge label. + * + * The invokingState fields from a context object to the root + * together form a stack of rule indication states where the root + * (bottom of the stack) has a -1 sentinel value. If we invoke start + * symbol s then call r1, which calls r2, the would look like + * this: + * + * SContext[-1] <- root node (bottom of the stack) + * R1Context[p] <- p in rule s called r1 + * R2Context[q] <- q in rule r1 called r2 + * + * So the top of the stack, _ctx, represents a call to the current + * rule and it holds the return address from another rule that invoke + * to this rule. To invoke a rule, we must always have a current context. + * + * The parent contexts are useful for computing lookahead sets and + * getting error information. + * + * These objects are used during parsing and prediction. + * For the special case of parsers, we use the subclass + * ParserRuleContext. + * + * @see ParserRuleContext + */ class RuleContext : public tree::RuleNode, public std::enable_shared_from_this { public: - typedef std::shared_ptr Ref; - - static const RuleContext::Ref EMPTY; + static const Ref EMPTY; - /// - /// What context invoked this rule? + /// What context invoked this rule? std::weak_ptr parent; - /// /// What state invoked the rule associated with this context? - /// The "return address" is the followState of invokingState - /// If parent is null, this should be -1. - /// + /// The "return address" is the followState of invokingState + /// If parent is null, this should be -1 and this context object represents the start rule. int invokingState; RuleContext(); @@ -81,18 +106,39 @@ namespace runtime { virtual int depth(); - /// A context is empty if there is no invoking state; meaning nobody call current context. + /// A context is empty if there is no invoking state; meaning nobody called current context. virtual bool isEmpty(); // satisfy the ParseTree / SyntaxTree interface virtual misc::Interval getSourceInterval() override; - virtual RuleContext::Ref getRuleContext() override; + virtual Ref getRuleContext() override; virtual std::wstring getText() override; virtual ssize_t getRuleIndex() const; + /** For rule associated with this parse tree internal node, return + * the outer alternative number used to match the input. Default + * implementation does not compute nor store this alt num. Create + * a subclass of ParserRuleContext with backing field and set + * option contextSuperClass. + * to set it. + * + * @since 4.5.3 + */ + virtual int getAltNumber() const; + + /** Set the outer alternative number for this context node. Default + * implementation does nothing to avoid backing field overhead for + * trees that don't need it. Create + * a subclass of ParserRuleContext with backing field and set + * option contextSuperClass. + * + * @since 4.5.3 + */ + virtual void setAltNumber(int altNumber); + virtual std::size_t getChildCount() override; template @@ -100,22 +146,6 @@ namespace runtime { return visitor->visitChildren(this); } - /// - /// Save this tree in a postscript file - virtual void save(Parser *parser, const std::wstring &fileName); - - /// - /// Save this tree in a postscript file using a particular font name and size - virtual void save(Parser *parser, const std::wstring &fileName, const std::wstring &fontName, int fontSize); - - /// - /// Save this tree in a postscript file - virtual void save(std::vector &ruleNames, const std::wstring &fileName); - - /// - /// Save this tree in a postscript file using a particular font name and size - virtual void save(std::vector &ruleNames, const std::wstring &fileName, const std::wstring &fontName, int fontSize); - /// /// Print out a whole tree, not just a node, in LISP format /// (root child1 .. childN). Print just a node if this is a leaf. @@ -135,15 +165,15 @@ namespace runtime { std::wstring toString(const std::vector &ruleNames); // recog null unless ParserRuleContext, in which case we use subclass toString(...) - std::wstring toString(Recognizer *recog, RuleContext::Ref stop); + std::wstring toString(Recognizer *recog, Ref stop); - virtual std::wstring toString(const std::vector &ruleNames, RuleContext::Ref stop); + virtual std::wstring toString(const std::vector &ruleNames, Ref stop); bool operator == (const RuleContext &other) { return this == &other; } // Simple address comparison. protected: virtual std::weak_ptr getParentReference() override; - virtual std::shared_ptr getChildReference(size_t i) override; + virtual Ref getChildReference(size_t i) override; private: void InitializeInstanceFields(); diff --git a/runtime/Cpp/runtime/Token.h b/runtime/Cpp/runtime/Token.h index 98218a72f..9e960007c 100755 --- a/runtime/Cpp/runtime/Token.h +++ b/runtime/Cpp/runtime/Token.h @@ -45,8 +45,6 @@ namespace runtime { /// class Token { public: - typedef std::shared_ptr Ref; - static const size_t INVALID_TYPE = 0; /// diff --git a/runtime/Cpp/runtime/TokenFactory.h b/runtime/Cpp/runtime/TokenFactory.h index ce82e2155..8be9e3ca4 100755 --- a/runtime/Cpp/runtime/TokenFactory.h +++ b/runtime/Cpp/runtime/TokenFactory.h @@ -47,10 +47,10 @@ namespace runtime { /// are wiped to -1 in the text override is set in the CommonToken. /// public: - virtual std::shared_ptr create(std::pair source, int type, const std::wstring &text, int channel, int start, int stop, int line, int charPositionInLine) = 0; + virtual Ref create(std::pair source, int type, const std::wstring &text, int channel, int start, int stop, int line, int charPositionInLine) = 0; /// Generically useful - virtual std::shared_ptr create(int type, const std::wstring &text) = 0; + virtual Ref create(int type, const std::wstring &text) = 0; }; } // namespace runtime diff --git a/runtime/Cpp/runtime/TokenSource.h b/runtime/Cpp/runtime/TokenSource.h index 7d0cc82be..0a6d1d924 100755 --- a/runtime/Cpp/runtime/TokenSource.h +++ b/runtime/Cpp/runtime/TokenSource.h @@ -31,7 +31,6 @@ #pragma once -#include "Token.h" #include "TokenFactory.h" namespace org { @@ -62,7 +61,7 @@ namespace runtime { /// to the parser. /// public: - virtual Token::Ref nextToken() = 0; + virtual Ref nextToken() = 0; /// /// Get the line number for the current position in the input stream. The @@ -109,7 +108,7 @@ namespace runtime { /// creating objects from the input. /// /// The currently used by this token source. - virtual std::shared_ptr> getTokenFactory() = 0; + virtual Ref> getTokenFactory() = 0; }; } // namespace runtime diff --git a/runtime/Cpp/runtime/TokenStream.h b/runtime/Cpp/runtime/TokenStream.h index ee304df55..197e21168 100755 --- a/runtime/Cpp/runtime/TokenStream.h +++ b/runtime/Cpp/runtime/TokenStream.h @@ -31,7 +31,6 @@ #pragma once -#include "Token.h" #include "IntStream.h" namespace org { @@ -54,7 +53,7 @@ namespace runtime { public: virtual ~TokenStream(); - virtual Token::Ref LT(ssize_t k) = 0; + virtual Ref LT(ssize_t k) = 0; /// /// Gets the at the specified {@code index} in the stream. When @@ -73,7 +72,7 @@ namespace runtime { /// if {code index} is less than 0 /// if the stream does not support /// retrieving the token at the specified index - virtual Token::Ref get(size_t index) const = 0; + virtual Ref get(size_t index) const = 0; /// Gets the underlying TokenSource which provides tokens for this stream. virtual TokenSource* getTokenSource() const = 0; @@ -161,7 +160,7 @@ namespace runtime { /// /// if this stream does not support /// this method for the specified tokens - virtual std::wstring getText(Token::Ref start, Token::Ref stop) = 0; + virtual std::wstring getText(Ref start, Ref stop) = 0; }; } // namespace runtime diff --git a/runtime/Cpp/runtime/TokenStreamRewriter.cpp b/runtime/Cpp/runtime/TokenStreamRewriter.cpp index 46330a553..31f54bc9c 100755 --- a/runtime/Cpp/runtime/TokenStreamRewriter.cpp +++ b/runtime/Cpp/runtime/TokenStreamRewriter.cpp @@ -299,7 +299,7 @@ std::wstring TokenStreamRewriter::getText(const std::wstring &programName, const while (i <= (size_t)stop && i < tokens->size()) { RewriteOperation *op = indexToOp[i]; indexToOp.erase(i); // remove so any left have index size-1 - Token::Ref t = tokens->get(i); + Ref t = tokens->get(i); if (op == nullptr) { // no operation at that index, just dump token if (t->getType() != EOF) { diff --git a/runtime/Cpp/runtime/UnbufferedTokenStream.cpp b/runtime/Cpp/runtime/UnbufferedTokenStream.cpp index fa5a21092..ae97c61f4 100755 --- a/runtime/Cpp/runtime/UnbufferedTokenStream.cpp +++ b/runtime/Cpp/runtime/UnbufferedTokenStream.cpp @@ -54,7 +54,7 @@ UnbufferedTokenStream::UnbufferedTokenStream(TokenSource *tokenSource, int buffe UnbufferedTokenStream::~UnbufferedTokenStream() { } -Token::Ref UnbufferedTokenStream::get(size_t i) const +Ref UnbufferedTokenStream::get(size_t i) const { // get absolute index size_t bufferStartIndex = getBufferStartIndex(); if (i < bufferStartIndex || i >= bufferStartIndex + _tokens.size()) { @@ -64,7 +64,7 @@ Token::Ref UnbufferedTokenStream::get(size_t i) const return _tokens[i - bufferStartIndex]; } -Token::Ref UnbufferedTokenStream::LT(ssize_t i) +Ref UnbufferedTokenStream::LT(ssize_t i) { if (i == -1) { return _lastToken; @@ -104,7 +104,7 @@ std::wstring UnbufferedTokenStream::getText(RuleContext* ctx) return getText(ctx->getSourceInterval()); } -std::wstring UnbufferedTokenStream::getText(Token::Ref start, Token::Ref stop) +std::wstring UnbufferedTokenStream::getText(Ref start, Ref stop) { return getText(misc::Interval(start->getTokenIndex(), stop->getTokenIndex())); } @@ -156,16 +156,16 @@ size_t UnbufferedTokenStream::fill(size_t n) return i; } - Token::Ref t = _tokenSource->nextToken(); + Ref t = _tokenSource->nextToken(); add(t); } return n; } -void UnbufferedTokenStream::add(Token::Ref t) +void UnbufferedTokenStream::add(Ref t) { - WritableToken::Ref writable = std::dynamic_pointer_cast(t); + Ref writable = std::dynamic_pointer_cast(t); if (writable) { writable->setTokenIndex(int(getBufferStartIndex() + _tokens.size())); } @@ -203,7 +203,7 @@ void UnbufferedTokenStream::release(ssize_t marker) if (_p > 0) { // Copy tokens[p]..tokens[n-1] to tokens[0]..tokens[(n-1)-p], reset ptrs // p is last valid token; move nothing if p==n as we have no valid char - std::vector(_tokens.begin() + (ssize_t)_p, _tokens.end()).swap(_tokens); + std::vector>(_tokens.begin() + (ssize_t)_p, _tokens.end()).swap(_tokens); _p = 0; } @@ -274,7 +274,7 @@ std::wstring UnbufferedTokenStream::getText(const misc::Interval &interval) std::wstringstream ss; for (size_t i = a; i <= b; i++) { - Token::Ref t = _tokens[i]; + Ref t = _tokens[i]; if (i > 0) ss << L", "; ss << t->getText(); diff --git a/runtime/Cpp/runtime/UnbufferedTokenStream.h b/runtime/Cpp/runtime/UnbufferedTokenStream.h index 60d0fe542..5a6d24c47 100755 --- a/runtime/Cpp/runtime/UnbufferedTokenStream.h +++ b/runtime/Cpp/runtime/UnbufferedTokenStream.h @@ -44,8 +44,8 @@ namespace runtime { UnbufferedTokenStream(TokenSource *tokenSource, int bufferSize); virtual ~UnbufferedTokenStream(); - virtual Token::Ref get(size_t i) const override; - virtual Token::Ref LT(ssize_t i) override; + virtual Ref get(size_t i) const override; + virtual Ref LT(ssize_t i) override; virtual ssize_t LA(ssize_t i) override; virtual TokenSource* getTokenSource() const override; @@ -53,7 +53,7 @@ namespace runtime { virtual std::wstring getText(const misc::Interval &interval) override; virtual std::wstring getText() override; virtual std::wstring getText(RuleContext *ctx) override; - virtual std::wstring getText(Token::Ref start, Token::Ref stop) override; + virtual std::wstring getText(Ref start, Ref stop) override; virtual void consume() override; @@ -83,7 +83,7 @@ namespace runtime { /// we start filling at index 0 again. /// - std::vector _tokens; + std::vector> _tokens; /// /// 0..n-1 index into of next token. @@ -104,13 +104,13 @@ namespace runtime { /// /// This is the {@code LT(-1)} token for the current position. /// - Token::Ref _lastToken; + Ref _lastToken; /// /// When {@code numMarkers > 0}, this is the {@code LT(-1)} token for the /// first token in . Otherwise, this is {@code null}. /// - Token::Ref _lastTokenBufferStart; + Ref _lastTokenBufferStart; /// /// Absolute token index. It's the index of the token about to be read via @@ -130,7 +130,7 @@ namespace runtime { /// then EOF was reached before {@code n} tokens could be added. /// virtual size_t fill(size_t n); - virtual void add(Token::Ref t); + virtual void add(Ref t); size_t getBufferStartIndex() const; diff --git a/runtime/Cpp/runtime/Vocabulary.h b/runtime/Cpp/runtime/Vocabulary.h new file mode 100755 index 000000000..0716d0b1b --- /dev/null +++ b/runtime/Cpp/runtime/Vocabulary.h @@ -0,0 +1,160 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Dan McLaughlin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +namespace org { +namespace antlr { +namespace v4 { +namespace runtime { +namespace dfa { + + /// This interface provides information about the vocabulary used by a + /// recognizer. + /// + /// + class Vocabulary { + public: + /// + /// Returns the highest token type value. It can be used to iterate from + /// zero to that number, inclusively, thus querying all stored entries. + /// the highest token type value + virtual int getMaxTokenType() const = 0; + + /// + /// Gets the string literal associated with a token type. The string returned + /// by this method, when not {@code null}, can be used unaltered in a parser + /// grammar to represent this token type. + /// + /// The following table shows examples of lexer rules and the literal + /// names assigned to the corresponding token types. + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + ///
RuleLiteral NameJava String Literal
{@code THIS : 'this';}{@code 'this'}{@code "'this'"}
{@code SQUOTE : '\'';}{@code '\''}{@code "'\\''"}
{@code ID : [A-Z]+;}n/a{@code null}
+ /// + /// The token type. + /// + /// The string literal associated with the specified token type, or + /// {@code null} if no string literal is associated with the type. + virtual std::wstring getLiteralName(ssize_t tokenType) const = 0; + + /// + /// Gets the symbolic name associated with a token type. The string returned + /// by this method, when not {@code null}, can be used unaltered in a parser + /// grammar to represent this token type. + /// + /// This method supports token types defined by any of the following + /// methods: + /// + ///
    + ///
  • Tokens created by lexer rules.
    • + ///
  • Tokens defined in a tokens{} block in a lexer or parser + /// grammar.
    • + ///
  • The implicitly defined {@code EOF} token, which has the token type + /// .
    • + ///
+ /// + /// The following table shows examples of lexer rules and the literal + /// names assigned to the corresponding token types. + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + ///
RuleSymbolic Name
{@code THIS : 'this';}{@code THIS}
{@code SQUOTE : '\'';}{@code SQUOTE}
{@code ID : [A-Z]+;}{@code ID}
+ /// + /// The token type. + /// + /// The symbolic name associated with the specified token type, or + /// {@code null} if no symbolic name is associated with the type. + virtual std::wstring getSymbolicName(ssize_t tokenType) const = 0; + + /// + /// Gets the display name of a token type. + /// + /// ANTLR provides a default implementation of this method, but + /// applications are free to override the behavior in any manner which makes + /// sense for the application. The default implementation returns the first + /// result from the following list which produces a non-{@code null} + /// result. + /// + ///
    + ///
  1. The result of
    2. + ///
  2. The result of
    3. + ///
  3. The result of
    4. + ///
+ /// + /// The token type. + /// + /// The display name of the token type, for use in error reporting or + /// other user-visible messages which reference specific token types. + virtual std::wstring getDisplayName(ssize_t tokenType) const = 0; + }; + +} // namespace atn +} // namespace runtime +} // namespace v4 +} // namespace antlr +} // namespace org diff --git a/runtime/Cpp/runtime/VocabularyImpl.cpp b/runtime/Cpp/runtime/VocabularyImpl.cpp new file mode 100755 index 000000000..cf4d60b32 --- /dev/null +++ b/runtime/Cpp/runtime/VocabularyImpl.cpp @@ -0,0 +1,127 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Dan McLaughlin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "VocabularyImpl.h" + +using namespace org::antlr::v4::runtime::dfa; + +const std::vector VocabularyImpl::EMPTY_NAMES; +const Ref VocabularyImpl::EMPTY_VOCABULARY = std::make_shared(EMPTY_NAMES, EMPTY_NAMES, EMPTY_NAMES); + +VocabularyImpl::VocabularyImpl(const std::vector &literalNames, const std::vector &symbolicNames) +: VocabularyImpl(literalNames, symbolicNames, {}) { +} + +VocabularyImpl::VocabularyImpl(const std::vector &literalNames, + const std::vector &symbolicNames, const std::vector &displayNames) + : _literalNames(!literalNames.empty() ? literalNames : EMPTY_NAMES), + _symbolicNames(!symbolicNames.empty() ? symbolicNames : EMPTY_NAMES), + _displayNames(!displayNames.empty() ? displayNames : EMPTY_NAMES), + _maxTokenType(std::max((int)_displayNames.size(), std::max((int)_literalNames.size(), (int)_symbolicNames.size())) - 1) { + // See note here on -1 part: https://github.com/antlr/antlr4/pull/1146 +} + +Ref VocabularyImpl::fromTokenNames(const std::vector &tokenNames) { + if (tokenNames.empty()) { + return EMPTY_VOCABULARY; + } + + std::vector literalNames = tokenNames; + std::vector symbolicNames = tokenNames; + for (size_t i = 0; i < tokenNames.size(); i++) { + std::wstring tokenName = tokenNames[i]; + if (tokenName == L"") { + continue; + } + + if (!tokenName.empty()) { + wchar_t firstChar = tokenName[0]; + if (firstChar == L'\'') { + symbolicNames[i] = L""; + continue; + } else if (std::isupper(firstChar)) { + literalNames[i] = L""; + continue; + } + } + + // wasn't a literal or symbolic name + literalNames[i] = L""; + symbolicNames[i] = L""; + } + + return std::make_shared(literalNames, symbolicNames, tokenNames); +} + +int VocabularyImpl::getMaxTokenType() const { + return _maxTokenType; +} + +std::wstring VocabularyImpl::getLiteralName(ssize_t tokenType) const { + if (tokenType >= 0 && tokenType < (int)_literalNames.size()) { + return _literalNames[tokenType]; + } + + return L""; +} + +std::wstring VocabularyImpl::getSymbolicName(ssize_t tokenType) const { + if (tokenType >= 0 && tokenType < (int)_symbolicNames.size()) { + return _symbolicNames[tokenType]; + } + + if (tokenType == EOF) { + return L"EOF"; + } + + return L""; +} + +std::wstring VocabularyImpl::getDisplayName(ssize_t tokenType) const { + if (tokenType >= 0 && tokenType < (int)_displayNames.size()) { + std::wstring displayName = _displayNames[tokenType]; + if (!displayName.empty()) { + return displayName; + } + } + + std::wstring literalName = getLiteralName(tokenType); + if (!literalName.empty()) { + return literalName; + } + + std::wstring symbolicName = getSymbolicName(tokenType); + if (!symbolicName.empty()) { + return symbolicName; + } + + return std::to_wstring(tokenType); +} diff --git a/runtime/Cpp/runtime/VocabularyImpl.h b/runtime/Cpp/runtime/VocabularyImpl.h new file mode 100755 index 000000000..2eb506e13 --- /dev/null +++ b/runtime/Cpp/runtime/VocabularyImpl.h @@ -0,0 +1,119 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Dan McLaughlin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "Vocabulary.h" + +namespace org { +namespace antlr { +namespace v4 { +namespace runtime { +namespace dfa { + + /// This class provides a default implementation of the + /// interface. + class VocabularyImpl : public Vocabulary { + public: + /// Gets an empty instance. + /// + /// + /// No literal or symbol names are assigned to token types, so + /// returns the numeric value for all tokens + /// except . + static const Ref EMPTY_VOCABULARY; + + /// + /// Constructs a new instance of from the specified + /// literal and symbolic token names. + /// + /// The literal names assigned to tokens, or {@code null} + /// if no literal names are assigned. + /// The symbolic names assigned to tokens, or + /// {@code null} if no symbolic names are assigned. + /// + /// + /// + VocabularyImpl(const std::vector &literalNames, const std::vector &symbolicNames); + + /// + /// Constructs a new instance of from the specified + /// literal, symbolic, and display token names. + /// + /// The literal names assigned to tokens, or {@code null} + /// if no literal names are assigned. + /// The symbolic names assigned to tokens, or + /// {@code null} if no symbolic names are assigned. + /// The display names assigned to tokens, or {@code null} + /// to use the values in {@code literalNames} and {@code symbolicNames} as + /// the source of display names, as described in + /// . + /// + /// + /// + /// + VocabularyImpl(const std::vector &literalNames, const std::vector &symbolicNames, + const std::vector &displayNames); + + /// + /// Returns a instance from the specified set of token + /// names. This method acts as a compatibility layer for the single + /// {@code tokenNames} array generated by previous releases of ANTLR. + /// + /// The resulting vocabulary instance returns {@code null} for + /// and , and the + /// value from {@code tokenNames} for the display names. + /// + /// The token names, or {@code null} if no token names are + /// available. + /// A instance which uses {@code tokenNames} for + /// the display names of tokens. + static Ref fromTokenNames(const std::vector &tokenNames); + + virtual int getMaxTokenType() const override; + virtual std::wstring getLiteralName(ssize_t tokenType) const override; + virtual std::wstring getSymbolicName(ssize_t tokenType) const override; + virtual std::wstring getDisplayName(ssize_t tokenType) const override; + + private: + static std::vector const EMPTY_NAMES; + + std::vector const _literalNames; + std::vector const _symbolicNames; + std::vector const _displayNames; + const int _maxTokenType; + }; + +} // namespace atn +} // namespace runtime +} // namespace v4 +} // namespace antlr +} // namespace org diff --git a/runtime/Cpp/runtime/WritableToken.h b/runtime/Cpp/runtime/WritableToken.h index 201ebdb11..5d826af74 100755 --- a/runtime/Cpp/runtime/WritableToken.h +++ b/runtime/Cpp/runtime/WritableToken.h @@ -40,18 +40,11 @@ namespace runtime { class WritableToken : public Token { public: - typedef std::shared_ptr Ref; - virtual void setText(const std::wstring &text) = 0; - virtual void setType(int ttype) = 0; - virtual void setLine(int line) = 0; - virtual void setCharPositionInLine(int pos) = 0; - virtual void setChannel(int channel) = 0; - virtual void setTokenIndex(int index) = 0; }; diff --git a/runtime/Cpp/runtime/antlrcpp-Prefix.h b/runtime/Cpp/runtime/antlrcpp-Prefix.h index 025a3de75..67ab9e6d4 100644 --- a/runtime/Cpp/runtime/antlrcpp-Prefix.h +++ b/runtime/Cpp/runtime/antlrcpp-Prefix.h @@ -98,3 +98,6 @@ #define NOEXCEPT #endif #endif + +template using Ref = std::shared_ptr; + diff --git a/runtime/Cpp/runtime/atn/ATN.cpp b/runtime/Cpp/runtime/atn/ATN.cpp index 3352ecca3..495d6368a 100755 --- a/runtime/Cpp/runtime/atn/ATN.cpp +++ b/runtime/Cpp/runtime/atn/ATN.cpp @@ -105,7 +105,7 @@ ATN& ATN::operator = (ATN &&other) NOEXCEPT { return *this; } -misc::IntervalSet ATN::nextTokens(ATNState *s, RuleContext::Ref ctx) const { +misc::IntervalSet ATN::nextTokens(ATNState *s, Ref ctx) const { LL1Analyzer analyzer(*this); return analyzer.LOOK(s, ctx); @@ -150,12 +150,12 @@ int ATN::getNumberOfDecisions() const { return (int)decisionToState.size(); } -misc::IntervalSet ATN::getExpectedTokens(int stateNumber, RuleContext::Ref context) const { +misc::IntervalSet ATN::getExpectedTokens(int stateNumber, Ref context) const { if (stateNumber < 0 || stateNumber >= (int)states.size()) { throw IllegalArgumentException("Invalid state number."); } - RuleContext::Ref ctx = context; + Ref ctx = context; ATNState *s = states.at((size_t)stateNumber); misc::IntervalSet following = nextTokens(s); if (!following.contains(Token::EPSILON)) { diff --git a/runtime/Cpp/runtime/atn/ATN.h b/runtime/Cpp/runtime/atn/ATN.h index f64a2a19d..7a602173f 100755 --- a/runtime/Cpp/runtime/atn/ATN.h +++ b/runtime/Cpp/runtime/atn/ATN.h @@ -32,7 +32,6 @@ #pragma once #include "RuleContext.h" -#include "LexerAction.h" namespace org { namespace antlr { @@ -80,7 +79,7 @@ namespace atn { /// For lexer ATNs, this is an array of {@link LexerAction} objects which may /// be referenced by action transitions in the ATN. - std::vector lexerActions; + std::vector> lexerActions; std::vector modeToStartState; @@ -93,7 +92,7 @@ namespace atn { /// the rule surrounding {@code s}. In other words, the set will be /// restricted to tokens reachable staying within {@code s}'s rule. /// - virtual misc::IntervalSet nextTokens(ATNState *s, RuleContext::Ref ctx) const; + virtual misc::IntervalSet nextTokens(ATNState *s, Ref ctx) const; /// /// Compute the set of valid tokens that can occur starting in {@code s} and @@ -130,7 +129,7 @@ namespace atn { /// specified state in the specified context. /// if the ATN does not contain a state with /// number {@code stateNumber} - virtual misc::IntervalSet getExpectedTokens(int stateNumber, RuleContext::Ref context) const; + virtual misc::IntervalSet getExpectedTokens(int stateNumber, Ref context) const; std::wstring toString() const; }; diff --git a/runtime/Cpp/runtime/atn/ATNConfig.cpp b/runtime/Cpp/runtime/atn/ATNConfig.cpp index 818898a8e..c4a79bc2b 100755 --- a/runtime/Cpp/runtime/atn/ATNConfig.cpp +++ b/runtime/Cpp/runtime/atn/ATNConfig.cpp @@ -31,6 +31,7 @@ #include "MurmurHash.h" #include "PredictionContext.h" +#include "SemanticContext.h" #include "ATNConfig.h" @@ -38,34 +39,34 @@ using namespace org::antlr::v4::runtime::atn; const size_t ATNConfig::SUPPRESS_PRECEDENCE_FILTER = 0x40000000; -ATNConfig::ATNConfig(ATNState *state, int alt, PredictionContext::Ref context) +ATNConfig::ATNConfig(ATNState *state, int alt, Ref context) : ATNConfig(state, alt, context, SemanticContext::NONE) { } -ATNConfig::ATNConfig(ATNState *state, int alt, PredictionContext::Ref context, SemanticContext::Ref semanticContext) +ATNConfig::ATNConfig(ATNState *state, int alt, Ref context, Ref semanticContext) : state(state), alt(alt), context(context), semanticContext(semanticContext) { reachesIntoOuterContext = 0; } -ATNConfig::ATNConfig(ATNConfig::Ref c) : ATNConfig(c, c->state, c->context, c->semanticContext) { +ATNConfig::ATNConfig(Ref c) : ATNConfig(c, c->state, c->context, c->semanticContext) { } -ATNConfig::ATNConfig(ATNConfig::Ref c, ATNState *state) : ATNConfig(c, state, c->context, c->semanticContext) { +ATNConfig::ATNConfig(Ref c, ATNState *state) : ATNConfig(c, state, c->context, c->semanticContext) { } -ATNConfig::ATNConfig(ATNConfig::Ref c, ATNState *state, SemanticContext::Ref semanticContext) +ATNConfig::ATNConfig(Ref c, ATNState *state, Ref semanticContext) : ATNConfig(c, state, c->context, semanticContext) { } -ATNConfig::ATNConfig(ATNConfig::Ref c, SemanticContext::Ref semanticContext) +ATNConfig::ATNConfig(Ref c, Ref semanticContext) : ATNConfig(c, c->state, c->context, semanticContext) { } -ATNConfig::ATNConfig(ATNConfig::Ref c, ATNState *state, PredictionContext::Ref context) +ATNConfig::ATNConfig(Ref c, ATNState *state, Ref context) : ATNConfig(c, state, context, c->semanticContext) { } -ATNConfig::ATNConfig(ATNConfig::Ref c, ATNState *state, PredictionContext::Ref context, SemanticContext::Ref semanticContext) +ATNConfig::ATNConfig(Ref c, ATNState *state, Ref context, Ref semanticContext) : state(state), alt(c->alt), context(context), semanticContext(semanticContext), reachesIntoOuterContext(c->reachesIntoOuterContext) { } diff --git a/runtime/Cpp/runtime/atn/ATNConfig.h b/runtime/Cpp/runtime/atn/ATNConfig.h index f430c6ef5..2672262c8 100755 --- a/runtime/Cpp/runtime/atn/ATNConfig.h +++ b/runtime/Cpp/runtime/atn/ATNConfig.h @@ -31,9 +31,6 @@ #pragma once -#include "PredictionContext.h" -#include "SemanticContext.h" - namespace org { namespace antlr { namespace v4 { @@ -50,8 +47,6 @@ namespace atn { /// class ATNConfig { public: - typedef std::shared_ptr Ref; - /// The ATN state associated with this configuration. ATNState * state; @@ -64,7 +59,7 @@ namespace atn { /// execution of the ATN simulator. /// /// Can be shared between multiple ANTConfig instances. - PredictionContext::Ref context; + Ref context; /** * We cannot execute predicates dependent upon local context unless @@ -92,17 +87,17 @@ namespace atn { int reachesIntoOuterContext; /// Can be shared between multiple ATNConfig instances. - SemanticContext::Ref semanticContext; + Ref semanticContext; - ATNConfig(ATNState *state, int alt, PredictionContext::Ref context); - ATNConfig(ATNState *state, int alt, PredictionContext::Ref context, SemanticContext::Ref semanticContext); + ATNConfig(ATNState *state, int alt, Ref context); + ATNConfig(ATNState *state, int alt, Ref context, Ref semanticContext); - ATNConfig(ATNConfig::Ref c); // dup - ATNConfig(ATNConfig::Ref c, ATNState *state); - ATNConfig(ATNConfig::Ref c, ATNState *state, SemanticContext::Ref semanticContext); - ATNConfig(ATNConfig::Ref c, SemanticContext::Ref semanticContext); - ATNConfig(ATNConfig::Ref c, ATNState *state, PredictionContext::Ref context); - ATNConfig(ATNConfig::Ref c, ATNState *state, PredictionContext::Ref context, SemanticContext::Ref semanticContext); + ATNConfig(Ref c); // dup + ATNConfig(Ref c, ATNState *state); + ATNConfig(Ref c, ATNState *state, Ref semanticContext); + ATNConfig(Ref c, Ref semanticContext); + ATNConfig(Ref c, ATNState *state, Ref context); + ATNConfig(Ref c, ATNState *state, Ref context, Ref semanticContext); virtual size_t hashCode() const; @@ -156,13 +151,25 @@ namespace atn { // Hash function for ATNConfig. namespace std { - using org::antlr::v4::runtime::atn::ATNConfig; + using org::antlr::v4::runtime::atn::ATNConfig; - template <> struct hash + template <> struct hash + { + size_t operator() (const ATNConfig &x) const { - size_t operator() (const ATNConfig &x) const - { - return x.hashCode(); - } - }; + return x.hashCode(); + } + }; + + template <> struct hash>> + { + size_t operator() (const std::vector> &vector) const + { + std::size_t seed = 0; + for (auto &config : vector) { + seed ^= config->hashCode() + 0x9e3779b9 + (seed << 6) + (seed >> 2); + } + return seed; + } + }; } diff --git a/runtime/Cpp/runtime/atn/ATNConfigSet.cpp b/runtime/Cpp/runtime/atn/ATNConfigSet.cpp index 7ec8c4498..c6271fc90 100755 --- a/runtime/Cpp/runtime/atn/ATNConfigSet.cpp +++ b/runtime/Cpp/runtime/atn/ATNConfigSet.cpp @@ -33,13 +33,14 @@ #include "ATNConfig.h" #include "ATNSimulator.h" #include "Exceptions.h" +#include "SemanticContext.h" #include "ATNConfigSet.h" using namespace org::antlr::v4::runtime::atn; using namespace antlrcpp; -size_t SimpleATNConfigHasher::operator()(const ATNConfig::Ref &k) const { +size_t SimpleATNConfigHasher::operator()(const Ref &k) const { size_t hashCode = 7; hashCode = 31 * hashCode + (size_t)k->state->stateNumber; hashCode = 31 * hashCode + (size_t)k->alt; @@ -47,7 +48,7 @@ size_t SimpleATNConfigHasher::operator()(const ATNConfig::Ref &k) const { return hashCode; } -bool SimpleATNConfigComparer::operator () (const ATNConfig::Ref &lhs, const ATNConfig::Ref &rhs) const { +bool SimpleATNConfigComparer::operator () (const Ref &lhs, const Ref &rhs) const { return lhs->state->stateNumber == rhs->state->stateNumber && lhs->alt == rhs->alt && lhs->semanticContext == rhs->semanticContext; } @@ -58,7 +59,7 @@ ATNConfigSet::ATNConfigSet(bool fullCtx) : fullCtx(fullCtx) { InitializeInstanceFields(); } -ATNConfigSet::ATNConfigSet(std::shared_ptr old) : ATNConfigSet(old->fullCtx) { +ATNConfigSet::ATNConfigSet(Ref old) : ATNConfigSet(old->fullCtx) { addAll(old); uniqueAlt = old->uniqueAlt; conflictingAlts = old->conflictingAlts; @@ -69,11 +70,11 @@ ATNConfigSet::ATNConfigSet(std::shared_ptr old) : ATNConfigSet(old ATNConfigSet::~ATNConfigSet() { } -bool ATNConfigSet::add(ATNConfig::Ref config) { +bool ATNConfigSet::add(Ref config) { return add(config, nullptr); } -bool ATNConfigSet::add(ATNConfig::Ref config, PredictionContextMergeCache *mergeCache) { +bool ATNConfigSet::add(Ref config, PredictionContextMergeCache *mergeCache) { if (_readonly) { throw IllegalStateException("This set is readonly"); } @@ -84,7 +85,7 @@ bool ATNConfigSet::add(ATNConfig::Ref config, PredictionContextMergeCache *merge dipsIntoOuterContext = true; } - ATNConfig::Ref existing = configLookup->getOrAdd(config); + Ref existing = configLookup->getOrAdd(config); if (existing == config) { // we added this new one _cachedHashCode = 0; configs.push_back(config); // track order here @@ -93,7 +94,7 @@ bool ATNConfigSet::add(ATNConfig::Ref config, PredictionContextMergeCache *merge } // a previous (s,i,pi,_), merge with it and save result bool rootIsWildcard = !fullCtx; - PredictionContext::Ref merged = PredictionContext::merge(existing->context, config->context, rootIsWildcard, mergeCache); + Ref merged = PredictionContext::merge(existing->context, config->context, rootIsWildcard, mergeCache); // no need to check for existing.context, config.context in cache // since only way to create new graphs is "call rule" and here. We // cache at both places. @@ -108,14 +109,14 @@ bool ATNConfigSet::add(ATNConfig::Ref config, PredictionContextMergeCache *merge return true; } -bool ATNConfigSet::addAll(std::shared_ptr other) { +bool ATNConfigSet::addAll(Ref other) { for (auto &c : other->configs) { add(c); } return false; } -std::vector ATNConfigSet::elements() { +std::vector> ATNConfigSet::elements() { return configs; } @@ -144,8 +145,8 @@ BitSet ATNConfigSet::getAlts() { return alts; } -std::vector ATNConfigSet::getPredicates() { - std::vector preds; +std::vector> ATNConfigSet::getPredicates() { + std::vector> preds; for (auto c : configs) { if (c->semanticContext != SemanticContext::NONE) { preds.push_back(c->semanticContext); @@ -154,7 +155,7 @@ std::vector ATNConfigSet::getPredicates() { return preds; } -ATNConfig::Ref ATNConfigSet::get(size_t i) const { +Ref ATNConfigSet::get(size_t i) const { return configs[i]; } @@ -202,13 +203,13 @@ bool ATNConfigSet::operator == (const ATNConfigSet &other) { size_t ATNConfigSet::hashCode() { if (isReadonly()) { if (_cachedHashCode == 0) { - _cachedHashCode = std::hash>()(configs); + _cachedHashCode = std::hash>>()(configs); } return _cachedHashCode; } - return std::hash>()(configs); + return std::hash>>()(configs); } size_t ATNConfigSet::size() { @@ -219,7 +220,7 @@ bool ATNConfigSet::isEmpty() { return configs.empty(); } -bool ATNConfigSet::contains(ATNConfig::Ref o) { +bool ATNConfigSet::contains(Ref o) { if (configLookup == nullptr) { throw UnsupportedOperationException("This method is not implemented for readonly sets."); } @@ -276,7 +277,7 @@ bool ATNConfigSet::remove(void *o) { } void ATNConfigSet::InitializeInstanceFields() { - configLookup = std::shared_ptr(new ConfigLookupImpl()); + configLookup = Ref(new ConfigLookupImpl()); uniqueAlt = 0; hasSemanticContext = false; dipsIntoOuterContext = false; diff --git a/runtime/Cpp/runtime/atn/ATNConfigSet.h b/runtime/Cpp/runtime/atn/ATNConfigSet.h index e2ad1e520..4c2680819 100755 --- a/runtime/Cpp/runtime/atn/ATNConfigSet.h +++ b/runtime/Cpp/runtime/atn/ATNConfigSet.h @@ -32,8 +32,8 @@ #pragma once #include "BitSet.h" -#include "ATNConfig.h" #include "ConfigLookup.h" +#include "PredictionContext.h" namespace org { namespace antlr { @@ -44,11 +44,11 @@ namespace atn { // Simpler hasher and comparer variants than those in ATNConfig (less fields, no murmur hash). struct SimpleATNConfigHasher { - size_t operator()(const ATNConfig::Ref &k) const; + size_t operator()(const Ref &k) const; }; struct SimpleATNConfigComparer { - bool operator()(const ATNConfig::Ref &lhs, const ATNConfig::Ref &rhs) const; + bool operator()(const Ref &lhs, const Ref &rhs) const; }; /// Specialized set that can track info about the set, with support for combining similar configurations using a @@ -57,11 +57,11 @@ namespace atn { public: /// All configs but hashed by (s, i, _, pi) not including context. Wiped out /// when we go readonly as this set becomes a DFA state. - std::shared_ptr configLookup; + Ref configLookup; /// /// Track the elements as they are added to the set; supports get(i) - std::vector configs; + std::vector> configs; // TO_DO: these fields make me pretty uncomfortable but nice to pack up info together, saves recomputation // TO_DO: can we track conflicts as they are added to save scanning configs later? @@ -87,11 +87,11 @@ namespace atn { const bool fullCtx; ATNConfigSet(bool fullCtx = true); - ATNConfigSet(std::shared_ptr old); + ATNConfigSet(Ref old); virtual ~ATNConfigSet(); - virtual bool add(ATNConfig::Ref config); + virtual bool add(Ref config); /// /// Adding a new config means merging contexts with existing configs for @@ -103,13 +103,13 @@ namespace atn { /// This method updates and /// when necessary. /// - virtual bool add(ATNConfig::Ref config, PredictionContextMergeCache *mergeCache); + virtual bool add(Ref config, PredictionContextMergeCache *mergeCache); /// /// Return a List holding list of configs - virtual std::vector elements(); + virtual std::vector> elements(); - virtual std::vector getStates(); + virtual std::vector getStates(); /** * Gets the complete set of represented alternatives for the configuration @@ -120,19 +120,19 @@ namespace atn { * @since 4.3 */ antlrcpp::BitSet getAlts(); - virtual std::vector getPredicates(); + virtual std::vector> getPredicates(); - virtual ATNConfig::Ref get(size_t i) const; + virtual Ref get(size_t i) const; virtual void optimizeConfigs(ATNSimulator *interpreter); - bool addAll(std::shared_ptr other); + bool addAll(Ref other); bool operator == (const ATNConfigSet &other); virtual size_t hashCode(); virtual size_t size(); virtual bool isEmpty(); - virtual bool contains(ATNConfig::Ref o); + virtual bool contains(Ref o); virtual void clear(); virtual bool isReadonly(); virtual void setReadonly(bool readonly); @@ -158,17 +158,3 @@ namespace atn { } // namespace v4 } // namespace antlr } // namespace org - -namespace std { - template <> struct hash> - { - size_t operator() (const std::vector &vector) const - { - std::size_t seed = 0; - for (auto &config : vector) { - seed ^= config->hashCode() + 0x9e3779b9 + (seed << 6) + (seed >> 2); - } - return seed; - } - }; -} diff --git a/runtime/Cpp/runtime/atn/ATNDeserializer.cpp b/runtime/Cpp/runtime/atn/ATNDeserializer.cpp index 29ad74c9b..75418bdc3 100755 --- a/runtime/Cpp/runtime/atn/ATNDeserializer.cpp +++ b/runtime/Cpp/runtime/atn/ATNDeserializer.cpp @@ -407,7 +407,7 @@ ATN ATNDeserializer::deserialize(const std::wstring& input) { int ruleIndex = static_cast(transition)->ruleIndex; int actionIndex = static_cast(transition)->actionIndex; - std::shared_ptr lexerAction = std::make_shared(ruleIndex, actionIndex); + Ref lexerAction = std::make_shared(ruleIndex, actionIndex); state->setTransition(i, new ActionTransition(transition->target, ruleIndex, (int)atn.lexerActions.size(), false)); /* mem-check freed in ATNState d-tor */ atn.lexerActions.push_back(lexerAction); } @@ -706,7 +706,7 @@ ATNState* ATNDeserializer::stateFactory(int type, int ruleIndex) { return s; } -LexerAction::Ref ATNDeserializer::lexerActionFactory(LexerActionType type, int data1, int data2) { +Ref ATNDeserializer::lexerActionFactory(LexerActionType type, int data1, int data2) { switch (type) { case LexerActionType::CHANNEL: return std::make_shared(data1); diff --git a/runtime/Cpp/runtime/atn/ATNDeserializer.h b/runtime/Cpp/runtime/atn/ATNDeserializer.h index c60e24988..a657101cc 100755 --- a/runtime/Cpp/runtime/atn/ATNDeserializer.h +++ b/runtime/Cpp/runtime/atn/ATNDeserializer.h @@ -78,7 +78,7 @@ namespace atn { /// introduced; otherwise, {@code false}. virtual bool isFeatureSupported(const Guid &feature, const Guid &actualUuid); void markPrecedenceDecisions(const ATN &atn); - LexerAction::Ref lexerActionFactory(LexerActionType type, int data1, int data2); + Ref lexerActionFactory(LexerActionType type, int data1, int data2); private: /// This is the earliest supported serialized UUID. diff --git a/runtime/Cpp/runtime/atn/ATNSerializer.cpp b/runtime/Cpp/runtime/atn/ATNSerializer.cpp index a01b8f816..d17efe13f 100755 --- a/runtime/Cpp/runtime/atn/ATNSerializer.cpp +++ b/runtime/Cpp/runtime/atn/ATNSerializer.cpp @@ -310,7 +310,7 @@ std::vector ATNSerializer::serialize() { // LEXER ACTIONS if (atn->grammarType == ATNType::LEXER) { data.push_back(atn->lexerActions.size()); - for (LexerAction::Ref action : atn->lexerActions) { + for (Ref action : atn->lexerActions) { data.push_back((size_t)action->getActionType()); switch (action->getActionType()) { case LexerActionType::CHANNEL: diff --git a/runtime/Cpp/runtime/atn/ATNSimulator.cpp b/runtime/Cpp/runtime/atn/ATNSimulator.cpp index dcec24c24..a7c5e5a70 100755 --- a/runtime/Cpp/runtime/atn/ATNSimulator.cpp +++ b/runtime/Cpp/runtime/atn/ATNSimulator.cpp @@ -41,19 +41,23 @@ using namespace org::antlr::v4::runtime; using namespace org::antlr::v4::runtime::dfa; using namespace org::antlr::v4::runtime::atn; -const std::shared_ptr ATNSimulator::ERROR = std::make_shared(INT32_MAX); +const Ref ATNSimulator::ERROR = std::make_shared(INT32_MAX); -ATNSimulator::ATNSimulator(const ATN &atn, std::shared_ptr sharedContextCache) +ATNSimulator::ATNSimulator(const ATN &atn, Ref sharedContextCache) : atn(atn), _sharedContextCache(sharedContextCache) { } -std::shared_ptr ATNSimulator::getSharedContextCache() { +void ATNSimulator::clearDFA() { + throw UnsupportedOperationException("This ATN simulator does not support clearing the DFA."); +} + +Ref ATNSimulator::getSharedContextCache() { return _sharedContextCache; } -PredictionContext::Ref ATNSimulator::getCachedContext(PredictionContext::Ref context) { +Ref ATNSimulator::getCachedContext(Ref context) { std::lock_guard lck(mtx); - std::map visited; + std::map, Ref> visited; return PredictionContext::getCachedContext(context, _sharedContextCache, visited); } diff --git a/runtime/Cpp/runtime/atn/ATNSimulator.h b/runtime/Cpp/runtime/atn/ATNSimulator.h index ad718e25c..feadd57de 100755 --- a/runtime/Cpp/runtime/atn/ATNSimulator.h +++ b/runtime/Cpp/runtime/atn/ATNSimulator.h @@ -43,18 +43,29 @@ namespace atn { class ATNSimulator { public: - virtual ~ATNSimulator() {}; - /// Must distinguish between missing edge and edge we know leads nowhere. - static const std::shared_ptr ERROR; + static const Ref ERROR; const ATN &atn; - ATNSimulator(const ATN &atn, std::shared_ptr sharedContextCache); + ATNSimulator(const ATN &atn, Ref sharedContextCache); + virtual ~ATNSimulator() {}; virtual void reset() = 0; - virtual std::shared_ptr getSharedContextCache(); - virtual PredictionContext::Ref getCachedContext(PredictionContext::Ref context); + /** + * Clear the DFA cache used by the current instance. Since the DFA cache may + * be shared by multiple ATN simulators, this method may affect the + * performance (but not accuracy) of other parsers which are being used + * concurrently. + * + * @throws UnsupportedOperationException if the current instance does not + * support clearing the DFA. + * + * @since 4.3 + */ + virtual void clearDFA(); + virtual Ref getSharedContextCache(); + virtual Ref getCachedContext(Ref context); /// @deprecated Use instead. static ATN deserialize(const std::wstring &data); @@ -97,7 +108,7 @@ namespace atn { /// more time I think and doesn't save on the overall footprint /// so it's not worth the complexity. /// - std::shared_ptr _sharedContextCache; + Ref _sharedContextCache; }; diff --git a/runtime/Cpp/runtime/atn/AmbiguityInfo.cpp b/runtime/Cpp/runtime/atn/AmbiguityInfo.cpp new file mode 100755 index 000000000..0011326e8 --- /dev/null +++ b/runtime/Cpp/runtime/atn/AmbiguityInfo.cpp @@ -0,0 +1,42 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "AmbiguityInfo.h" + +using namespace org::antlr::v4::runtime; +using namespace org::antlr::v4::runtime::atn; + +AmbiguityInfo::AmbiguityInfo(int decision, Ref configs, const antlrcpp::BitSet &ambigAlts, + TokenStream *input, size_t startIndex, size_t stopIndex, bool fullCtx) + : DecisionEventInfo(decision, configs, input, startIndex, stopIndex, fullCtx) { + + this->ambigAlts = ambigAlts; +} diff --git a/runtime/Cpp/runtime/atn/AmbiguityInfo.h b/runtime/Cpp/runtime/atn/AmbiguityInfo.h new file mode 100755 index 000000000..b29d344d1 --- /dev/null +++ b/runtime/Cpp/runtime/atn/AmbiguityInfo.h @@ -0,0 +1,100 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "DecisionEventInfo.h" +#include "BitSet.h" + +namespace org { +namespace antlr { +namespace v4 { +namespace runtime { +namespace atn { + + /// + /// This class represents profiling event information for an ambiguity. + /// Ambiguities are decisions where a particular input resulted in an SLL + /// conflict, followed by LL prediction also reaching a conflict state + /// (indicating a true ambiguity in the grammar). + /// + /// + /// This event may be reported during SLL prediction in cases where the + /// conflicting SLL configuration set provides sufficient information to + /// determine that the SLL conflict is truly an ambiguity. For example, if none + /// of the ATN configurations in the conflicting SLL configuration set have + /// traversed a global follow transition (i.e. + /// is 0 for all configurations), then + /// the result of SLL prediction for that input is known to be equivalent to the + /// result of LL prediction for that input. + /// + /// + /// In some cases, the minimum represented alternative in the conflicting LL + /// configuration set is not equal to the minimum represented alternative in the + /// conflicting SLL configuration set. Grammars and inputs which result in this + /// scenario are unable to use , which in turn means + /// they cannot use the two-stage parsing strategy to improve parsing performance + /// for that input. + /// + /// + /// + class AmbiguityInfo : public DecisionEventInfo { + public: + /// The set of alternative numbers for this decision event that lead to a valid parse. + antlrcpp::BitSet ambigAlts; + + /// + /// Constructs a new instance of the class with the + /// specified detailed ambiguity information. + /// + /// The decision number + /// The final configuration set identifying the ambiguous + /// alternatives for the current input + /// The set of alternatives in the decision that lead to a valid parse. + /// The predicted alt is the min(ambigAlts) + /// The input token stream + /// The start index for the current prediction + /// The index at which the ambiguity was identified during + /// prediction + /// {@code true} if the ambiguity was identified during LL + /// prediction; otherwise, {@code false} if the ambiguity was identified + /// during SLL prediction + AmbiguityInfo(int decision, Ref configs, const antlrcpp::BitSet &ambigAlts, TokenStream *input, + size_t startIndex, size_t stopIndex, bool fullCtx); + }; + +} // namespace atn +} // namespace runtime +} // namespace v4 +} // namespace antlr +} // namespace org diff --git a/runtime/Cpp/runtime/atn/ArrayPredictionContext.cpp b/runtime/Cpp/runtime/atn/ArrayPredictionContext.cpp index d9f9495ed..93079938e 100755 --- a/runtime/Cpp/runtime/atn/ArrayPredictionContext.cpp +++ b/runtime/Cpp/runtime/atn/ArrayPredictionContext.cpp @@ -36,7 +36,7 @@ using namespace org::antlr::v4::runtime::atn; -ArrayPredictionContext::ArrayPredictionContext(SingletonPredictionContext::Ref a) +ArrayPredictionContext::ArrayPredictionContext(Ref a) : ArrayPredictionContext({ a->parent }, { a->returnState }) { } diff --git a/runtime/Cpp/runtime/atn/ArrayPredictionContext.h b/runtime/Cpp/runtime/atn/ArrayPredictionContext.h index 16c6ab544..1e7865be3 100755 --- a/runtime/Cpp/runtime/atn/ArrayPredictionContext.h +++ b/runtime/Cpp/runtime/atn/ArrayPredictionContext.h @@ -52,7 +52,7 @@ namespace atn { /// Sorted for merge, no duplicates; if present, EMPTY_RETURN_STATE is always last. const std::vector returnStates; - ArrayPredictionContext(SingletonPredictionContext::Ref a); + ArrayPredictionContext(Ref a); ArrayPredictionContext(const std::vector> &parents, const std::vector &returnStates); diff --git a/runtime/Cpp/runtime/atn/ConfigLookup.h b/runtime/Cpp/runtime/atn/ConfigLookup.h index d08b2c843..17e8ae550 100644 --- a/runtime/Cpp/runtime/atn/ConfigLookup.h +++ b/runtime/Cpp/runtime/atn/ConfigLookup.h @@ -29,8 +29,6 @@ #pragma once -#include "ATNConfig.h" - namespace org { namespace antlr { namespace v4 { @@ -48,17 +46,17 @@ namespace atn { ConfigLookupIterator& operator++ () { ++*_iterator; return *this; }; bool operator != (const ConfigLookupIterator& rhs) const { return *_iterator != *rhs._iterator; }; - ATNConfig::Ref operator * () const { return **_iterator; }; + Ref operator * () const { return **_iterator; }; private: - std::shared_ptr _iterator; + Ref _iterator; }; virtual ~ConfigLookup() {} // Java iterator interface. - virtual ATNConfig::Ref getOrAdd(ATNConfig::Ref config) = 0; + virtual Ref getOrAdd(Ref config) = 0; virtual bool isEmpty() const = 0; - virtual bool contains(ATNConfig::Ref config) const = 0; + virtual bool contains(Ref config) const = 0; virtual void clear() = 0; // STL iterator interface. @@ -71,18 +69,18 @@ namespace atn { public: virtual ConfigLookupIteratorImpl& operator ++ () = 0; virtual bool operator != (const ConfigLookupIteratorImpl&) const = 0; - virtual ATNConfig::Ref operator * () const = 0; + virtual Ref operator * () const = 0; virtual void* underlyingIterator () = 0; virtual const void* underlyingIterator () const = 0; }; }; template - class ConfigLookupImpl: public ConfigLookup, std::unordered_set { + class ConfigLookupImpl: public ConfigLookup, std::unordered_set, Hasher, Comparer> { public: - using Set = std::unordered_set; + using Set = std::unordered_set, Hasher, Comparer>; - virtual ATNConfig::Ref getOrAdd(ATNConfig::Ref config) override { + virtual Ref getOrAdd(Ref config) override { auto result = Set::find(config); if (result != Set::end()) // Can potentially be a different config instance which however is considered equal to the given config @@ -97,7 +95,7 @@ namespace atn { return Set::empty(); } - virtual bool contains(ATNConfig::Ref config) const override { + virtual bool contains(Ref config) const override { return Set::count(config) > 0; } @@ -112,19 +110,19 @@ namespace atn { ConfigLookupIterator begin() override { return ConfigLookupIterator( new ConfigLookupImpl::ConfigLookupIteratorImpl( - std::unordered_set::begin())); /* mem check: managed by shared_ptr in the iterator */ + std::unordered_set, Hasher, Comparer>::begin())); /* mem check: managed by shared_ptr in the iterator */ } ConfigLookupIterator end() override { return ConfigLookupIterator( new ConfigLookupImpl::ConfigLookupIteratorImpl( - std::unordered_set::end())); /* mem check: managed by shared_ptr in the iterator */ + std::unordered_set, Hasher, Comparer>::end())); /* mem check: managed by shared_ptr in the iterator */ } protected: class ConfigLookupIteratorImpl : public ConfigLookup::ConfigLookupIteratorImpl { public: - using UnderlyingIterator = typename std::unordered_set::iterator; + using UnderlyingIterator = typename std::unordered_set, Hasher, Comparer>::iterator; ConfigLookupIteratorImpl(UnderlyingIterator&& iterator) : _iterator(std::move(iterator)) { } @@ -137,7 +135,7 @@ namespace atn { return *reinterpret_cast(underlyingIterator()) != *reinterpret_cast(rhs.underlyingIterator()); } - ATNConfig::Ref operator * () const override { + Ref operator * () const override { return *_iterator; } diff --git a/runtime/Cpp/runtime/atn/ContextSensitivityInfo.cpp b/runtime/Cpp/runtime/atn/ContextSensitivityInfo.cpp new file mode 100755 index 000000000..1de81032d --- /dev/null +++ b/runtime/Cpp/runtime/atn/ContextSensitivityInfo.cpp @@ -0,0 +1,40 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "ContextSensitivityInfo.h" + +using namespace org::antlr::v4::runtime; +using namespace org::antlr::v4::runtime::atn; + +ContextSensitivityInfo::ContextSensitivityInfo(int decision, Ref configs, TokenStream *input, + size_t startIndex, size_t stopIndex) + : DecisionEventInfo(decision, configs, input, startIndex, stopIndex, true) { +} diff --git a/runtime/Cpp/runtime/atn/ContextSensitivityInfo.h b/runtime/Cpp/runtime/atn/ContextSensitivityInfo.h new file mode 100755 index 000000000..253680d35 --- /dev/null +++ b/runtime/Cpp/runtime/atn/ContextSensitivityInfo.h @@ -0,0 +1,79 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "DecisionEventInfo.h" + +namespace org { +namespace antlr { +namespace v4 { +namespace runtime { +namespace atn { + + /// + /// This class represents profiling event information for a context sensitivity. + /// Context sensitivities are decisions where a particular input resulted in an + /// SLL conflict, but LL prediction produced a single unique alternative. + /// + /// + /// In some cases, the unique alternative identified by LL prediction is not + /// equal to the minimum represented alternative in the conflicting SLL + /// configuration set. Grammars and inputs which result in this scenario are + /// unable to use , which in turn means they cannot use + /// the two-stage parsing strategy to improve parsing performance for that + /// input. + /// + /// + /// + class ContextSensitivityInfo : public DecisionEventInfo { + public: + /// + /// Constructs a new instance of the class + /// with the specified detailed context sensitivity information. + /// + /// The decision number + /// The final configuration set containing the unique + /// alternative identified by full-context prediction + /// The input token stream + /// The start index for the current prediction + /// The index at which the context sensitivity was + /// identified during full-context prediction + ContextSensitivityInfo(int decision, Ref configs, TokenStream *input, size_t startIndex, size_t stopIndex); + }; + +} // namespace atn +} // namespace runtime +} // namespace v4 +} // namespace antlr +} // namespace org diff --git a/runtime/Cpp/runtime/atn/DecisionEventInfo.cpp b/runtime/Cpp/runtime/atn/DecisionEventInfo.cpp new file mode 100755 index 000000000..f668ee861 --- /dev/null +++ b/runtime/Cpp/runtime/atn/DecisionEventInfo.cpp @@ -0,0 +1,40 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "DecisionEventInfo.h" + +using namespace org::antlr::v4::runtime; +using namespace org::antlr::v4::runtime::atn; + +DecisionEventInfo::DecisionEventInfo(int decision, Ref configs, TokenStream *input, size_t startIndex, + size_t stopIndex, bool fullCtx) + : decision(decision), configs(configs), input(input), startIndex(startIndex), stopIndex(stopIndex), fullCtx(fullCtx) { +} diff --git a/runtime/Cpp/runtime/atn/DecisionEventInfo.h b/runtime/Cpp/runtime/atn/DecisionEventInfo.h new file mode 100755 index 000000000..7921610ca --- /dev/null +++ b/runtime/Cpp/runtime/atn/DecisionEventInfo.h @@ -0,0 +1,100 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +namespace org { +namespace antlr { +namespace v4 { +namespace runtime { +namespace atn { + + /// + /// This is the base class for gathering detailed information about prediction + /// events which occur during parsing. + /// + /// Note that we could record the parser call stack at the time this event + /// occurred but in the presence of left recursive rules, the stack is kind of + /// meaningless. It's better to look at the individual configurations for their + /// individual stacks. Of course that is a object + /// not a parse tree node and so it does not have information about the extent + /// (start...stop) of the various subtrees. Examining the stack tops of all + /// configurations provide the return states for the rule invocations. + /// From there you can get the enclosing rule. + /// + /// @since 4.3 + /// + class DecisionEventInfo { + public: + /// + /// The invoked decision number which this event is related to. + /// + /// + const int decision; + + /// + /// The configuration set containing additional information relevant to the + /// prediction state when the current event occurred, or {@code null} if no + /// additional information is relevant or available. + /// + const Ref configs; + + /// + /// The input token stream which is being parsed. + /// + const TokenStream *input; + + /// + /// The token index in the input stream at which the current prediction was + /// originally invoked. + /// + const size_t startIndex; + + /// + /// The token index in the input stream at which the current event occurred. + /// + const size_t stopIndex; + + /// + /// {@code true} if the current event occurred during LL prediction; + /// otherwise, {@code false} if the input occurred during SLL prediction. + /// + const bool fullCtx; + + DecisionEventInfo(int decision, Ref configs, TokenStream *input, size_t startIndex, size_t stopIndex, + bool fullCtx); + }; + +} // namespace atn +} // namespace runtime +} // namespace v4 +} // namespace antlr +} // namespace org diff --git a/runtime/Cpp/runtime/atn/DecisionInfo.cpp b/runtime/Cpp/runtime/atn/DecisionInfo.cpp new file mode 100755 index 000000000..f1085e335 --- /dev/null +++ b/runtime/Cpp/runtime/atn/DecisionInfo.cpp @@ -0,0 +1,51 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "ErrorInfo.h" +#include "LookaheadEventInfo.h" + +#include "DecisionInfo.h" + +using namespace org::antlr::v4::runtime::atn; + +DecisionInfo::DecisionInfo(size_t decision) : decision(decision) { +} + +std::wstring DecisionInfo::toString() const { + std::wstringstream ss; + + ss << L"{decision=" << decision << L", contextSensitivities=" << contextSensitivities.size() << L", errors="; + ss << errors.size() << L", ambiguities=" << ambiguities.size() << L", SLL_lookahead=" << SLL_TotalLook; + ss << L", SLL_ATNTransitions=" << SLL_ATNTransitions << L", SLL_DFATransitions=" << SLL_DFATransitions; + ss << L", LL_Fallback=" << LL_Fallback << L", LL_lookahead=" << LL_TotalLook << L", LL_ATNTransitions=" << LL_ATNTransitions << L'}'; + + return ss.str(); +} diff --git a/runtime/Cpp/runtime/atn/DecisionInfo.h b/runtime/Cpp/runtime/atn/DecisionInfo.h new file mode 100755 index 000000000..3c9cd8d06 --- /dev/null +++ b/runtime/Cpp/runtime/atn/DecisionInfo.h @@ -0,0 +1,259 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "ContextSensitivityInfo.h" +#include "AmbiguityInfo.h" +#include "PredicateEvalInfo.h" +#include "ErrorInfo.h" + +namespace org { +namespace antlr { +namespace v4 { +namespace runtime { +namespace atn { + + class LookaheadEventInfo; + + /// + /// This class contains profiling gathered for a particular decision. + /// + /// + /// Parsing performance in ANTLR 4 is heavily influenced by both static factors + /// (e.g. the form of the rules in the grammar) and dynamic factors (e.g. the + /// choice of input and the state of the DFA cache at the time profiling + /// operations are started). For best results, gather and use aggregate + /// statistics from a large sample of inputs representing the inputs expected in + /// production before using the results to make changes in the grammar. + /// + /// @since 4.3 + /// + class DecisionInfo { + public: + /// + /// The decision number, which is an index into . + /// + const size_t decision; + + /// + /// The total number of times was + /// invoked for this decision. + /// + long long invocations = 0; + + /// + /// The total time spent in for + /// this decision, in nanoseconds. + /// + /// + /// The value of this field contains the sum of differential results obtained + /// by , and is not adjusted to compensate for JIT + /// and/or garbage collection overhead. For best accuracy, use a modern JVM + /// implementation that provides precise results from + /// , and perform profiling in a separate process + /// which is warmed up by parsing the input prior to profiling. If desired, + /// call to reset the DFA cache to its initial + /// state before starting the profiling measurement pass. + /// + long long timeInPrediction = 0; + + /// + /// The sum of the lookahead required for SLL prediction for this decision. + /// Note that SLL prediction is used before LL prediction for performance + /// reasons even when or + /// is used. + /// + long long SLL_TotalLook = 0; + + /// + /// Gets the minimum lookahead required for any single SLL prediction to + /// complete for this decision, by reaching a unique prediction, reaching an + /// SLL conflict state, or encountering a syntax error. + /// + long long SLL_MinLook = 0; + + /// + /// Gets the maximum lookahead required for any single SLL prediction to + /// complete for this decision, by reaching a unique prediction, reaching an + /// SLL conflict state, or encountering a syntax error. + /// + long long SLL_MaxLook = 0; + + /// Gets the associated with the event where the + /// value was set. + Ref SLL_MaxLookEvent; + + /// + /// The sum of the lookahead required for LL prediction for this decision. + /// Note that LL prediction is only used when SLL prediction reaches a + /// conflict state. + /// + long long LL_TotalLook = 0; + + /// + /// Gets the minimum lookahead required for any single LL prediction to + /// complete for this decision. An LL prediction completes when the algorithm + /// reaches a unique prediction, a conflict state (for + /// , an ambiguity state (for + /// , or a syntax error. + /// + long long LL_MinLook = 0; + + /// + /// Gets the maximum lookahead required for any single LL prediction to + /// complete for this decision. An LL prediction completes when the algorithm + /// reaches a unique prediction, a conflict state (for + /// , an ambiguity state (for + /// , or a syntax error. + /// + long long LL_MaxLook = 0; + + /// + /// Gets the associated with the event where the + /// value was set. + /// + Ref LL_MaxLookEvent; + + /// + /// A collection of instances describing the + /// context sensitivities encountered during LL prediction for this decision. + /// + /// + std::vector contextSensitivities; + + /// + /// A collection of instances describing the parse errors + /// identified during calls to for + /// this decision. + /// + /// + std::vector errors; + + /// + /// A collection of instances describing the + /// ambiguities encountered during LL prediction for this decision. + /// + /// + std::vector ambiguities; + + /// + /// A collection of instances describing the + /// results of evaluating individual predicates during prediction for this + /// decision. + /// + /// + std::vector predicateEvals; + + /// + /// The total number of ATN transitions required during SLL prediction for + /// this decision. An ATN transition is determined by the number of times the + /// DFA does not contain an edge that is required for prediction, resulting + /// in on-the-fly computation of that edge. + /// + /// + /// If DFA caching of SLL transitions is employed by the implementation, ATN + /// computation may cache the computed edge for efficient lookup during + /// future parsing of this decision. Otherwise, the SLL parsing algorithm + /// will use ATN transitions exclusively. + /// + /// + /// + /// + long long SLL_ATNTransitions = 0; + + /// + /// The total number of DFA transitions required during SLL prediction for + /// this decision. + /// + /// If the ATN simulator implementation does not use DFA caching for SLL + /// transitions, this value will be 0. + /// + /// + /// + long long SLL_DFATransitions = 0; + + /// + /// Gets the total number of times SLL prediction completed in a conflict + /// state, resulting in fallback to LL prediction. + /// + /// Note that this value is not related to whether or not + /// may be used successfully with a particular + /// grammar. If the ambiguity resolution algorithm applied to the SLL + /// conflicts for this decision produce the same result as LL prediction for + /// this decision, would produce the same overall + /// parsing result as . + /// + long long LL_Fallback = 0; + + /// + /// The total number of ATN transitions required during LL prediction for + /// this decision. An ATN transition is determined by the number of times the + /// DFA does not contain an edge that is required for prediction, resulting + /// in on-the-fly computation of that edge. + /// + /// + /// If DFA caching of LL transitions is employed by the implementation, ATN + /// computation may cache the computed edge for efficient lookup during + /// future parsing of this decision. Otherwise, the LL parsing algorithm will + /// use ATN transitions exclusively. + /// + /// + /// + /// + long long LL_ATNTransitions = 0; + + /// + /// The total number of DFA transitions required during LL prediction for + /// this decision. + /// + /// If the ATN simulator implementation does not use DFA caching for LL + /// transitions, this value will be 0. + /// + /// + /// + long long LL_DFATransitions = 0; + + /// + /// Constructs a new instance of the class to contain + /// statistics for a particular decision. + /// + /// The decision number + DecisionInfo(size_t decision); + + std::wstring toString() const; + }; + +} // namespace atn +} // namespace runtime +} // namespace v4 +} // namespace antlr +} // namespace org diff --git a/runtime/Cpp/runtime/atn/ErrorInfo.cpp b/runtime/Cpp/runtime/atn/ErrorInfo.cpp new file mode 100755 index 000000000..2978a4487 --- /dev/null +++ b/runtime/Cpp/runtime/atn/ErrorInfo.cpp @@ -0,0 +1,41 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "ATNConfigSet.h" + +#include "ErrorInfo.h" + +using namespace org::antlr::v4::runtime; +using namespace org::antlr::v4::runtime::atn; + +ErrorInfo::ErrorInfo(int decision, Ref configs, TokenStream *input, int startIndex, int stopIndex, bool fullCtx) + : DecisionEventInfo(decision, configs, input, startIndex, stopIndex, fullCtx) { +} diff --git a/runtime/Cpp/runtime/atn/ErrorInfo.h b/runtime/Cpp/runtime/atn/ErrorInfo.h new file mode 100755 index 000000000..1d37b3a0c --- /dev/null +++ b/runtime/Cpp/runtime/atn/ErrorInfo.h @@ -0,0 +1,74 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "DecisionEventInfo.h" + +namespace org { +namespace antlr { +namespace v4 { +namespace runtime { +namespace atn { + + /// + /// This class represents profiling event information for a syntax error + /// identified during prediction. Syntax errors occur when the prediction + /// algorithm is unable to identify an alternative which would lead to a + /// successful parse. + /// + /// + /// + class ErrorInfo : public DecisionEventInfo { + public: + /// + /// Constructs a new instance of the class with the + /// specified detailed syntax error information. + /// + /// The decision number + /// The final configuration set reached during prediction + /// prior to reaching the state + /// The input token stream + /// The start index for the current prediction + /// The index at which the syntax error was identified + /// {@code true} if the syntax error was identified during LL + /// prediction; otherwise, {@code false} if the syntax error was identified + /// during SLL prediction + ErrorInfo(int decision, Ref configs, TokenStream *input, int startIndex, int stopIndex, bool fullCtx); + }; + +} // namespace atn +} // namespace runtime +} // namespace v4 +} // namespace antlr +} // namespace org diff --git a/runtime/Cpp/runtime/atn/LL1Analyzer.cpp b/runtime/Cpp/runtime/atn/LL1Analyzer.cpp index 91a2b13db..3109e9ba3 100755 --- a/runtime/Cpp/runtime/atn/LL1Analyzer.cpp +++ b/runtime/Cpp/runtime/atn/LL1Analyzer.cpp @@ -40,6 +40,8 @@ #include "ATNConfig.h" #include "EmptyPredictionContext.h" +#include "CPPUtils.h" + #include "LL1Analyzer.h" using namespace org::antlr::v4::runtime; @@ -60,7 +62,7 @@ std::vector LL1Analyzer::getDecisionLookahead(ATNState *s) co for (size_t alt = 0; alt < s->getNumberOfTransitions(); alt++) { bool seeThruPreds = false; // fail to get lookahead upon pred - std::unordered_set lookBusy; + std::unordered_set> lookBusy; antlrcpp::BitSet callRuleStack; _LOOK(s->transition(alt)->target, nullptr, PredictionContext::EMPTY, look[alt], lookBusy, callRuleStack, seeThruPreds, false); @@ -74,25 +76,25 @@ std::vector LL1Analyzer::getDecisionLookahead(ATNState *s) co return look; } -misc::IntervalSet LL1Analyzer::LOOK(ATNState *s, RuleContext::Ref ctx) const { +misc::IntervalSet LL1Analyzer::LOOK(ATNState *s, Ref ctx) const { return LOOK(s, nullptr, ctx); } -misc::IntervalSet LL1Analyzer::LOOK(ATNState *s, ATNState *stopState, RuleContext::Ref ctx) const { +misc::IntervalSet LL1Analyzer::LOOK(ATNState *s, ATNState *stopState, Ref ctx) const { misc::IntervalSet r; bool seeThruPreds = true; // ignore preds; get all lookahead - PredictionContext::Ref lookContext = ctx != nullptr ? PredictionContext::fromRuleContext(_atn, ctx) : nullptr; + Ref lookContext = ctx != nullptr ? PredictionContext::fromRuleContext(_atn, ctx) : nullptr; - std::unordered_set lookBusy; + std::unordered_set> lookBusy; antlrcpp::BitSet callRuleStack; _LOOK(s, stopState, lookContext, r, lookBusy, callRuleStack, seeThruPreds, true); return r; } -void LL1Analyzer::_LOOK(ATNState *s, ATNState *stopState, PredictionContext::Ref ctx, misc::IntervalSet &look, - std::unordered_set &lookBusy, antlrcpp::BitSet &calledRuleStack, bool seeThruPreds, bool addEOF) const { - ATNConfig::Ref c = std::make_shared(s, 0, ctx); +void LL1Analyzer::_LOOK(ATNState *s, ATNState *stopState, Ref ctx, misc::IntervalSet &look, + std::unordered_set> &lookBusy, antlrcpp::BitSet &calledRuleStack, bool seeThruPreds, bool addEOF) const { + Ref c = std::make_shared(s, 0, ctx); if (lookBusy.count(c) > 0) // Keep in mind comparison is based on members of the class, not the actual instance. return; @@ -109,7 +111,7 @@ void LL1Analyzer::_LOOK(ATNState *s, ATNState *stopState, PredictionContext::Ref } } - if (is(s)) { + if (is(s)) { if (ctx == nullptr) { look.add(Token::EPSILON); return; @@ -146,7 +148,7 @@ void LL1Analyzer::_LOOK(ATNState *s, ATNState *stopState, PredictionContext::Ref continue; } - PredictionContext::Ref newContext = SingletonPredictionContext::create(ctx, (static_cast(t))->followState->stateNumber); + Ref newContext = SingletonPredictionContext::create(ctx, (static_cast(t))->followState->stateNumber); auto onExit = finally([&] { calledRuleStack[(size_t)((static_cast(t))->target->ruleIndex)] = false; }); diff --git a/runtime/Cpp/runtime/atn/LL1Analyzer.h b/runtime/Cpp/runtime/atn/LL1Analyzer.h index fce7c81de..6c29c3055 100755 --- a/runtime/Cpp/runtime/atn/LL1Analyzer.h +++ b/runtime/Cpp/runtime/atn/LL1Analyzer.h @@ -80,7 +80,7 @@ namespace atn { /// /// The set of tokens that can follow {@code s} in the ATN in the /// specified {@code ctx}. - virtual misc::IntervalSet LOOK(ATNState *s, RuleContext::Ref ctx) const; + virtual misc::IntervalSet LOOK(ATNState *s, Ref ctx) const; /// /// Compute set of tokens that can follow {@code s} in the ATN in the @@ -99,7 +99,7 @@ namespace atn { /// /// The set of tokens that can follow {@code s} in the ATN in the /// specified {@code ctx}. - virtual misc::IntervalSet LOOK(ATNState *s, ATNState *stopState, RuleContext::Ref ctx) const; + virtual misc::IntervalSet LOOK(ATNState *s, ATNState *stopState, Ref ctx) const; /// /// Compute set of tokens that can follow {@code s} in the ATN in the @@ -131,8 +131,8 @@ namespace atn { /// outermost context is reached. This parameter has no effect if {@code ctx} /// is {@code null}. protected: - virtual void _LOOK(ATNState *s, ATNState *stopState, PredictionContext::Ref ctx, misc::IntervalSet &look, - std::unordered_set &lookBusy, antlrcpp::BitSet &calledRuleStack, bool seeThruPreds, bool addEOF) const; + virtual void _LOOK(ATNState *s, ATNState *stopState, Ref ctx, misc::IntervalSet &look, + std::unordered_set> &lookBusy, antlrcpp::BitSet &calledRuleStack, bool seeThruPreds, bool addEOF) const; }; } // namespace atn diff --git a/runtime/Cpp/runtime/atn/LexerATNConfig.cpp b/runtime/Cpp/runtime/atn/LexerATNConfig.cpp index def568bba..3b9d4723b 100755 --- a/runtime/Cpp/runtime/atn/LexerATNConfig.cpp +++ b/runtime/Cpp/runtime/atn/LexerATNConfig.cpp @@ -33,6 +33,8 @@ #include "DecisionState.h" #include "PredictionContext.h" #include "SemanticContext.h" +#include "LexerActionExecutor.h" + #include "CPPUtils.h" #include "LexerATNConfig.h" @@ -40,32 +42,37 @@ using namespace org::antlr::v4::runtime::atn; using namespace antlrcpp; -LexerATNConfig::LexerATNConfig(ATNState *state, int alt, PredictionContext::Ref context) - : ATNConfig(state, alt, context, SemanticContext::NONE), passedThroughNonGreedyDecision(false) { +LexerATNConfig::LexerATNConfig(ATNState *state, int alt, Ref context) + : ATNConfig(state, alt, context, SemanticContext::NONE), _passedThroughNonGreedyDecision(false) { } -LexerATNConfig::LexerATNConfig(ATNState *state, int alt, PredictionContext::Ref context, int actionIndex) - : ATNConfig(state, alt, context, SemanticContext::NONE), passedThroughNonGreedyDecision(false) { - lexerActionIndex = actionIndex; +LexerATNConfig::LexerATNConfig(ATNState *state, int alt, Ref context, + Ref lexerActionExecutor) + : ATNConfig(state, alt, context, SemanticContext::NONE), _passedThroughNonGreedyDecision(false), + _lexerActionExecutor(lexerActionExecutor) { } -LexerATNConfig::LexerATNConfig(LexerATNConfig::Ref c, ATNState *state) - : ATNConfig(c, state, c->context, c->semanticContext), passedThroughNonGreedyDecision(checkNonGreedyDecision(c, state)) { - lexerActionIndex = c->lexerActionIndex; +LexerATNConfig::LexerATNConfig(Ref c, ATNState *state) + : ATNConfig(c, state, c->context, c->semanticContext), _passedThroughNonGreedyDecision(checkNonGreedyDecision(c, state)), + _lexerActionExecutor(c->_lexerActionExecutor) { } -LexerATNConfig::LexerATNConfig(LexerATNConfig::Ref c, ATNState *state, int actionIndex) - : ATNConfig(c, state, c->context, c->semanticContext), passedThroughNonGreedyDecision(checkNonGreedyDecision(c, state)) { - lexerActionIndex = actionIndex; +LexerATNConfig::LexerATNConfig(Ref c, ATNState *state, Ref lexerActionExecutor) + : ATNConfig(c, state, c->context, c->semanticContext), _passedThroughNonGreedyDecision(checkNonGreedyDecision(c, state)), + _lexerActionExecutor(lexerActionExecutor) { } -LexerATNConfig::LexerATNConfig(LexerATNConfig::Ref c, ATNState *state, PredictionContext::Ref context) - : ATNConfig(c, state, context, c->semanticContext), passedThroughNonGreedyDecision(checkNonGreedyDecision(c, state)) { - lexerActionIndex = c->lexerActionIndex; +LexerATNConfig::LexerATNConfig(Ref c, ATNState *state, Ref context) + : ATNConfig(c, state, context, c->semanticContext), _passedThroughNonGreedyDecision(checkNonGreedyDecision(c, state)), + _lexerActionExecutor(c->_lexerActionExecutor) { +} + +Ref LexerATNConfig::getLexerActionExecutor() const { + return _lexerActionExecutor; } bool LexerATNConfig::hasPassedThroughNonGreedyDecision() { - return passedThroughNonGreedyDecision; + return _passedThroughNonGreedyDecision; } size_t LexerATNConfig::hashCode() const { @@ -74,20 +81,25 @@ size_t LexerATNConfig::hashCode() const { hashCode = misc::MurmurHash::update(hashCode, (size_t)alt); hashCode = misc::MurmurHash::update(hashCode, (size_t)context.get()); hashCode = misc::MurmurHash::update(hashCode, (size_t)semanticContext.get()); - hashCode = misc::MurmurHash::update(hashCode, passedThroughNonGreedyDecision ? 1 : 0); - hashCode = misc::MurmurHash::finish(hashCode, 5); + hashCode = misc::MurmurHash::update(hashCode, _passedThroughNonGreedyDecision ? 1 : 0); + hashCode = misc::MurmurHash::update(hashCode, (size_t)_lexerActionExecutor.get()); + hashCode = misc::MurmurHash::finish(hashCode, 6); return hashCode; } bool LexerATNConfig::operator == (const LexerATNConfig& other) const { - if (passedThroughNonGreedyDecision != other.passedThroughNonGreedyDecision) + if (_passedThroughNonGreedyDecision != other._passedThroughNonGreedyDecision) return false; + if (_lexerActionExecutor != other._lexerActionExecutor) { + return false; + } + return ATNConfig::operator == (other); } -bool LexerATNConfig::checkNonGreedyDecision(LexerATNConfig::Ref source, ATNState *target) { - return source->passedThroughNonGreedyDecision || +bool LexerATNConfig::checkNonGreedyDecision(Ref source, ATNState *target) { + return source->_passedThroughNonGreedyDecision || (is(target) && (static_cast(target))->nonGreedy); } diff --git a/runtime/Cpp/runtime/atn/LexerATNConfig.h b/runtime/Cpp/runtime/atn/LexerATNConfig.h index 0586b02f7..b433f24fd 100755 --- a/runtime/Cpp/runtime/atn/LexerATNConfig.h +++ b/runtime/Cpp/runtime/atn/LexerATNConfig.h @@ -41,18 +41,18 @@ namespace atn { class LexerATNConfig : public ATNConfig { public: - typedef std::shared_ptr Ref; - - /// Capture lexer action we traverse. - int lexerActionIndex = -1; + LexerATNConfig(ATNState *state, int alt, Ref context); + LexerATNConfig(ATNState *state, int alt, Ref context, Ref lexerActionExecutor); - explicit LexerATNConfig(ATNState *state, int alt, PredictionContext::Ref context); - explicit LexerATNConfig(ATNState *state, int alt, PredictionContext::Ref context, int actionIndex); - - explicit LexerATNConfig(LexerATNConfig::Ref c, ATNState *state); - explicit LexerATNConfig(LexerATNConfig::Ref c, ATNState *state, int actionIndex); - explicit LexerATNConfig(LexerATNConfig::Ref c, ATNState *state, PredictionContext::Ref context); + LexerATNConfig(Ref c, ATNState *state); + LexerATNConfig(Ref c, ATNState *state, Ref lexerActionExecutor); + LexerATNConfig(Ref c, ATNState *state, Ref context); + /** + * Gets the {@link LexerActionExecutor} capable of executing the embedded + * action(s) for the current configuration. + */ + Ref getLexerActionExecutor() const; bool hasPassedThroughNonGreedyDecision(); virtual size_t hashCode() const override; @@ -60,9 +60,13 @@ namespace atn { bool operator == (const LexerATNConfig& other) const; private: - const bool passedThroughNonGreedyDecision; + /** + * This is the backing field for {@link #getLexerActionExecutor}. + */ + const Ref _lexerActionExecutor; + const bool _passedThroughNonGreedyDecision; - static bool checkNonGreedyDecision(LexerATNConfig::Ref source, ATNState *target); + static bool checkNonGreedyDecision(Ref source, ATNState *target); }; } // namespace atn diff --git a/runtime/Cpp/runtime/atn/LexerATNSimulator.cpp b/runtime/Cpp/runtime/atn/LexerATNSimulator.cpp index 5db78f9b7..b169df0b7 100755 --- a/runtime/Cpp/runtime/atn/LexerATNSimulator.cpp +++ b/runtime/Cpp/runtime/atn/LexerATNSimulator.cpp @@ -44,6 +44,7 @@ #include "DFAState.h" #include "LexerATNConfig.h" +#include "LexerActionExecutor.h" #include "EmptyPredictionContext.h" #include "LexerATNSimulator.h" @@ -69,12 +70,12 @@ int LexerATNSimulator::match_calls = 0; LexerATNSimulator::LexerATNSimulator(const ATN &atn, std::vector &decisionToDFA, - std::shared_ptr sharedContextCache) + Ref sharedContextCache) : LexerATNSimulator(nullptr, atn, decisionToDFA, sharedContextCache) { } LexerATNSimulator::LexerATNSimulator(Lexer *recog, const ATN &atn, std::vector &decisionToDFA, - std::shared_ptr sharedContextCache) + Ref sharedContextCache) : ATNSimulator(atn, sharedContextCache), _recog(recog), _decisionToDFA(decisionToDFA) { InitializeInstanceFields(); } @@ -96,7 +97,7 @@ int LexerATNSimulator::match(CharStream *input, size_t mode) { }); _startIndex = (int)input->index(); - prevAccept.reset(); + _prevAccept.reset(); const dfa::DFA &dfa = _decisionToDFA[mode]; if (dfa.s0 == nullptr) { return matchATN(input); @@ -108,7 +109,7 @@ int LexerATNSimulator::match(CharStream *input, size_t mode) { } void LexerATNSimulator::reset() { - prevAccept.reset(); + _prevAccept.reset(); _startIndex = 0; // Originally -1, but that would require a signed type with many casts. // The initial value is never tested, so it doesn't matter which value is set here. _line = 1; @@ -116,6 +117,14 @@ void LexerATNSimulator::reset() { _mode = Lexer::DEFAULT_MODE; } +void LexerATNSimulator::clearDFA() { + int size = (int)_decisionToDFA.size(); + _decisionToDFA.clear(); + for (int d = 0; d < size; ++d) { + _decisionToDFA.push_back(dfa::DFA(atn.getDecisionState(d), d)); + } +} + int LexerATNSimulator::matchATN(CharStream *input) { ATNState *startState = (ATNState *)atn.modeToStartState[_mode]; @@ -125,7 +134,7 @@ int LexerATNSimulator::matchATN(CharStream *input) { size_t old_mode = _mode; - std::shared_ptr s0_closure = computeStartState(input, startState); + Ref s0_closure = computeStartState(input, startState); bool suppressEdge = s0_closure->hasSemanticContext; s0_closure->hasSemanticContext = false; @@ -149,6 +158,12 @@ int LexerATNSimulator::execATN(CharStream *input, dfa::DFAState *ds0) { std::wcout << L"start state closure=" << ds0->configs << std::endl; } + if (ds0->isAcceptState) { + // allow zero-length tokens + // ml: in Java code this method uses 3 params. The first is a member var of the class anyway (_prevAccept), so why pass it here? + captureSimState(input, ds0); + } + ssize_t t = input->LA(1); dfa::DFAState *s = ds0; // s is current/from DFA state @@ -183,6 +198,14 @@ int LexerATNSimulator::execATN(CharStream *input, dfa::DFAState *ds0) { break; } + // If this is a consumable input element, make sure to consume before + // capturing the accept state so the input index, line, and char + // position accurately reflect the state of the interpreter at the + // end of the token. + if (t != EOF) { + consume(input); + } + if (target->isAcceptState) { captureSimState(input, target); if (t == EOF) { @@ -190,11 +213,7 @@ int LexerATNSimulator::execATN(CharStream *input, dfa::DFAState *ds0) { } } - if (t != EOF) { - consume(input); - t = input->LA(1); - } - + t = input->LA(1); s = target; // flip; current DFA target becomes new src/from state } @@ -215,16 +234,19 @@ dfa::DFAState *LexerATNSimulator::getExistingTargetState(dfa::DFAState *s, ssize } dfa::DFAState *LexerATNSimulator::computeTargetState(CharStream *input, dfa::DFAState *s, ssize_t t) { - std::shared_ptr reach = std::make_shared(); + Ref reach = std::make_shared(); // if we don't find an existing DFA state // Fill reach starting from closure, following t transitions getReachableConfigSet(input, s->configs, reach, t); if (reach->isEmpty()) { // we got nowhere on t from s - // we got nowhere on t, don't throw out this knowledge; it'd - // cause a failover from DFA later. - addDFAEdge(s, t, ERROR.get()); + if (!reach->hasSemanticContext) { + // we got nowhere on t, don't throw out this knowledge; it'd + // cause a failover from DFA later. + addDFAEdge(s, t, ERROR.get()); + } + // stop when we can't match any more char return ERROR.get(); } @@ -233,12 +255,11 @@ dfa::DFAState *LexerATNSimulator::computeTargetState(CharStream *input, dfa::DFA return addDFAEdge(s, t, reach); } -int LexerATNSimulator::failOrAccept(CharStream *input, std::shared_ptr reach, ssize_t t) { - if (prevAccept.dfaState != nullptr) { - int ruleIndex = prevAccept.dfaState->lexerRuleIndex; - int actionIndex = prevAccept.dfaState->lexerActionIndex; - accept(input, ruleIndex, actionIndex, (size_t)prevAccept.index, prevAccept.line, (size_t)prevAccept.charPos); - return prevAccept.dfaState->prediction; +int LexerATNSimulator::failOrAccept(CharStream *input, Ref reach, ssize_t t) { + if (_prevAccept.dfaState != nullptr) { + Ref lexerActionExecutor = _prevAccept.dfaState->lexerActionExecutor; + accept(input, lexerActionExecutor, _startIndex, (size_t)_prevAccept.index, _prevAccept.line, (size_t)_prevAccept.charPos); + return _prevAccept.dfaState->prediction; } else { // if no accept and EOF is first char, return EOF if (t == EOF && input->index() == (size_t)_startIndex) { @@ -249,13 +270,13 @@ int LexerATNSimulator::failOrAccept(CharStream *input, std::shared_ptr closure, - std::shared_ptr reach, ssize_t t) { +void LexerATNSimulator::getReachableConfigSet(CharStream *input, Ref closure_, + Ref reach, ssize_t t) { // this is used to skip processing for configs which have a lower priority // than a config that already reached an accept state for the same rule int skipAlt = ATN::INVALID_ALT_NUMBER; - for (auto c : closure->configs) { + for (auto c : closure_->configs) { bool currentAltReachedAcceptState = c->alt == skipAlt; if (currentAltReachedAcceptState && (std::static_pointer_cast(c))->hasPassedThroughNonGreedyDecision()) { continue; @@ -270,8 +291,16 @@ void LexerATNSimulator::getReachableConfigSet(CharStream *input, std::shared_ptr Transition *trans = c->state->transition(ti); ATNState *target = getReachableTarget(trans, (int)t); if (target != nullptr) { - if (this->closure(input, std::make_shared(std::static_pointer_cast(c), target), - reach, currentAltReachedAcceptState, true)) { + Ref lexerActionExecutor = std::static_pointer_cast(c)->getLexerActionExecutor(); + if (lexerActionExecutor != nullptr) { + lexerActionExecutor = lexerActionExecutor->fixOffsetBeforeMatch((int)input->index() - _startIndex); + } + + bool treatEofAsEpsilon = t == EOF; + Ref config = std::make_shared(std::static_pointer_cast(c), + target, lexerActionExecutor); + + if (closure(input, config, reach, currentAltReachedAcceptState, true, treatEofAsEpsilon)) { // any remaining configs for this alt have a lower priority than // the one that just reached an accept state. skipAlt = c->alt; @@ -282,27 +311,20 @@ void LexerATNSimulator::getReachableConfigSet(CharStream *input, std::shared_ptr } } -void LexerATNSimulator::accept(CharStream *input, int ruleIndex, int actionIndex, size_t index, size_t line, size_t charPos) { +void LexerATNSimulator::accept(CharStream *input, Ref lexerActionExecutor, int startIndex, + size_t index, size_t line, size_t charPos) { if (debug) { std::wcout << L"ACTION "; - if (_recog != nullptr) { - std::wcout << _recog->getRuleNames()[(size_t)ruleIndex]; - } else { - std::wcout << ruleIndex; - } - std::wcout << ":" << actionIndex << std::endl; - } - - if (actionIndex >= 0 && _recog != nullptr) { - _recog->action(nullptr, ruleIndex, actionIndex); + std::wcout << toString(lexerActionExecutor) << std::endl; } // seek to after last char in token input->seek(index); _line = line; _charPositionInLine = (int)charPos; - if (input->LA(1) != EOF) { - consume(input); + + if (lexerActionExecutor != nullptr && _recog != nullptr) { + lexerActionExecutor->execute(_recog, input, _startIndex); } } @@ -314,19 +336,19 @@ atn::ATNState *LexerATNSimulator::getReachableTarget(Transition *trans, ssize_t return nullptr; } -std::shared_ptr LexerATNSimulator::computeStartState(CharStream *input, ATNState *p) { - std::shared_ptr initialContext = PredictionContext::EMPTY; // ml: the purpose of this assignment is unclear - std::shared_ptr configs = std::make_shared(); +Ref LexerATNSimulator::computeStartState(CharStream *input, ATNState *p) { + Ref initialContext = PredictionContext::EMPTY; // ml: the purpose of this assignment is unclear + Ref configs = std::make_shared(); for (size_t i = 0; i < p->getNumberOfTransitions(); i++) { ATNState *target = p->transition(i)->target; - LexerATNConfig::Ref c = std::make_shared(target, (int)(i + 1), initialContext); - closure(input, c, configs, false, false); + Ref c = std::make_shared(target, (int)(i + 1), initialContext); + closure(input, c, configs, false, false, false); } return configs; } -bool LexerATNSimulator::closure(CharStream *input, LexerATNConfig::Ref config, std::shared_ptr configs, - bool currentAltReachedAcceptState, bool speculative) { +bool LexerATNSimulator::closure(CharStream *input, Ref config, Ref configs, + bool currentAltReachedAcceptState, bool speculative, bool treatEofAsEpsilon) { if (debug) { std::wcout << L"closure(" << config->toString(true) << L")" << std::endl; } @@ -355,8 +377,8 @@ bool LexerATNSimulator::closure(CharStream *input, LexerATNConfig::Ref config, s if (config->context->getReturnState(i) != PredictionContext::EMPTY_RETURN_STATE) { std::weak_ptr newContext = config->context->getParent(i); // "pop" return state ATNState *returnState = atn.states[(size_t)config->context->getReturnState(i)]; - LexerATNConfig::Ref c = std::make_shared(returnState, config->alt, newContext.lock()); - currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState, speculative); + Ref c = std::make_shared(config, returnState, newContext.lock()); + currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon); } } } @@ -374,22 +396,23 @@ bool LexerATNSimulator::closure(CharStream *input, LexerATNConfig::Ref config, s ATNState *p = config->state; for (size_t i = 0; i < p->getNumberOfTransitions(); i++) { Transition *t = p->transition(i); - LexerATNConfig::Ref c = getEpsilonTarget(input, config, t, configs, speculative); + Ref c = getEpsilonTarget(input, config, t, configs, speculative, treatEofAsEpsilon); if (c != nullptr) { - currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState, speculative); + currentAltReachedAcceptState = closure(input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon); } } return currentAltReachedAcceptState; } -LexerATNConfig::Ref LexerATNSimulator::getEpsilonTarget(CharStream *input, LexerATNConfig::Ref config, Transition *t, - std::shared_ptr configs, bool speculative) { - LexerATNConfig::Ref c = nullptr; +Ref LexerATNSimulator::getEpsilonTarget(CharStream *input, Ref config, Transition *t, + Ref configs, bool speculative, bool treatEofAsEpsilon) { + + Ref c = nullptr; switch (t->getSerializationType()) { case Transition::RULE: { RuleTransition *ruleTransition = static_cast(t); - PredictionContext::Ref newContext = SingletonPredictionContext::create(config->context, ruleTransition->followState->stateNumber); + Ref newContext = SingletonPredictionContext::create(config->context, ruleTransition->followState->stateNumber); c = std::make_shared(config, t->target, newContext); } break; @@ -426,13 +449,47 @@ LexerATNConfig::Ref LexerATNSimulator::getEpsilonTarget(CharStream *input, Lexer } } break; - // ignore actions; just exec one per rule upon accept + case Transition::ACTION: - c = std::make_shared(config, t->target, (static_cast(t))->actionIndex); - break; + if (config->context == nullptr|| config->context->hasEmptyPath()) { + // execute actions anywhere in the start rule for a token. + // + // TODO: if the entry rule is invoked recursively, some + // actions may be executed during the recursive call. The + // problem can appear when hasEmptyPath() is true but + // isEmpty() is false. In this case, the config needs to be + // split into two contexts - one with just the empty path + // and another with everything but the empty path. + // Unfortunately, the current algorithm does not allow + // getEpsilonTarget to return two configurations, so + // additional modifications are needed before we can support + // the split operation. + Ref lexerActionExecutor = LexerActionExecutor::append(config->getLexerActionExecutor(), + atn.lexerActions[static_cast(t)->actionIndex]); + c = std::make_shared(config, t->target, lexerActionExecutor); + break; + } + else { + // ignore actions in referenced rules + c = std::make_shared(config, t->target); + break; + } + case Transition::EPSILON: c = std::make_shared(config, t->target); break; + + case Transition::ATOM: + case Transition::RANGE: + case Transition::SET: + if (treatEofAsEpsilon) { + if (t->matches(EOF, Lexer::MIN_CHAR_VALUE, Lexer::MAX_CHAR_VALUE)) { + c = std::make_shared(config, t->target); + break; + } + } + + break; } return c; @@ -465,13 +522,13 @@ bool LexerATNSimulator::evaluatePredicate(CharStream *input, int ruleIndex, int } void LexerATNSimulator::captureSimState(CharStream *input, dfa::DFAState *dfaState) { - prevAccept.index = (int)input->index(); - prevAccept.line = _line; - prevAccept.charPos = _charPositionInLine; - prevAccept.dfaState = dfaState; + _prevAccept.index = (int)input->index(); + _prevAccept.line = _line; + _prevAccept.charPos = _charPositionInLine; + _prevAccept.dfaState = dfaState; } -dfa::DFAState *LexerATNSimulator::addDFAEdge(dfa::DFAState *from, ssize_t t, std::shared_ptr q) { +dfa::DFAState *LexerATNSimulator::addDFAEdge(dfa::DFAState *from, ssize_t t, Ref q) { /* leading to this call, ATNConfigSet.hasSemanticContext is used as a * marker indicating dynamic predicate evaluation makes this edge * dependent on the specific input sequence, so the static edge in the @@ -514,14 +571,14 @@ void LexerATNSimulator::addDFAEdge(dfa::DFAState *p, ssize_t t, dfa::DFAState *q p->edges[(size_t)(t - MIN_DFA_EDGE)] = q; // connect } -dfa::DFAState *LexerATNSimulator::addDFAState(std::shared_ptr configs) { +dfa::DFAState *LexerATNSimulator::addDFAState(Ref configs) { /* the lexer evaluates predicates on-the-fly; by this point configs * should not contain any configurations with unevaluated predicates. */ assert(!configs->hasSemanticContext); dfa::DFAState *proposed = new dfa::DFAState(configs); /* mem-check: managed by the DFA or deleted below */ - ATNConfig::Ref firstConfigWithRuleStopState = nullptr; + Ref firstConfigWithRuleStopState = nullptr; for (auto c : configs->configs) { if (is(c->state)) { firstConfigWithRuleStopState = c; @@ -531,9 +588,8 @@ dfa::DFAState *LexerATNSimulator::addDFAState(std::shared_ptr conf if (firstConfigWithRuleStopState != nullptr) { proposed->isAcceptState = true; - proposed->lexerRuleIndex = firstConfigWithRuleStopState->state->ruleIndex; - proposed->lexerActionIndex = (std::static_pointer_cast(firstConfigWithRuleStopState))->lexerActionIndex; - proposed->prediction = atn.ruleToTokenType[(size_t)proposed->lexerRuleIndex]; + proposed->lexerActionExecutor = std::dynamic_pointer_cast(firstConfigWithRuleStopState)->getLexerActionExecutor(); + proposed->prediction = atn.ruleToTokenType[firstConfigWithRuleStopState->state->ruleIndex]; } dfa::DFA &dfa = _decisionToDFA[_mode]; diff --git a/runtime/Cpp/runtime/atn/LexerATNSimulator.h b/runtime/Cpp/runtime/atn/LexerATNSimulator.h index 9e4b05752..67e96062d 100755 --- a/runtime/Cpp/runtime/atn/LexerATNSimulator.h +++ b/runtime/Cpp/runtime/atn/LexerATNSimulator.h @@ -33,6 +33,7 @@ #include "ATNSimulator.h" #include "LexerATNConfig.h" +#include "ATNConfigSet.h" namespace org { namespace antlr { @@ -109,22 +110,23 @@ namespace atn { protected: size_t _mode; - /// - /// Used during DFA/ATN exec to record the most recent accept configuration info - SimState prevAccept; + /// Used during DFA/ATN exec to record the most recent accept configuration info. + SimState _prevAccept; public: static int match_calls; LexerATNSimulator(const ATN &atn, std::vector &decisionToDFA, - std::shared_ptr sharedContextCache); + Ref sharedContextCache); LexerATNSimulator(Lexer *recog, const ATN &atn, std::vector &decisionToDFA, - std::shared_ptr sharedContextCache); + Ref sharedContextCache); virtual void copyState(LexerATNSimulator *simulator); virtual int match(CharStream *input, size_t mode); virtual void reset() override; + virtual void clearDFA() override; + protected: virtual int matchATN(CharStream *input); virtual int execATN(CharStream *input, dfa::DFAState *ds0); @@ -154,20 +156,22 @@ namespace atn { /// returns . virtual dfa::DFAState *computeTargetState(CharStream *input, dfa::DFAState *s, ssize_t t); - virtual int failOrAccept(CharStream *input, std::shared_ptr reach, ssize_t t); + virtual int failOrAccept(CharStream *input, Ref reach, ssize_t t); /// /// Given a starting configuration set, figure out all ATN configurations /// we can reach upon input {@code t}. Parameter {@code reach} is a return /// parameter. /// - void getReachableConfigSet(CharStream *input, std::shared_ptr closure, std::shared_ptr reach, ssize_t t); + void getReachableConfigSet(CharStream *input, Ref closure_, // closure_ as we have a closure() already + Ref reach, ssize_t t); - virtual void accept(CharStream *input, int ruleIndex, int actionIndex, size_t index, size_t line, size_t charPos); + virtual void accept(CharStream *input, Ref lexerActionExecutor, int startIndex, size_t index, + size_t line, size_t charPos); virtual ATNState *getReachableTarget(Transition *trans, ssize_t t); - virtual std::shared_ptr computeStartState(CharStream *input, ATNState *p); + virtual Ref computeStartState(CharStream *input, ATNState *p); /// /// Since the alternatives within any lexer decision are ordered by @@ -178,12 +182,12 @@ namespace atn { /// /// {@code true} if an accept state is reached, otherwise /// {@code false}. - virtual bool closure(CharStream *input, LexerATNConfig::Ref config, std::shared_ptr configs, - bool currentAltReachedAcceptState, bool speculative); + virtual bool closure(CharStream *input, Ref config, Ref configs, + bool currentAltReachedAcceptState, bool speculative, bool treatEofAsEpsilon); // side-effect: can alter configs.hasSemanticContext - virtual LexerATNConfig::Ref getEpsilonTarget(CharStream *input, LexerATNConfig::Ref config, Transition *t, - std::shared_ptr configs, bool speculative); + virtual Ref getEpsilonTarget(CharStream *input, Ref config, Transition *t, + Ref configs, bool speculative, bool treatEofAsEpsilon); /// /// Evaluate a predicate specified in the lexer. @@ -208,7 +212,7 @@ namespace atn { virtual bool evaluatePredicate(CharStream *input, int ruleIndex, int predIndex, bool speculative); virtual void captureSimState(CharStream *input, dfa::DFAState *dfaState); - virtual dfa::DFAState *addDFAEdge(dfa::DFAState *from, ssize_t t, std::shared_ptr q); + virtual dfa::DFAState* addDFAEdge(dfa::DFAState *from, ssize_t t, Ref q); virtual void addDFAEdge(dfa::DFAState *p, ssize_t t, dfa::DFAState *q); /// @@ -217,7 +221,7 @@ namespace atn { /// configuration containing an ATN rule stop state. Later, when /// traversing the DFA, we will know which rule to accept. /// - virtual dfa::DFAState *addDFAState(std::shared_ptr configs); + virtual dfa::DFAState *addDFAState(Ref configs); public: dfa::DFA& getDFA(size_t mode); diff --git a/runtime/Cpp/runtime/atn/LexerAction.h b/runtime/Cpp/runtime/atn/LexerAction.h index ba14991b8..3b9432e0e 100755 --- a/runtime/Cpp/runtime/atn/LexerAction.h +++ b/runtime/Cpp/runtime/atn/LexerAction.h @@ -32,7 +32,6 @@ #pragma once #include "LexerActionType.h" -#include "Lexer.h" namespace org { namespace antlr { @@ -50,8 +49,6 @@ namespace atn { /// class LexerAction { public: - typedef std::shared_ptr Ref; - virtual ~LexerAction() {}; /// @@ -82,7 +79,7 @@ namespace atn { /// positioned correctly prior to calling this method. /// /// The lexer instance. - virtual void execute(Lexer::Ref lexer) = 0; + virtual void execute(Lexer *lexer) = 0; virtual size_t hashCode() const = 0; virtual bool operator == (const LexerAction &obj) const = 0; diff --git a/runtime/Cpp/runtime/atn/LexerActionExecutor.cpp b/runtime/Cpp/runtime/atn/LexerActionExecutor.cpp new file mode 100755 index 000000000..367bb8ec5 --- /dev/null +++ b/runtime/Cpp/runtime/atn/LexerActionExecutor.cpp @@ -0,0 +1,124 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Dan McLaughlin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "MurmurHash.h" +#include "LexerIndexedCustomAction.h" +#include "CPPUtils.h" +#include "Arrays.h" + +#include "LexerActionExecutor.h" + +using namespace org::antlr::v4::runtime::atn; +using namespace org::antlr::v4::runtime::misc; +using namespace antlrcpp; + +LexerActionExecutor::LexerActionExecutor(const std::vector> &lexerActions) + : _lexerActions(lexerActions), _hashCode(generateHashCode()) { +} + +Ref LexerActionExecutor::append(Ref lexerActionExecutor, Ref lexerAction) { + if (lexerActionExecutor == nullptr) { + return std::make_shared(std::vector> { lexerAction }); + } + + std::vector> lexerActions = lexerActionExecutor->_lexerActions; // Make a copy. + lexerActions.push_back(lexerAction); + return std::make_shared(lexerActions); +} + +Ref LexerActionExecutor::fixOffsetBeforeMatch(int offset) { + std::vector> updatedLexerActions; + for (size_t i = 0; i < _lexerActions.size(); i++) { + if (_lexerActions[i]->isPositionDependent() && !is(_lexerActions[i])) { + if (updatedLexerActions.empty()) { + updatedLexerActions = _lexerActions; // Make a copy. + } + + updatedLexerActions[i] = std::make_shared(offset, _lexerActions[i]); + } + } + + if (updatedLexerActions.empty()) { + return shared_from_this(); + } + + return std::make_shared(updatedLexerActions); +} + +std::vector> LexerActionExecutor::getLexerActions() const { + return _lexerActions; +} + +void LexerActionExecutor::execute(Lexer *lexer, CharStream *input, int startIndex) { + bool requiresSeek = false; + size_t stopIndex = input->index(); + + auto onExit = finally([requiresSeek, input, stopIndex]() { + if (requiresSeek) { + input->seek(stopIndex); + } + }); + for (auto lexerAction : _lexerActions) { + if (is(lexerAction)) { + int offset = (std::static_pointer_cast(lexerAction))->getOffset(); + input->seek(startIndex + offset); + lexerAction = std::static_pointer_cast(lexerAction)->getAction(); + requiresSeek = (size_t)(startIndex + offset) != stopIndex; + } else if (lexerAction->isPositionDependent()) { + input->seek(stopIndex); + requiresSeek = false; + } + + lexerAction->execute(lexer); + } +} + +size_t LexerActionExecutor::hashCode() const { + return _hashCode; +} + +bool LexerActionExecutor::operator == (const LexerActionExecutor &obj) const { + if (&obj == this) { + return true; + } + + return _hashCode == obj._hashCode && Arrays::equals(_lexerActions, obj._lexerActions); +} + +size_t LexerActionExecutor::generateHashCode() const { + size_t hash = MurmurHash::initialize(); + for (auto lexerAction : _lexerActions) { + hash = MurmurHash::update(hash, (size_t)lexerAction.get()); + } + MurmurHash::finish(hash, _lexerActions.size()); + + return hash; +} diff --git a/runtime/Cpp/runtime/atn/LexerActionExecutor.h b/runtime/Cpp/runtime/atn/LexerActionExecutor.h new file mode 100755 index 000000000..35e1d12b5 --- /dev/null +++ b/runtime/Cpp/runtime/atn/LexerActionExecutor.h @@ -0,0 +1,144 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Dan McLaughlin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "CharStream.h" +#include "LexerAction.h" + +namespace org { +namespace antlr { +namespace v4 { +namespace runtime { +namespace atn { + + /// Represents an executor for a sequence of lexer actions which traversed during + /// the matching operation of a lexer rule (token). + /// + /// The executor tracks position information for position-dependent lexer actions + /// efficiently, ensuring that actions appearing only at the end of the rule do + /// not cause bloating of the created for the lexer. + class LexerActionExecutor : public std::enable_shared_from_this { + public: + /// + /// Constructs an executor for a sequence of actions. + /// The lexer actions to execute. + LexerActionExecutor(const std::vector> &lexerActions); + + /// + /// Creates a which executes the actions for + /// the input {@code lexerActionExecutor} followed by a specified + /// {@code lexerAction}. + /// + /// The executor for actions already traversed by + /// the lexer while matching a token within a particular + /// . If this is {@code null}, the method behaves as + /// though it were an empty executor. + /// The lexer action to execute after the actions + /// specified in {@code lexerActionExecutor}. + /// + /// A for executing the combine actions + /// of {@code lexerActionExecutor} and {@code lexerAction}. + static Ref append(Ref lexerActionExecutor, const Ref lexerAction); + + /// + /// Creates a which encodes the current offset + /// for position-dependent lexer actions. + /// + /// Normally, when the executor encounters lexer actions where + /// returns {@code true}, it calls + /// on the input to set the input + /// position to the end of the current token. This behavior provides + /// for efficient DFA representation of lexer actions which appear at the end + /// of a lexer rule, even when the lexer rule matches a variable number of + /// characters. + /// + /// Prior to traversing a match transition in the ATN, the current offset + /// from the token start index is assigned to all position-dependent lexer + /// actions which have not already been assigned a fixed offset. By storing + /// the offsets relative to the token start index, the DFA representation of + /// lexer actions which appear in the middle of tokens remains efficient due + /// to sharing among tokens of the same length, regardless of their absolute + /// position in the input stream. + /// + /// If the current executor already has offsets assigned to all + /// position-dependent lexer actions, the method returns {@code this}. + /// + /// The current offset to assign to all position-dependent + /// lexer actions which do not already have offsets assigned. + /// + /// A which stores input stream offsets + /// for all position-dependent lexer actions. + virtual Ref fixOffsetBeforeMatch(int offset); + + /// + /// Gets the lexer actions to be executed by this executor. + /// The lexer actions to be executed by this executor. + virtual std::vector> getLexerActions() const; + + /// + /// Execute the actions encapsulated by this executor within the context of a + /// particular . + /// + /// This method calls to set the position of the + /// {@code input} prior to calling + /// on a position-dependent action. Before the + /// method returns, the input position will be restored to the same position + /// it was in when the method was invoked. + /// + /// The lexer instance. + /// The input stream which is the source for the current token. + /// When this method is called, the current for + /// {@code input} should be the start of the following token, i.e. 1 + /// character past the end of the current token. + /// The token start index. This value may be passed to + /// to set the {@code input} position to the beginning + /// of the token. + virtual void execute(Lexer *lexer, CharStream *input, int startIndex); + + virtual size_t hashCode() const; + virtual bool operator == (const LexerActionExecutor &obj) const; + + private: + const std::vector> _lexerActions; + + /// Caches the result of since the hash code is an element + /// of the performance-critical operation. + const size_t _hashCode; + + size_t generateHashCode() const; + }; + +} // namespace atn +} // namespace runtime +} // namespace v4 +} // namespace antlr +} // namespace org diff --git a/runtime/Cpp/runtime/atn/LexerChannelAction.cpp b/runtime/Cpp/runtime/atn/LexerChannelAction.cpp index 992fa8479..c661b0dc6 100755 --- a/runtime/Cpp/runtime/atn/LexerChannelAction.cpp +++ b/runtime/Cpp/runtime/atn/LexerChannelAction.cpp @@ -30,6 +30,7 @@ */ #include "MurmurHash.h" +#include "Lexer.h" #include "LexerChannelAction.h" @@ -51,7 +52,7 @@ bool LexerChannelAction::isPositionDependent() const { return false; } -void LexerChannelAction::execute(Lexer::Ref lexer) { +void LexerChannelAction::execute(Lexer *lexer) { lexer->setChannel(_channel); } diff --git a/runtime/Cpp/runtime/atn/LexerChannelAction.h b/runtime/Cpp/runtime/atn/LexerChannelAction.h index 104f78014..134c7e38f 100755 --- a/runtime/Cpp/runtime/atn/LexerChannelAction.h +++ b/runtime/Cpp/runtime/atn/LexerChannelAction.h @@ -78,7 +78,7 @@ namespace atn { /// This action is implemented by calling with the /// value provided by . /// - virtual void execute(Lexer::Ref lexer) override; + virtual void execute(Lexer *lexer) override; virtual size_t hashCode() const override; virtual bool operator == (const LexerAction &obj) const override; diff --git a/runtime/Cpp/runtime/atn/LexerCustomAction.cpp b/runtime/Cpp/runtime/atn/LexerCustomAction.cpp index 377499852..b90932d9e 100755 --- a/runtime/Cpp/runtime/atn/LexerCustomAction.cpp +++ b/runtime/Cpp/runtime/atn/LexerCustomAction.cpp @@ -31,6 +31,7 @@ #include "MurmurHash.h" #include "CPPUtils.h" +#include "Lexer.h" #include "LexerCustomAction.h" @@ -56,7 +57,7 @@ bool LexerCustomAction::isPositionDependent() const { return true; } -void LexerCustomAction::execute(Lexer::Ref lexer) { +void LexerCustomAction::execute(Lexer *lexer) { lexer->action(nullptr, _ruleIndex, _actionIndex); } diff --git a/runtime/Cpp/runtime/atn/LexerCustomAction.h b/runtime/Cpp/runtime/atn/LexerCustomAction.h index d28680f66..a68f9e9f3 100755 --- a/runtime/Cpp/runtime/atn/LexerCustomAction.h +++ b/runtime/Cpp/runtime/atn/LexerCustomAction.h @@ -33,7 +33,6 @@ #include "LexerAction.h" #include "LexerActionType.h" -#include "Lexer.h" namespace org { namespace antlr { @@ -102,7 +101,7 @@ namespace atn { /// Custom actions are implemented by calling with the /// appropriate rule and action indexes. /// - virtual void execute(Lexer::Ref lexer) override; + virtual void execute(Lexer *lexer) override; virtual size_t hashCode() const override; virtual bool operator == (const LexerAction &obj) const override; diff --git a/runtime/Cpp/runtime/atn/LexerIndexedCustomAction.cpp b/runtime/Cpp/runtime/atn/LexerIndexedCustomAction.cpp new file mode 100755 index 000000000..9100c003c --- /dev/null +++ b/runtime/Cpp/runtime/atn/LexerIndexedCustomAction.cpp @@ -0,0 +1,87 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Dan McLaughlin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "MurmurHash.h" +#include "Lexer.h" +#include "CPPUtils.h" + +#include "LexerIndexedCustomAction.h" + +using namespace org::antlr::v4::runtime::atn; +using namespace org::antlr::v4::runtime::misc; + +LexerIndexedCustomAction::LexerIndexedCustomAction(int offset, Ref action) : _offset(offset), _action(action) { +} + +int LexerIndexedCustomAction::getOffset() const { + return _offset; +} + +Ref LexerIndexedCustomAction::getAction() const { + return _action; +} + +LexerActionType LexerIndexedCustomAction::getActionType() const { + return _action->getActionType(); +} + +bool LexerIndexedCustomAction::isPositionDependent() const { + return true; +} + +void LexerIndexedCustomAction::execute(Lexer *lexer) { + // assume the input stream position was properly set by the calling code + _action->execute(lexer); +} + +size_t LexerIndexedCustomAction::hashCode() const { + size_t hash = MurmurHash::initialize(); + hash = MurmurHash::update(hash, _offset); + hash = MurmurHash::update(hash, (size_t)_action.get()); + return MurmurHash::finish(hash, 2); +} + +bool LexerIndexedCustomAction::operator == (const LexerAction &obj) const { + if (&obj == this) { + return true; + } + + const LexerIndexedCustomAction *action = dynamic_cast(&obj); + if (action == nullptr) { + return false; + } + + return _offset == action->_offset && _action == action->_action; +} + +std::wstring LexerIndexedCustomAction::toString() const { + return antlrcpp::toString(this); +} diff --git a/runtime/Cpp/runtime/atn/LexerIndexedCustomAction.h b/runtime/Cpp/runtime/atn/LexerIndexedCustomAction.h new file mode 100755 index 000000000..4e88017ec --- /dev/null +++ b/runtime/Cpp/runtime/atn/LexerIndexedCustomAction.h @@ -0,0 +1,114 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Dan McLaughlin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "RuleContext.h" +#include "LexerAction.h" + +namespace org { +namespace antlr { +namespace v4 { +namespace runtime { +namespace atn { + + /// + /// This implementation of is used for tracking input offsets + /// for position-dependent actions within a . + /// + /// This action is not serialized as part of the ATN, and is only required for + /// position-dependent lexer actions which appear at a location other than the + /// end of a rule. For more information about DFA optimizations employed for + /// lexer actions, see and + /// . + /// + /// @author Sam Harwell + /// @since 4.2 + /// + class LexerIndexedCustomAction final : public LexerAction { + public: + /// + /// Constructs a new indexed custom action by associating a character offset + /// with a . + /// + /// Note: This class is only required for lexer actions for which + /// returns {@code true}. + /// + /// The offset into the input , relative to + /// the token start index, at which the specified lexer action should be + /// executed. + /// The lexer action to execute at a particular offset in the + /// input . + LexerIndexedCustomAction(int offset, Ref action); + + /// + /// Gets the location in the input at which the lexer + /// action should be executed. The value is interpreted as an offset relative + /// to the token start index. + /// + /// The location in the input at which the lexer + /// action should be executed. + int getOffset() const; + + /// + /// Gets the lexer action to execute. + /// + /// A object which executes the lexer action. + Ref getAction() const; + + /// + /// {@inheritDoc} + /// + /// This method returns the result of calling + /// on the returned by . + virtual LexerActionType getActionType() const override; + + /// + /// {@inheritDoc} + /// This method returns {@code true}. + virtual bool isPositionDependent() const override; + + virtual void execute(Lexer *lexer) override; + virtual size_t hashCode() const override; + virtual bool operator == (const LexerAction &obj) const override; + virtual std::wstring toString() const override; + + private: + const int _offset; + const Ref _action; + }; + +} // namespace atn +} // namespace runtime +} // namespace v4 +} // namespace antlr +} // namespace org + diff --git a/runtime/Cpp/runtime/atn/LexerModeAction.cpp b/runtime/Cpp/runtime/atn/LexerModeAction.cpp index 1ac93eb14..5aa8b7a86 100755 --- a/runtime/Cpp/runtime/atn/LexerModeAction.cpp +++ b/runtime/Cpp/runtime/atn/LexerModeAction.cpp @@ -30,6 +30,7 @@ */ #include "MurmurHash.h" +#include "Lexer.h" #include "LexerModeAction.h" @@ -51,7 +52,7 @@ bool LexerModeAction::isPositionDependent() const { return false; } -void LexerModeAction::execute(Lexer::Ref lexer) { +void LexerModeAction::execute(Lexer *lexer) { lexer->mode(_mode); } diff --git a/runtime/Cpp/runtime/atn/LexerModeAction.h b/runtime/Cpp/runtime/atn/LexerModeAction.h index 929344c52..8b3bced0e 100755 --- a/runtime/Cpp/runtime/atn/LexerModeAction.h +++ b/runtime/Cpp/runtime/atn/LexerModeAction.h @@ -33,7 +33,6 @@ #include "LexerAction.h" #include "LexerActionType.h" -#include "Lexer.h" namespace org { namespace antlr { @@ -77,7 +76,7 @@ namespace atn { /// This action is implemented by calling with the /// value provided by . /// - virtual void execute(Lexer::Ref lexer) override; + virtual void execute(Lexer *lexer) override; virtual size_t hashCode() const override; virtual bool operator == (const LexerAction &obj) const override; diff --git a/runtime/Cpp/runtime/atn/LexerMoreAction.cpp b/runtime/Cpp/runtime/atn/LexerMoreAction.cpp index b3909e009..4daf15d69 100755 --- a/runtime/Cpp/runtime/atn/LexerMoreAction.cpp +++ b/runtime/Cpp/runtime/atn/LexerMoreAction.cpp @@ -30,13 +30,14 @@ */ #include "MurmurHash.h" +#include "Lexer.h" #include "LexerMoreAction.h" using namespace org::antlr::v4::runtime::atn; using namespace org::antlr::v4::runtime::misc; -const std::shared_ptr LexerMoreAction::INSTANCE { new LexerMoreAction() }; +const Ref LexerMoreAction::INSTANCE { new LexerMoreAction() }; LexerMoreAction::LexerMoreAction() { } @@ -49,7 +50,7 @@ bool LexerMoreAction::isPositionDependent() const { return false; } -void LexerMoreAction::execute(Lexer::Ref lexer) { +void LexerMoreAction::execute(Lexer *lexer) { lexer->more(); } diff --git a/runtime/Cpp/runtime/atn/LexerMoreAction.h b/runtime/Cpp/runtime/atn/LexerMoreAction.h index cb52bb699..5f62a14a3 100755 --- a/runtime/Cpp/runtime/atn/LexerMoreAction.h +++ b/runtime/Cpp/runtime/atn/LexerMoreAction.h @@ -33,7 +33,6 @@ #include "LexerAction.h" #include "LexerActionType.h" -#include "Lexer.h" namespace org { namespace antlr { @@ -55,7 +54,7 @@ namespace atn { /// /// Provides a singleton instance of this parameterless lexer action. /// - static const std::shared_ptr INSTANCE; + static const Ref INSTANCE; /// /// {@inheritDoc} @@ -72,7 +71,7 @@ namespace atn { /// /// This action is implemented by calling . /// - virtual void execute(Lexer::Ref lexer) override; + virtual void execute(Lexer *lexer) override; virtual size_t hashCode() const override; virtual bool operator == (const LexerAction &obj) const override; diff --git a/runtime/Cpp/runtime/atn/LexerPopModeAction.cpp b/runtime/Cpp/runtime/atn/LexerPopModeAction.cpp index 1ddbb525a..040c81056 100755 --- a/runtime/Cpp/runtime/atn/LexerPopModeAction.cpp +++ b/runtime/Cpp/runtime/atn/LexerPopModeAction.cpp @@ -30,12 +30,14 @@ */ #include "MurmurHash.h" +#include "Lexer.h" + #include "LexerPopModeAction.h" using namespace org::antlr::v4::runtime::atn; using namespace org::antlr::v4::runtime::misc; -const std::shared_ptr LexerPopModeAction::INSTANCE { new LexerPopModeAction() }; +const Ref LexerPopModeAction::INSTANCE { new LexerPopModeAction() }; LexerPopModeAction::LexerPopModeAction() { } @@ -48,7 +50,7 @@ bool LexerPopModeAction::isPositionDependent() const { return false; } -void LexerPopModeAction::execute(Lexer::Ref lexer) { +void LexerPopModeAction::execute(Lexer *lexer) { lexer->popMode(); } diff --git a/runtime/Cpp/runtime/atn/LexerPopModeAction.h b/runtime/Cpp/runtime/atn/LexerPopModeAction.h index 47b37cab5..3c8e33aee 100755 --- a/runtime/Cpp/runtime/atn/LexerPopModeAction.h +++ b/runtime/Cpp/runtime/atn/LexerPopModeAction.h @@ -33,7 +33,6 @@ #include "LexerAction.h" #include "LexerActionType.h" -#include "Lexer.h" namespace org { namespace antlr { @@ -55,7 +54,7 @@ namespace atn { /// /// Provides a singleton instance of this parameterless lexer action. /// - static const std::shared_ptr INSTANCE; + static const Ref INSTANCE; /// /// {@inheritDoc} @@ -72,7 +71,7 @@ namespace atn { /// /// This action is implemented by calling . /// - virtual void execute(Lexer::Ref lexer) override; + virtual void execute(Lexer *lexer) override; virtual size_t hashCode() const override; virtual bool operator == (const LexerAction &obj) const override; diff --git a/runtime/Cpp/runtime/atn/LexerPushModeAction.cpp b/runtime/Cpp/runtime/atn/LexerPushModeAction.cpp index df6132ed4..f2753c132 100755 --- a/runtime/Cpp/runtime/atn/LexerPushModeAction.cpp +++ b/runtime/Cpp/runtime/atn/LexerPushModeAction.cpp @@ -30,6 +30,7 @@ */ #include "MurmurHash.h" +#include "Lexer.h" #include "LexerPushModeAction.h" @@ -51,7 +52,7 @@ bool LexerPushModeAction::isPositionDependent() const { return false; } -void LexerPushModeAction::execute(Lexer::Ref lexer) { +void LexerPushModeAction::execute(Lexer *lexer) { lexer->pushMode(_mode); } diff --git a/runtime/Cpp/runtime/atn/LexerPushModeAction.h b/runtime/Cpp/runtime/atn/LexerPushModeAction.h index 11ba9ea60..54fbe463e 100755 --- a/runtime/Cpp/runtime/atn/LexerPushModeAction.h +++ b/runtime/Cpp/runtime/atn/LexerPushModeAction.h @@ -33,7 +33,6 @@ #include "LexerAction.h" #include "LexerActionType.h" -#include "Lexer.h" namespace org { namespace antlr { @@ -77,7 +76,7 @@ namespace atn { /// This action is implemented by calling with the /// value provided by . /// - virtual void execute(Lexer::Ref lexer) override; + virtual void execute(Lexer *lexer) override; virtual size_t hashCode() const override; virtual bool operator == (const LexerAction &obj) const override; diff --git a/runtime/Cpp/runtime/atn/LexerSkipAction.cpp b/runtime/Cpp/runtime/atn/LexerSkipAction.cpp index 9a8b6db0a..bf724c408 100755 --- a/runtime/Cpp/runtime/atn/LexerSkipAction.cpp +++ b/runtime/Cpp/runtime/atn/LexerSkipAction.cpp @@ -30,13 +30,14 @@ */ #include "MurmurHash.h" +#include "Lexer.h" #include "LexerSkipAction.h" using namespace org::antlr::v4::runtime::atn; using namespace org::antlr::v4::runtime::misc; -const std::shared_ptr LexerSkipAction::INSTANCE { new LexerSkipAction() }; +const Ref LexerSkipAction::INSTANCE { new LexerSkipAction() }; LexerSkipAction::LexerSkipAction() { } @@ -49,7 +50,7 @@ bool LexerSkipAction::isPositionDependent() const { return false; } -void LexerSkipAction::execute(Lexer::Ref lexer) { +void LexerSkipAction::execute(Lexer *lexer) { lexer->skip(); } diff --git a/runtime/Cpp/runtime/atn/LexerSkipAction.h b/runtime/Cpp/runtime/atn/LexerSkipAction.h index 5b4faea4e..c868a28ef 100755 --- a/runtime/Cpp/runtime/atn/LexerSkipAction.h +++ b/runtime/Cpp/runtime/atn/LexerSkipAction.h @@ -33,7 +33,6 @@ #include "LexerAction.h" #include "LexerActionType.h" -#include "Lexer.h" namespace org { namespace antlr { @@ -53,7 +52,7 @@ namespace atn { class LexerSkipAction final : public LexerAction { public: /// Provides a singleton instance of this parameterless lexer action. - static const std::shared_ptr INSTANCE; + static const Ref INSTANCE; /// /// {@inheritDoc} @@ -70,7 +69,7 @@ namespace atn { /// /// This action is implemented by calling . /// - virtual void execute(Lexer::Ref lexer) override; + virtual void execute(Lexer *lexer) override; virtual size_t hashCode() const override; virtual bool operator == (const LexerAction &obj) const override; diff --git a/runtime/Cpp/runtime/atn/LexerTypeAction.cpp b/runtime/Cpp/runtime/atn/LexerTypeAction.cpp index 0b0f642c9..e29b16f9d 100755 --- a/runtime/Cpp/runtime/atn/LexerTypeAction.cpp +++ b/runtime/Cpp/runtime/atn/LexerTypeAction.cpp @@ -30,6 +30,7 @@ */ #include "MurmurHash.h" +#include "Lexer.h" #include "LexerTypeAction.h" @@ -51,7 +52,7 @@ bool LexerTypeAction::isPositionDependent() const { return false; } -void LexerTypeAction::execute(Lexer::Ref lexer) { +void LexerTypeAction::execute(Lexer *lexer) { lexer->setType(_type); } diff --git a/runtime/Cpp/runtime/atn/LexerTypeAction.h b/runtime/Cpp/runtime/atn/LexerTypeAction.h index 919c7554b..1faf48ca5 100755 --- a/runtime/Cpp/runtime/atn/LexerTypeAction.h +++ b/runtime/Cpp/runtime/atn/LexerTypeAction.h @@ -33,7 +33,6 @@ #include "LexerActionType.h" #include "LexerAction.h" -#include "Lexer.h" namespace org { namespace antlr { @@ -71,7 +70,7 @@ namespace atn { /// This action is implemented by calling with the /// value provided by . /// - virtual void execute(Lexer::Ref lexer) override; + virtual void execute(Lexer *lexer) override; virtual size_t hashCode() const override; virtual bool operator == (const LexerAction &obj) const override; diff --git a/runtime/Cpp/runtime/atn/LookaheadEventInfo.cpp b/runtime/Cpp/runtime/atn/LookaheadEventInfo.cpp new file mode 100755 index 000000000..8095854df --- /dev/null +++ b/runtime/Cpp/runtime/atn/LookaheadEventInfo.cpp @@ -0,0 +1,42 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "LookaheadEventInfo.h" + +using namespace org::antlr::v4::runtime; +using namespace org::antlr::v4::runtime::atn; + +LookaheadEventInfo::LookaheadEventInfo(int decision, Ref configs, int predictedAlt, TokenStream *input, + int startIndex, int stopIndex, bool fullCtx) + : DecisionEventInfo(decision, configs, input, startIndex, stopIndex, fullCtx) { + + this->predictedAlt = predictedAlt; +} diff --git a/runtime/Cpp/runtime/atn/LookaheadEventInfo.h b/runtime/Cpp/runtime/atn/LookaheadEventInfo.h new file mode 100755 index 000000000..af8da2ff5 --- /dev/null +++ b/runtime/Cpp/runtime/atn/LookaheadEventInfo.h @@ -0,0 +1,74 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "DecisionEventInfo.h" + +namespace org { +namespace antlr { +namespace v4 { +namespace runtime { +namespace atn { + + /// This class represents profiling event information for tracking the lookahead + /// depth required in order to make a prediction. + class LookaheadEventInfo : public DecisionEventInfo { + public: + /// The alternative chosen by adaptivePredict(), not necessarily + /// the outermost alt shown for a rule; left-recursive rules have + /// user-level alts that differ from the rewritten rule with a (...) block + /// and a (..)* loop. + int predictedAlt = 0; + + /// + /// Constructs a new instance of the class with + /// the specified detailed lookahead information. + /// + /// The decision number + /// The final configuration set containing the necessary + /// information to determine the result of a prediction, or {@code null} if + /// the final configuration set is not available + /// The input token stream + /// The start index for the current prediction + /// The index at which the prediction was finally made + /// {@code true} if the current lookahead is part of an LL + /// prediction; otherwise, {@code false} if the current lookahead is part of + /// an SLL prediction + LookaheadEventInfo(int decision, Ref configs, int predictedAlt, TokenStream *input, int startIndex, + int stopIndex, bool fullCtx); + }; + +} // namespace atn +} // namespace runtime +} // namespace v4 +} // namespace antlr +} // namespace org diff --git a/runtime/Cpp/runtime/atn/OrderedATNConfigSet.cpp b/runtime/Cpp/runtime/atn/OrderedATNConfigSet.cpp index a2b11f890..faa55afe6 100755 --- a/runtime/Cpp/runtime/atn/OrderedATNConfigSet.cpp +++ b/runtime/Cpp/runtime/atn/OrderedATNConfigSet.cpp @@ -34,6 +34,6 @@ using namespace org::antlr::v4::runtime::atn; OrderedATNConfigSet::OrderedATNConfigSet() : ATNConfigSet() { - configLookup = std::shared_ptr( + configLookup = Ref( new ConfigLookupImpl()); /* mem-check: managed by shared_ptr in ConfigLookupImpl */ } diff --git a/runtime/Cpp/runtime/atn/OrderedATNConfigSet.h b/runtime/Cpp/runtime/atn/OrderedATNConfigSet.h index 4b4aa7e70..ada5a0f6e 100755 --- a/runtime/Cpp/runtime/atn/OrderedATNConfigSet.h +++ b/runtime/Cpp/runtime/atn/OrderedATNConfigSet.h @@ -41,13 +41,13 @@ namespace atn { struct OrderedATNConfigHasher { - size_t operator()(const ATNConfig::Ref &config) const { - return std::hash()(config); + size_t operator()(const Ref &config) const { + return std::hash>()(config); } }; struct OrderedATNConfigComparer { - bool operator()(const ATNConfig::Ref &lhs, const ATNConfig::Ref &rhs) const + bool operator()(const Ref &lhs, const Ref &rhs) const { return lhs == rhs; } diff --git a/runtime/Cpp/runtime/atn/ParseInfo.cpp b/runtime/Cpp/runtime/atn/ParseInfo.cpp new file mode 100755 index 000000000..c111389a2 --- /dev/null +++ b/runtime/Cpp/runtime/atn/ParseInfo.cpp @@ -0,0 +1,125 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "ProfilingATNSimulator.h" +#include "DFA.h" + +#include "ParseInfo.h" + +using namespace org::antlr::v4::runtime::atn; + +ParseInfo::ParseInfo(ProfilingATNSimulator *atnSimulator) : _atnSimulator(atnSimulator) { +} + +std::vector ParseInfo::getDecisionInfo() { + return _atnSimulator->getDecisionInfo(); +} + +std::vector ParseInfo::getLLDecisions() { + std::vector decisions = _atnSimulator->getDecisionInfo(); + std::vector LL; + for (size_t i = 0; i < decisions.size(); ++i) { + long long fallBack = decisions[i].LL_Fallback; + if (fallBack > 0) { + LL.push_back(i); + } + } + return LL; +} + +long long ParseInfo::getTotalTimeInPrediction() { + std::vector decisions = _atnSimulator->getDecisionInfo(); + long long t = 0; + for (size_t i = 0; i < decisions.size(); ++i) { + t += decisions[i].timeInPrediction; + } + return t; +} + +long long ParseInfo::getTotalSLLLookaheadOps() { + std::vector decisions = _atnSimulator->getDecisionInfo(); + long long k = 0; + for (size_t i = 0; i < decisions.size(); ++i) { + k += decisions[i].SLL_TotalLook; + } + return k; +} + +long long ParseInfo::getTotalLLLookaheadOps() { + std::vector decisions = _atnSimulator->getDecisionInfo(); + long long k = 0; + for (size_t i = 0; i < decisions.size(); i++) { + k += decisions[i].LL_TotalLook; + } + return k; +} + +long long ParseInfo::getTotalSLLATNLookaheadOps() { + std::vector decisions = _atnSimulator->getDecisionInfo(); + long long k = 0; + for (size_t i = 0; i < decisions.size(); ++i) { + k += decisions[i].SLL_ATNTransitions; + } + return k; +} + +long long ParseInfo::getTotalLLATNLookaheadOps() { + std::vector decisions = _atnSimulator->getDecisionInfo(); + long long k = 0; + for (size_t i = 0; i < decisions.size(); ++i) { + k += decisions[i].LL_ATNTransitions; + } + return k; +} + +long long ParseInfo::getTotalATNLookaheadOps() { + std::vector decisions = _atnSimulator->getDecisionInfo(); + long long k = 0; + for (size_t i = 0; i < decisions.size(); ++i) { + k += decisions[i].SLL_ATNTransitions; + k += decisions[i].LL_ATNTransitions; + } + return k; +} + +int ParseInfo::getDFASize() { + int n = 0; + std::vector decisionToDFA = _atnSimulator->decisionToDFA; + for (size_t i = 0; i < decisionToDFA.size(); ++i) { + n += getDFASize((int)i); + } + return n; +} + +size_t ParseInfo::getDFASize(int decision) { + dfa::DFA &decisionToDFA = _atnSimulator->decisionToDFA[decision]; + return decisionToDFA.states.size(); +} diff --git a/runtime/Cpp/runtime/atn/ParseInfo.h b/runtime/Cpp/runtime/atn/ParseInfo.h new file mode 100755 index 000000000..8df3c4058 --- /dev/null +++ b/runtime/Cpp/runtime/atn/ParseInfo.h @@ -0,0 +1,130 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "DecisionInfo.h" + +namespace org { +namespace antlr { +namespace v4 { +namespace runtime { +namespace atn { + + class ProfilingATNSimulator; + + /// This class provides access to specific and aggregate statistics gathered + /// during profiling of a parser. + class ParseInfo { + public: + ParseInfo(ProfilingATNSimulator *atnSimulator); + + /// + /// Gets an array of instances containing the profiling + /// information gathered for each decision in the ATN. + /// + /// An array of instances, indexed by decision + /// number. + virtual std::vector getDecisionInfo(); + + /// + /// Gets the decision numbers for decisions that required one or more + /// full-context predictions during parsing. These are decisions for which + /// is non-zero. + /// + /// A list of decision numbers which required one or more + /// full-context predictions during parsing. + virtual std::vector getLLDecisions(); + + /// + /// Gets the total time spent during prediction across all decisions made + /// during parsing. This value is the sum of + /// for all decisions. + /// + virtual long long getTotalTimeInPrediction(); + + /// + /// Gets the total number of SLL lookahead operations across all decisions + /// made during parsing. This value is the sum of + /// for all decisions. + /// + virtual long long getTotalSLLLookaheadOps(); + + /// + /// Gets the total number of LL lookahead operations across all decisions + /// made during parsing. This value is the sum of + /// for all decisions. + /// + virtual long long getTotalLLLookaheadOps(); + + /// + /// Gets the total number of ATN lookahead operations for SLL prediction + /// across all decisions made during parsing. + /// + virtual long long getTotalSLLATNLookaheadOps(); + + /// + /// Gets the total number of ATN lookahead operations for LL prediction + /// across all decisions made during parsing. + /// + virtual long long getTotalLLATNLookaheadOps(); + + /// + /// Gets the total number of ATN lookahead operations for SLL and LL + /// prediction across all decisions made during parsing. + /// + /// + /// This value is the sum of and + /// . + /// + virtual long long getTotalATNLookaheadOps(); + + /// + /// Gets the total number of DFA states stored in the DFA cache for all + /// decisions in the ATN. + /// + virtual int getDFASize(); + + /// + /// Gets the total number of DFA states stored in the DFA cache for a + /// particular decision. + /// + virtual size_t getDFASize(int decision); + + protected: + const ProfilingATNSimulator *_atnSimulator; // non-owning, we are created by this simulator. + }; + +} // namespace atn +} // namespace runtime +} // namespace v4 +} // namespace antlr +} // namespace org diff --git a/runtime/Cpp/runtime/atn/ParserATNSimulator.cpp b/runtime/Cpp/runtime/atn/ParserATNSimulator.cpp index b993937fd..0c7cbb02f 100755 --- a/runtime/Cpp/runtime/atn/ParserATNSimulator.cpp +++ b/runtime/Cpp/runtime/atn/ParserATNSimulator.cpp @@ -46,12 +46,16 @@ #include "PredicateTransition.h" #include "PrecedencePredicateTransition.h" #include "ActionTransition.h" +#include "EpsilonTransition.h" #include "RuleStopState.h" #include "ATNConfigSet.h" #include "ATNConfig.h" #include "Interval.h" #include "ANTLRErrorListener.h" +#include "Vocabulary.h" +#include "VocabularyImpl.h" + #include "Arrays.h" #include "ParserATNSimulator.h" @@ -62,20 +66,28 @@ using namespace org::antlr::v4::runtime::atn; using namespace antlrcpp; ParserATNSimulator::ParserATNSimulator(const ATN &atn, std::vector &decisionToDFA, - std::shared_ptr sharedContextCache) + Ref sharedContextCache) : ParserATNSimulator(nullptr, atn, decisionToDFA, sharedContextCache) { } ParserATNSimulator::ParserATNSimulator(Parser *parser, const ATN &atn, std::vector &decisionToDFA, - std::shared_ptr sharedContextCache) -: ATNSimulator(atn, sharedContextCache), parser(parser), _decisionToDFA(decisionToDFA) { + Ref sharedContextCache) +: ATNSimulator(atn, sharedContextCache), parser(parser), decisionToDFA(decisionToDFA) { InitializeInstanceFields(); } void ParserATNSimulator::reset() { } -int ParserATNSimulator::adaptivePredict(TokenStream *input, int decision, ParserRuleContext::Ref outerContext) { +void ParserATNSimulator::clearDFA() { + int size = (int)decisionToDFA.size(); + decisionToDFA.clear(); + for (int d = 0; d < size; ++d) { + decisionToDFA.push_back(dfa::DFA(atn.getDecisionState(d), d)); + } +} + +int ParserATNSimulator::adaptivePredict(TokenStream *input, int decision, Ref outerContext) { if (debug || debug_list_atn_decisions) { std::wcout << L"adaptivePredict decision " << decision << L" exec LA(1)==" << getLookaheadName(input) << L" line " << input->LT(1)->getLine() << L":" << input->LT(1)->getCharPositionInLine() << std::endl; } @@ -83,20 +95,33 @@ int ParserATNSimulator::adaptivePredict(TokenStream *input, int decision, Parser _input = input; _startIndex = (int)input->index(); _outerContext = outerContext; - dfa::DFA &dfa = _decisionToDFA[(size_t)decision]; + dfa::DFA &dfa = decisionToDFA[(size_t)decision]; + _dfa = &dfa; ssize_t m = input->mark(); - size_t index = input->index(); + size_t index = _startIndex; // Now we are certain to have a specific decision's DFA // But, do we still need an initial state? auto onExit = finally([&] { mergeCache.clear(); // wack cache after each prediction + _dfa = nullptr; input->seek(index); input->release(m); }); - if (dfa.s0 == nullptr) { + dfa::DFAState *s0; + if (dfa.isPrecedenceDfa()) { + // the start state for a precedence DFA depends on the current + // parser precedence, and is provided by a DFA method. + s0 = dfa.getPrecedenceStartState(parser->getPrecedence()); + } + else { + // the start state for a "regular" DFA is just s0 + s0 = dfa.s0; + } + + if (s0 == nullptr) { if (outerContext == nullptr) { outerContext = std::dynamic_pointer_cast(RuleContext::EMPTY); } @@ -104,25 +129,44 @@ int ParserATNSimulator::adaptivePredict(TokenStream *input, int decision, Parser std::wcout << L"predictATN decision " << dfa.decision << L" exec LA(1)==" << getLookaheadName(input) << L", outerContext=" << outerContext->toString(parser) << std::endl; } bool fullCtx = false; - std::shared_ptr s0_closure = computeStartState(dynamic_cast(dfa.atnStartState), + Ref s0_closure = computeStartState(dynamic_cast(dfa.atnStartState), ParserRuleContext::EMPTY, fullCtx); - dfa::DFAState *newState = new dfa::DFAState(s0_closure); /* mem-check: managed by the DFA or deleted below */ - dfa.s0 = addDFAState(dfa, newState); - if (dfa.s0 != newState) { - delete newState; // If there was already a state with this config set we don't need the new one. + if (dfa.isPrecedenceDfa()) { + /* If this is a precedence DFA, we use applyPrecedenceFilter + * to convert the computed start state to a precedence start + * state. We then use DFA.setPrecedenceStartState to set the + * appropriate start state for the precedence level rather + * than simply setting DFA.s0. + */ + dfa.s0->configs = s0_closure; // not used for prediction but useful to know start configs anyway + s0_closure = applyPrecedenceFilter(s0_closure); + + dfa::DFAState *newState = new dfa::DFAState(s0_closure); /* mem-check: managed by the DFA or deleted below */ + s0 = addDFAState(dfa, newState); + dfa.setPrecedenceStartState(parser->getPrecedence(), s0); + if (s0 != newState) { + delete newState; // If there was already a state with this config set we don't need the new one. + } + } else { + dfa::DFAState *newState = new dfa::DFAState(s0_closure); /* mem-check: managed by the DFA or deleted below */ + s0 = addDFAState(dfa, newState); + dfa.s0 = s0; + if (s0 != newState) { + delete newState; // If there was already a state with this config set we don't need the new one. + } } } // We can start with an existing DFA. int alt = execATN(dfa, dfa.s0, input, index, outerContext); if (debug) { - std::wcout << "DFA after predictATN: " << dfa.toString(parser->getTokenNames()) << std::endl; + std::wcout << "DFA after predictATN: " << dfa.toString(parser->getVocabulary()) << std::endl; } return alt; } int ParserATNSimulator::execATN(dfa::DFA &dfa, dfa::DFAState *s0, TokenStream *input, size_t startIndex, - ParserRuleContext::Ref outerContext) { + Ref outerContext) { if (debug || debug_list_atn_decisions) { std::wcout << L"execATN decision " << dfa.decision << L" exec LA(1)==" << getLookaheadName(input) << L" line " << input->LT(1)->getLine() << L":" << input->LT(1)->getCharPositionInLine() << std::endl; @@ -192,7 +236,7 @@ int ParserATNSimulator::execATN(dfa::DFA &dfa, dfa::DFAState *s0, TokenStream *i std::wcout << L"ctx sensitive state " << outerContext << L" in " << D << std::endl; } bool fullCtx = true; - std::shared_ptr s0_closure = computeStartState(dfa.atnStartState, outerContext, fullCtx); + Ref s0_closure = computeStartState(dfa.atnStartState, outerContext, fullCtx); reportAttemptingFullContext(dfa, conflictingAlts, D->configs, startIndex, input->index()); int alt = execATNWithFullContext(dfa, D, s0_closure, input, startIndex, outerContext); return alt; @@ -240,7 +284,7 @@ dfa::DFAState *ParserATNSimulator::getExistingTargetState(dfa::DFAState *previou } dfa::DFAState *ParserATNSimulator::computeTargetState(dfa::DFA &dfa, dfa::DFAState *previousD, ssize_t t) { - std::shared_ptr reach = computeReachSet(previousD->configs, t, false); + Ref reach = computeReachSet(previousD->configs, t, false); if (reach == nullptr) { addDFAEdge(dfa, previousD, t, ERROR.get()); return ERROR.get(); @@ -299,7 +343,7 @@ void ParserATNSimulator::predicateDFAState(dfa::DFAState *dfaState, DecisionStat // Update DFA so reach becomes accept state with (predicate,alt) // pairs if preds found for conflicting alts BitSet altsToCollectPredsFrom = getConflictingAltsOrUniqueAlt(dfaState->configs); - std::vector altToPred = getPredsForAmbigAlts(altsToCollectPredsFrom, dfaState->configs, nalts); + std::vector> altToPred = getPredsForAmbigAlts(altsToCollectPredsFrom, dfaState->configs, nalts); if (!altToPred.empty()) { dfaState->predicates = getPredicatePredictions(altsToCollectPredsFrom, altToPred); dfaState->prediction = ATN::INVALID_ALT_NUMBER; // make sure we use preds @@ -311,15 +355,15 @@ void ParserATNSimulator::predicateDFAState(dfa::DFAState *dfaState, DecisionStat } } -int ParserATNSimulator::execATNWithFullContext(dfa::DFA &dfa, dfa::DFAState *D, std::shared_ptr s0, - TokenStream *input, size_t startIndex, ParserRuleContext::Ref outerContext) { +int ParserATNSimulator::execATNWithFullContext(dfa::DFA &dfa, dfa::DFAState *D, Ref s0, + TokenStream *input, size_t startIndex, Ref outerContext) { if (debug || debug_list_atn_decisions) { std::cout << "execATNWithFullContext " << s0 << std::endl; } bool fullCtx = true; bool foundExactAmbig = false; - std::shared_ptr reach; - std::shared_ptr previous = s0; + Ref reach; + Ref previous = s0; input->seek(startIndex); ssize_t t = input->LA(1); int predictedAlt; @@ -335,11 +379,13 @@ int ParserATNSimulator::execATNWithFullContext(dfa::DFA &dfa, dfa::DFAState *D, // ATN states in SLL implies LL will also get nowhere. // If conflict in states that dip out, choose min since we // will get error no matter what. - int alt = getAltThatFinishedDecisionEntryRule(previous); + NoViableAltException e = noViableAlt(input, outerContext, previous, startIndex); + input->seek(startIndex); + int alt = getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule(previous, outerContext); if (alt != ATN::INVALID_ALT_NUMBER) { return alt; } - throw noViableAlt(input, outerContext, previous, startIndex); + throw e; } std::vector altSubSets =PredictionModeClass::getConflictingAltSubsets(reach); @@ -352,7 +398,6 @@ int ParserATNSimulator::execATNWithFullContext(dfa::DFA &dfa, dfa::DFAState *D, << std::endl; } - // System.out.println("altSubSets: "+altSubSets); reach->uniqueAlt = getUniqueAlt(reach); // unique prediction? if (reach->uniqueAlt != ATN::INVALID_ALT_NUMBER) { @@ -419,18 +464,17 @@ int ParserATNSimulator::execATNWithFullContext(dfa::DFA &dfa, dfa::DFAState *D, the fact that we should predict alternative 1. We just can't say for sure that there is an ambiguity without looking further. */ - reportAmbiguity(dfa, D, (size_t)startIndex, input->index(), foundExactAmbig, BitSet(), reach); + reportAmbiguity(dfa, D, (size_t)startIndex, input->index(), foundExactAmbig, reach->getAlts(), reach); return predictedAlt; } -std::shared_ptr ParserATNSimulator::computeReachSet(std::shared_ptr closure, - ssize_t t, bool fullCtx) { +Ref ParserATNSimulator::computeReachSet(Ref closure_, ssize_t t, bool fullCtx) { if (debug) { - std::wcout << L"in computeReachSet, starting closure: " << closure << std::endl; + std::wcout << L"in computeReachSet, starting closure: " << closure_ << std::endl; } - std::shared_ptr intermediate = std::make_shared(fullCtx); + Ref intermediate = std::make_shared(fullCtx); /* Configurations already in a rule stop state indicate reaching the end * of the decision rule (local context) or end of the start rule (full @@ -442,15 +486,15 @@ std::shared_ptr ParserATNSimulator::computeReachSet(std::shared_pt * ensure that the alternative matching the longest overall sequence is * chosen when multiple such configurations can match the input. */ - std::vector skippedStopStates; + std::vector> skippedStopStates; // First figure out where we can reach on input t - for (auto c : closure->configs) { + for (auto c : closure_->configs) { if (debug) { std::wcout << L"testing " << getTokenName(t) << L" at " << c->toString() << std::endl; } - if (is(c->state)) { + if (is(c->state)) { assert(c->context->isEmpty()); if (fullCtx || t == EOF) { @@ -471,7 +515,7 @@ std::shared_ptr ParserATNSimulator::computeReachSet(std::shared_pt } // Now figure out where the reach operation can take us... - std::shared_ptr reach; + Ref reach; /* This block optimizes the reach operation for intermediate sets which * trivially indicate a termination state for the overall @@ -480,9 +524,9 @@ std::shared_ptr ParserATNSimulator::computeReachSet(std::shared_pt * The conditions assume that intermediate * contains all configurations relevant to the reach set, but this * condition is not true when one or more configurations have been - * withheld in skippedStopStates. + * withheld in skippedStopStates, or when the current symbol is EOF. */ - if (skippedStopStates.empty()) { + if (skippedStopStates.empty() && t != EOF) { if (intermediate->size() == 1) { // Don't pursue the closure if there is just one state. // It can only have one alternative; just add to result @@ -501,10 +545,11 @@ std::shared_ptr ParserATNSimulator::computeReachSet(std::shared_pt */ if (reach == nullptr) { reach = std::make_shared(fullCtx); - std::set closureBusy; + std::set> closureBusy; + bool treatEofAsEpsilon = t == EOF; for (auto c : intermediate->configs) { - this->closure(c, reach, closureBusy, false, fullCtx); + closure(c, reach, closureBusy, false, fullCtx, treatEofAsEpsilon); } } @@ -551,13 +596,13 @@ std::shared_ptr ParserATNSimulator::computeReachSet(std::shared_pt return reach; } -std::shared_ptr ParserATNSimulator::removeAllConfigsNotInRuleStopState(std::shared_ptr configs, +Ref ParserATNSimulator::removeAllConfigsNotInRuleStopState(Ref configs, bool lookToEndOfRule) { if (PredictionModeClass::allConfigsInRuleStopStates(configs)) { return configs; } - std::shared_ptr result = std::make_shared(configs->fullCtx); + Ref result = std::make_shared(configs->fullCtx); for (auto config : configs->configs) { if (is(config->state)) { @@ -577,22 +622,70 @@ std::shared_ptr ParserATNSimulator::removeAllConfigsNotInRuleStopS return result; } -std::shared_ptr ParserATNSimulator::computeStartState(ATNState *p, RuleContext::Ref ctx, bool fullCtx) { +Ref ParserATNSimulator::computeStartState(ATNState *p, Ref ctx, bool fullCtx) { // always at least the implicit call to start rule - PredictionContext::Ref initialContext = PredictionContext::fromRuleContext(atn, ctx); - std::shared_ptr configs = std::make_shared(fullCtx); + Ref initialContext = PredictionContext::fromRuleContext(atn, ctx); + Ref configs = std::make_shared(fullCtx); for (size_t i = 0; i < p->getNumberOfTransitions(); i++) { ATNState *target = p->transition(i)->target; - ATNConfig::Ref c = std::make_shared(target, (int)i + 1, initialContext); - std::set closureBusy; - closure(c, configs, closureBusy, true, fullCtx); + Ref c = std::make_shared(target, (int)i + 1, initialContext); + std::set> closureBusy; + closure(c, configs, closureBusy, true, fullCtx, false); } return configs; } -atn::ATNState *ParserATNSimulator::getReachableTarget(Transition *trans, int ttype) { +Ref ParserATNSimulator::applyPrecedenceFilter(Ref configs) { + std::map> statesFromAlt1; + Ref configSet = std::make_shared(configs->fullCtx); + for (Ref config : configs->configs) { + // handle alt 1 first + if (config->alt != 1) { + continue; + } + + Ref updatedContext = config->semanticContext->evalPrecedence(parser, _outerContext); + if (updatedContext == nullptr) { + // the configuration was eliminated + continue; + } + + statesFromAlt1[config->state->stateNumber] = config->context; + if (updatedContext != config->semanticContext) { + configSet->add(std::make_shared(config, updatedContext), &mergeCache); + } + else { + configSet->add(config, &mergeCache); + } + } + + for (Ref config : configs->configs) { + if (config->alt == 1) { + // already handled + continue; + } + + if (!config->isPrecedenceFilterSuppressed()) { + /* In the future, this elimination step could be updated to also + * filter the prediction context for alternatives predicting alt>1 + * (basically a graph subtraction algorithm). + */ + auto iterator = statesFromAlt1.find(config->state->stateNumber); + if (iterator != statesFromAlt1.end() && iterator->second == config->context) { + // eliminated + continue; + } + } + + configSet->add(config, &mergeCache); + } + + return configSet; +} + +atn::ATNState* ParserATNSimulator::getReachableTarget(Transition *trans, int ttype) { if (trans->matches(ttype, 0, (int)atn.maxTokenType)) { return trans->target; } @@ -601,8 +694,8 @@ atn::ATNState *ParserATNSimulator::getReachableTarget(Transition *trans, int tty } // Note that caller must memory manage the returned value from this function -std::vector ParserATNSimulator::getPredsForAmbigAlts(const BitSet &ambigAlts, - std::shared_ptr configs, size_t nalts) { +std::vector> ParserATNSimulator::getPredsForAmbigAlts(const BitSet &ambigAlts, + Ref configs, size_t nalts) { // REACH=[1|1|[]|0:0, 1|2|[]|0:1] /* altToPred starts as an array of all null contexts. The entry at index i * corresponds to alternative i. altToPred[i] may have one of three values: @@ -615,7 +708,7 @@ std::vector ParserATNSimulator::getPredsForAmbigAlts(const * * From this, it is clear that NONE||anything==NONE. */ - std::vector altToPred; + std::vector> altToPred; for (auto c : configs->configs) { if (ambigAlts.test((size_t)c->alt)) { @@ -645,11 +738,11 @@ std::vector ParserATNSimulator::getPredsForAmbigAlts(const } std::vector ParserATNSimulator::getPredicatePredictions(const antlrcpp::BitSet &ambigAlts, - std::vector altToPred) { + std::vector> altToPred) { std::vector pairs; bool containsPredicate = false; for (size_t i = 1; i < altToPred.size(); i++) { - SemanticContext::Ref pred = altToPred[i]; + Ref pred = altToPred[i]; // unpredicted is indicated by SemanticContext.NONE assert(pred != nullptr); @@ -669,10 +762,30 @@ std::vector ParserATNSimulator::getPredicatePre return pairs; } -int ParserATNSimulator::getAltThatFinishedDecisionEntryRule(std::shared_ptr configs) { +int ParserATNSimulator::getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule(Ref configs, + Ref outerContext) +{ + std::pair, Ref> sets = splitAccordingToSemanticValidity(configs, outerContext); + Ref semValidConfigs = sets.first; + Ref semInvalidConfigs = sets.second; + int alt = getAltThatFinishedDecisionEntryRule(semValidConfigs); + if (alt != ATN::INVALID_ALT_NUMBER) { // semantically/syntactically viable path exists + return alt; + } + // Is there a syntactically valid path with a failed pred? + if (!semInvalidConfigs->configs.empty()) { + alt = getAltThatFinishedDecisionEntryRule(semInvalidConfigs); + if (alt != ATN::INVALID_ALT_NUMBER) { // syntactically viable path exists + return alt; + } + } + return ATN::INVALID_ALT_NUMBER; +} + +int ParserATNSimulator::getAltThatFinishedDecisionEntryRule(Ref configs) { misc::IntervalSet alts; for (auto c : configs->configs) { - if (c->reachesIntoOuterContext > 0 || (is(c->state) && c->context->hasEmptyPath())) { + if (c->getOuterContextDepth() > 0 || (is(c->state) && c->context->hasEmptyPath())) { alts.add(c->alt); } } @@ -682,8 +795,28 @@ int ParserATNSimulator::getAltThatFinishedDecisionEntryRule(std::shared_ptr, Ref> ParserATNSimulator::splitAccordingToSemanticValidity(Ref configs, + Ref outerContext) { + + Ref succeeded = std::make_shared(configs->fullCtx); + Ref failed = std::make_shared(configs->fullCtx); + for (Ref c : configs->configs) { + if (c->semanticContext != SemanticContext::NONE) { + bool predicateEvaluationResult = evalSemanticContext(c->semanticContext, outerContext, c->alt, configs->fullCtx); + if (predicateEvaluationResult) { + succeeded->add(c); + } else { + failed->add(c); + } + } else { + succeeded->add(c); + } + } + return { succeeded, failed }; +} + BitSet ParserATNSimulator::evalSemanticContext(std::vector predPredictions, - ParserRuleContext::Ref outerContext, bool complete) { + Ref outerContext, bool complete) { BitSet predictions; for (auto prediction : predPredictions) { if (prediction->pred == SemanticContext::NONE) { @@ -694,7 +827,8 @@ BitSet ParserATNSimulator::evalSemanticContext(std::vectorpred->eval(parser, outerContext); + bool fullCtx = false; // in dfa + bool predicateEvaluationResult = evalSemanticContext(prediction->pred, outerContext, prediction->alt, fullCtx); if (debug || dfa_debug) { std::wcout << L"eval pred " << prediction->toString() << L" = " << predicateEvaluationResult << std::endl; } @@ -713,21 +847,26 @@ BitSet ParserATNSimulator::evalSemanticContext(std::vector configs, - std::set &closureBusy, bool collectPredicates, bool fullCtx) { +bool ParserATNSimulator::evalSemanticContext(Ref pred, Ref parserCallStack, int alt, + bool fullCtx) { + return pred->eval(parser, parserCallStack); +} + +void ParserATNSimulator::closure(Ref config, Ref configs, std::set> &closureBusy, + bool collectPredicates, bool fullCtx, bool treatEofAsEpsilon) { const int initialDepth = 0; - closureCheckingStopState(config, configs, closureBusy, collectPredicates, fullCtx, initialDepth); + closureCheckingStopState(config, configs, closureBusy, collectPredicates, fullCtx, initialDepth, treatEofAsEpsilon); assert(!fullCtx || !configs->dipsIntoOuterContext); } -void ParserATNSimulator::closureCheckingStopState(ATNConfig::Ref config, std::shared_ptr configs, - std::set &closureBusy, bool collectPredicates, bool fullCtx, int depth) { +void ParserATNSimulator::closureCheckingStopState(Ref config, Ref configs, + std::set> &closureBusy, bool collectPredicates, bool fullCtx, int depth, bool treatEofAsEpsilon) { if (debug) { std::wcout << L"closure(" << config->toString(true) << L")" << std::endl; } - if (is(config->state)) { + if (is(config->state)) { // We hit rule end. If we have context info, use it // run thru all possible stack tops in ctx if (!config->context->isEmpty()) { @@ -741,20 +880,24 @@ void ParserATNSimulator::closureCheckingStopState(ATNConfig::Ref config, std::sh if (debug) { std::wcout << L"FALLING off rule " << getRuleName((size_t)config->state->ruleIndex) << std::endl; } - closure_(config, configs, closureBusy, collectPredicates, fullCtx, depth); + closure_(config, configs, closureBusy, collectPredicates, fullCtx, depth, treatEofAsEpsilon); } continue; } ATNState *returnState = atn.states[(size_t)config->context->getReturnState(i)]; std::weak_ptr newContext = config->context->getParent(i); // "pop" return state - ATNConfig::Ref c = std::make_shared(returnState, config->alt, newContext.lock(), config->semanticContext); + Ref c = std::make_shared(returnState, config->alt, newContext.lock(), config->semanticContext); // While we have context to pop back from, we may have // gotten that context AFTER having falling off a rule. // Make sure we track that we are now out of context. + // + // This assignment also propagates the + // isPrecedenceFilterSuppressed() value to the new + // configuration. c->reachesIntoOuterContext = config->reachesIntoOuterContext; assert(depth > INT_MIN); - closureCheckingStopState(c, configs, closureBusy, collectPredicates, fullCtx, depth - 1); + closureCheckingStopState(c, configs, closureBusy, collectPredicates, fullCtx, depth - 1, treatEofAsEpsilon); } return; } else if (fullCtx) { @@ -769,22 +912,33 @@ void ParserATNSimulator::closureCheckingStopState(ATNConfig::Ref config, std::sh } } - closure_(config, configs, closureBusy, collectPredicates, fullCtx, depth); + closure_(config, configs, closureBusy, collectPredicates, fullCtx, depth, treatEofAsEpsilon); } -void ParserATNSimulator::closure_(ATNConfig::Ref config, std::shared_ptr configs, - std::set &closureBusy, bool collectPredicates, bool fullCtx, int depth) { +void ParserATNSimulator::closure_(Ref config, Ref configs, + std::set> &closureBusy, bool collectPredicates, bool fullCtx, int depth, bool treatEofAsEpsilon) { ATNState *p = config->state; // optimization if (!p->onlyHasEpsilonTransitions()) { + // make sure to not return here, because EOF transitions can act as + // both epsilon transitions and non-epsilon transitions. configs->add(config, &mergeCache); } for (size_t i = 0; i < p->getNumberOfTransitions(); i++) { Transition *t = p->transition(i); bool continueCollecting = !is(t) && collectPredicates; - ATNConfig::Ref c = getEpsilonTarget(config, t, continueCollecting, depth == 0, fullCtx); + Ref c = getEpsilonTarget(config, t, continueCollecting, depth == 0, fullCtx, treatEofAsEpsilon); if (c != nullptr) { + bool alreadyInSet = closureBusy.count(c) > 0; + if (!alreadyInSet) { + closureBusy.insert(c); + } + if (!t->isEpsilon() && alreadyInSet) { + // avoid infinite recursion for EOF* and EOF+ + continue; + } + int newDepth = depth; if (is(config->state)) { assert(!fullCtx); @@ -801,6 +955,13 @@ void ParserATNSimulator::closure_(ATNConfig::Ref config, std::shared_ptrisPrecedenceDfa()) { + int outermostPrecedenceReturn = dynamic_cast(t)->outermostPrecedenceReturn(); + if (outermostPrecedenceReturn == _dfa->atnStartState->ruleIndex) { + c->setPrecedenceFilterSuppressed(true); + } + } + c->reachesIntoOuterContext++; configs->dipsIntoOuterContext = true; // TO_DO: can remove? only care when we add to set per middle of this method assert(newDepth > INT_MIN); @@ -816,7 +977,7 @@ void ParserATNSimulator::closure_(ATNConfig::Ref config, std::shared_ptr"; } -ATNConfig::Ref ParserATNSimulator::getEpsilonTarget(ATNConfig::Ref config, Transition *t, bool collectPredicates, bool inContext, bool fullCtx) { +Ref ParserATNSimulator::getEpsilonTarget(Ref config, Transition *t, bool collectPredicates, + bool inContext, bool fullCtx, bool treatEofAsEpsilon) { switch (t->getSerializationType()) { case Transition::RULE: return ruleTransition(config, static_cast(t)); @@ -845,19 +1007,32 @@ ATNConfig::Ref ParserATNSimulator::getEpsilonTarget(ATNConfig::Ref config, Trans case Transition::EPSILON: return std::make_shared(config, t->target); + case Transition::ATOM: + case Transition::RANGE: + case Transition::SET: + // EOF transitions act like epsilon transitions after the first EOF + // transition is traversed + if (treatEofAsEpsilon) { + if (t->matches(EOF, 0, 1)) { + return std::make_shared(config, t->target); + } + } + + return nullptr; + default: return nullptr; } } -ATNConfig::Ref ParserATNSimulator::actionTransition(ATNConfig::Ref config, ActionTransition *t) { +Ref ParserATNSimulator::actionTransition(Ref config, ActionTransition *t) { if (debug) { std::wcout << L"ACTION edge " << t->ruleIndex << L":" << t->actionIndex << std::endl; } return std::make_shared(config, t->target); } -ATNConfig::Ref ParserATNSimulator::precedenceTransition(ATNConfig::Ref config, PrecedencePredicateTransition *pt, +Ref ParserATNSimulator::precedenceTransition(Ref config, PrecedencePredicateTransition *pt, bool collectPredicates, bool inContext, bool fullCtx) { if (debug) { std::wcout << L"PRED (collectPredicates=" << collectPredicates << L") " << pt->precedence << L">=_p" << L", ctx dependent=true" << std::endl; @@ -866,9 +1041,9 @@ ATNConfig::Ref ParserATNSimulator::precedenceTransition(ATNConfig::Ref config, P } } - ATNConfig::Ref c = nullptr; + Ref c; if (collectPredicates && inContext) { - std::shared_ptr predicate = pt->getPredicate(); + Ref predicate = pt->getPredicate(); if (fullCtx) { // In full context mode, we can evaluate predicates on-the-fly @@ -877,13 +1052,13 @@ ATNConfig::Ref ParserATNSimulator::precedenceTransition(ATNConfig::Ref config, P // later during conflict resolution. size_t currentPosition = _input->index(); _input->seek((size_t)_startIndex); - bool predSucceeds = predicate->eval(parser, _outerContext); + bool predSucceeds = evalSemanticContext(pt->getPredicate(), _outerContext, config->alt, fullCtx); _input->seek(currentPosition); if (predSucceeds) { c = std::make_shared(config, pt->target); // no pred context } } else { - std::shared_ptr newSemCtx = std::make_shared(config->semanticContext, predicate); + Ref newSemCtx = std::make_shared(config->semanticContext, predicate); c = std::make_shared(config, pt->target, newSemCtx); } } else { @@ -896,7 +1071,8 @@ ATNConfig::Ref ParserATNSimulator::precedenceTransition(ATNConfig::Ref config, P return c; } -ATNConfig::Ref ParserATNSimulator::predTransition(ATNConfig::Ref config, PredicateTransition *pt, bool collectPredicates, bool inContext, bool fullCtx) { +Ref ParserATNSimulator::predTransition(Ref config, PredicateTransition *pt, bool collectPredicates, + bool inContext, bool fullCtx) { if (debug) { std::wcout << L"PRED (collectPredicates=" << collectPredicates << L") " << pt->ruleIndex << L":" << pt->predIndex << L", ctx dependent=" << pt->isCtxDependent << std::endl; if (parser != nullptr) { @@ -904,9 +1080,9 @@ ATNConfig::Ref ParserATNSimulator::predTransition(ATNConfig::Ref config, Predica } } - ATNConfig::Ref c = nullptr; + Ref c = nullptr; if (collectPredicates && (!pt->isCtxDependent || (pt->isCtxDependent && inContext))) { - std::shared_ptr predicate = pt->getPredicate(); + Ref predicate = pt->getPredicate(); if (fullCtx) { // In full context mode, we can evaluate predicates on-the-fly // during closure, which dramatically reduces the size of @@ -914,13 +1090,13 @@ ATNConfig::Ref ParserATNSimulator::predTransition(ATNConfig::Ref config, Predica // later during conflict resolution. size_t currentPosition = _input->index(); _input->seek((size_t)_startIndex); - bool predSucceeds = predicate->eval(parser, _outerContext); + bool predSucceeds = evalSemanticContext(pt->getPredicate(), _outerContext, config->alt, fullCtx); _input->seek(currentPosition); if (predSucceeds) { c = std::make_shared(config, pt->target); // no pred context } } else { - std::shared_ptr newSemCtx = std::make_shared(config->semanticContext, predicate); + Ref newSemCtx = std::make_shared(config->semanticContext, predicate); c = std::make_shared(config, pt->target, newSemCtx); } } else { @@ -933,22 +1109,22 @@ ATNConfig::Ref ParserATNSimulator::predTransition(ATNConfig::Ref config, Predica return c; } -ATNConfig::Ref ParserATNSimulator::ruleTransition(ATNConfig::Ref config, RuleTransition *t) { +Ref ParserATNSimulator::ruleTransition(Ref config, RuleTransition *t) { if (debug) { std::wcout << L"CALL rule " << getRuleName((size_t)t->target->ruleIndex) << L", ctx=" << config->context << std::endl; } atn::ATNState *returnState = t->followState; - PredictionContext::Ref newContext = SingletonPredictionContext::create(config->context, returnState->stateNumber); + Ref newContext = SingletonPredictionContext::create(config->context, returnState->stateNumber); return std::make_shared(config, t->target, newContext); } -BitSet ParserATNSimulator::getConflictingAlts(std::shared_ptr configs) { +BitSet ParserATNSimulator::getConflictingAlts(Ref configs) { std::vector altsets = PredictionModeClass::getConflictingAltSubsets(configs); return PredictionModeClass::getAlts(altsets); } -BitSet ParserATNSimulator::getConflictingAltsOrUniqueAlt(std::shared_ptr configs) { +BitSet ParserATNSimulator::getConflictingAltsOrUniqueAlt(Ref configs) { BitSet conflictingAlts; if (configs->uniqueAlt != ATN::INVALID_ALT_NUMBER) { conflictingAlts.set((size_t)configs->uniqueAlt); @@ -962,16 +1138,14 @@ std::wstring ParserATNSimulator::getTokenName(ssize_t t) { if (t == EOF) { return L"EOF"; } - if (parser != nullptr) { - std::vector tokensNames = parser->getTokenNames(); - if (t >= (ssize_t)tokensNames.size()) { - std::wcerr << t << L" type out of range: " << Arrays::listToString(tokensNames, L", "); - std::wcerr << Arrays::toString(((CommonTokenStream*)parser->getInputStream())->getTokens()); - } else { - return tokensNames[(size_t)t] + L"<" + std::to_wstring(t) + L">"; - } + + Ref vocabulary = parser != nullptr ? parser->getVocabulary() : dfa::VocabularyImpl::EMPTY_VOCABULARY; + std::wstring displayName = vocabulary->getDisplayName(t); + if (displayName == std::to_wstring(t)) { + return displayName; } - return std::to_wstring(t); + + return displayName + L"<" + std::to_wstring(t) + L">"; } std::wstring ParserATNSimulator::getLookaheadName(TokenStream *input) { @@ -999,12 +1173,12 @@ void ParserATNSimulator::dumpDeadEndConfigs(NoViableAltException &nvae) { } } -NoViableAltException ParserATNSimulator::noViableAlt(TokenStream *input, ParserRuleContext::Ref outerContext, - std::shared_ptr configs, size_t startIndex) { +NoViableAltException ParserATNSimulator::noViableAlt(TokenStream *input, Ref outerContext, + Ref configs, size_t startIndex) { return NoViableAltException(parser, input, input->get(startIndex), input->LT(1), configs, outerContext); } -int ParserATNSimulator::getUniqueAlt(std::shared_ptr configs) { +int ParserATNSimulator::getUniqueAlt(Ref configs) { int alt = ATN::INVALID_ALT_NUMBER; for (auto c : configs->configs) { if (alt == ATN::INVALID_ALT_NUMBER) { @@ -1030,7 +1204,6 @@ dfa::DFAState *ParserATNSimulator::addDFAEdge(dfa::DFA &dfa, dfa::DFAState *from return to; } - { std::lock_guard lck(mtx); if (from->edges.empty()) @@ -1039,11 +1212,11 @@ dfa::DFAState *ParserATNSimulator::addDFAEdge(dfa::DFA &dfa, dfa::DFAState *from } if (debug) { - std::vector names; + Ref vocabulary = dfa::VocabularyImpl::EMPTY_VOCABULARY; if (parser != nullptr) { - names = parser->getTokenNames(); + vocabulary = parser->getVocabulary(); } - std::wcout << L"DFA=\n" << dfa.toString(names) << std::endl; + std::wcout << L"DFA=\n" << dfa.toString(vocabulary) << std::endl; } return to; @@ -1076,7 +1249,7 @@ dfa::DFAState *ParserATNSimulator::addDFAState(dfa::DFA &dfa, dfa::DFAState *D) } void ParserATNSimulator::reportAttemptingFullContext(dfa::DFA &dfa, const antlrcpp::BitSet &conflictingAlts, - std::shared_ptr configs, size_t startIndex, size_t stopIndex) { + Ref configs, size_t startIndex, size_t stopIndex) { if (debug || retry_debug) { misc::Interval interval = misc::Interval((int)startIndex, (int)stopIndex); std::wcout << L"reportAttemptingFullContext decision=" << dfa.decision << L":" << configs << L", input=" << parser->getTokenStream()->getText(interval) << std::endl; @@ -1086,7 +1259,7 @@ void ParserATNSimulator::reportAttemptingFullContext(dfa::DFA &dfa, const antlrc } } -void ParserATNSimulator::reportContextSensitivity(dfa::DFA &dfa, int prediction, std::shared_ptr configs, +void ParserATNSimulator::reportContextSensitivity(dfa::DFA &dfa, int prediction, Ref configs, size_t startIndex, size_t stopIndex) { if (debug || retry_debug) { misc::Interval interval = misc::Interval((int)startIndex, (int)stopIndex); @@ -1098,24 +1271,8 @@ void ParserATNSimulator::reportContextSensitivity(dfa::DFA &dfa, int prediction, } void ParserATNSimulator::reportAmbiguity(dfa::DFA &dfa, dfa::DFAState *D, size_t startIndex, size_t stopIndex, - bool exact, const antlrcpp::BitSet &ambigAlts, std::shared_ptr configs) { + bool exact, const antlrcpp::BitSet &ambigAlts, Ref configs) { if (debug || retry_debug) { - // ParserATNPathFinder finder = new ParserATNPathFinder(parser, atn); - // int i = 1; - // for (Transition t : dfa.atnStartState.transitions) { - // System.out.println("ALT "+i+"="); - // System.out.println(startIndex+".."+stopIndex+", len(input)="+parser.getInputStream().size()); - // TraceTree path = finder.trace(t.target, parser.getContext(), (TokenStream)parser.getInputStream(), - // startIndex, stopIndex); - // if ( path!=null ) { - // System.out.println("path = "+path.toStringTree()); - // for (TraceTree leaf : path.leaves) { - // List states = path.getPathToNode(leaf); - // System.out.println("states="+states); - // } - // } - // i++; - // } misc::Interval interval = misc::Interval((int)startIndex, (int)stopIndex); std::wcout << L"reportAmbiguity " << ambigAlts << L":" << configs << L", input=" << parser->getTokenStream()->getText(interval) << std::endl; } @@ -1132,6 +1289,10 @@ atn::PredictionMode ParserATNSimulator::getPredictionMode() { return mode; } +Parser* ParserATNSimulator::getParser() { + return parser; +} + void ParserATNSimulator::InitializeInstanceFields() { mode = PredictionMode::LL; _startIndex = 0; diff --git a/runtime/Cpp/runtime/atn/ParserATNSimulator.h b/runtime/Cpp/runtime/atn/ParserATNSimulator.h index ece315751..73a948304 100755 --- a/runtime/Cpp/runtime/atn/ParserATNSimulator.h +++ b/runtime/Cpp/runtime/atn/ParserATNSimulator.h @@ -44,204 +44,233 @@ namespace v4 { namespace runtime { namespace atn { - /// - /// The embodiment of the adaptive LL(*), ALL(*), parsing strategy. - /// - /// The basic complexity of the adaptive strategy makes it harder to - /// understand. We begin with ATN simulation to build paths in a - /// DFA. Subsequent prediction requests go through the DFA first. If - /// they reach a state without an edge for the current symbol, the - /// algorithm fails over to the ATN simulation to complete the DFA - /// path for the current input (until it finds a conflict state or - /// uniquely predicting state). - /// - /// All of that is done without using the outer context because we - /// want to create a DFA that is not dependent upon the rule - /// invocation stack when we do a prediction. One DFA works in all - /// contexts. We avoid using context not necessarily because it's - /// slower, although it can be, but because of the DFA caching - /// problem. The closure routine only considers the rule invocation - /// stack created during prediction beginning in the decision rule. For - /// example, if prediction occurs without invoking another rule's - /// ATN, there are no context stacks in the configurations. - /// When lack of context leads to a conflict, we don't know if it's - /// an ambiguity or a weakness in the strong LL(*) parsing strategy - /// (versus full LL(*)). - /// - /// When SLL yields a configuration set with conflict, we rewind the - /// input and retry the ATN simulation, this time using - /// full outer context without adding to the DFA. Configuration context - /// stacks will be the full invocation stacks from the start rule. If - /// we get a conflict using full context, then we can definitively - /// say we have a true ambiguity for that input sequence. If we don't - /// get a conflict, it implies that the decision is sensitive to the - /// outer context. (It is not context-sensitive in the sense of - /// context-sensitive grammars.) - /// - /// The next time we reach this DFA state with an SLL conflict, through - /// DFA simulation, we will again retry the ATN simulation using full - /// context mode. This is slow because we can't save the results and have - /// to "interpret" the ATN each time we get that input. - /// - /// CACHING FULL CONTEXT PREDICTIONS - /// - /// We could cache results from full context to predicted - /// alternative easily and that saves a lot of time but doesn't work - /// in presence of predicates. The set of visible predicates from - /// the ATN start state changes depending on the context, because - /// closure can fall off the end of a rule. I tried to cache - /// tuples (stack context, semantic context, predicted alt) but it - /// was slower than interpreting and much more complicated. Also - /// required a huge amount of memory. The goal is not to create the - /// world's fastest parser anyway. I'd like to keep this algorithm - /// simple. By launching multiple threads, we can improve the speed - /// of parsing across a large number of files. - /// - /// There is no strict ordering between the amount of input used by - /// SLL vs LL, which makes it really hard to build a cache for full - /// context. Let's say that we have input A B C that leads to an SLL - /// conflict with full context X. That implies that using X we - /// might only use A B but we could also use A B C D to resolve - /// conflict. Input A B C D could predict alternative 1 in one - /// position in the input and A B C E could predict alternative 2 in - /// another position in input. The conflicting SLL configurations - /// could still be non-unique in the full context prediction, which - /// would lead us to requiring more input than the original A B C. To - /// make a prediction cache work, we have to track the exact input used - /// during the previous prediction. That amounts to a cache that maps X - /// to a specific DFA for that context. - /// - /// Something should be done for left-recursive expression predictions. - /// They are likely LL(1) + pred eval. Easier to do the whole SLL unless - /// error and retry with full LL thing Sam does. - /// - /// AVOIDING FULL CONTEXT PREDICTION - /// - /// We avoid doing full context retry when the outer context is empty, - /// we did not dip into the outer context by falling off the end of the - /// decision state rule, or when we force SLL mode. - /// - /// As an example of the not dip into outer context case, consider - /// as super constructor calls versus function calls. One grammar - /// might look like this: - /// - /// ctorBody : '{' superCall? stat* '}' ; - /// - /// Or, you might see something like - /// - /// stat : superCall ';' | expression ';' | ... ; - /// - /// In both cases I believe that no closure operations will dip into the - /// outer context. In the first case ctorBody in the worst case will stop - /// at the '}'. In the 2nd case it should stop at the ';'. Both cases - /// should stay within the entry rule and not dip into the outer context. - /// - /// PREDICATES - /// - /// Predicates are always evaluated if present in either SLL or LL both. - /// SLL and LL simulation deals with predicates differently. SLL collects - /// predicates as it performs closure operations like ANTLR v3 did. It - /// delays predicate evaluation until it reaches and accept state. This - /// allows us to cache the SLL ATN simulation whereas, if we had evaluated - /// predicates on-the-fly during closure, the DFA state configuration sets - /// would be different and we couldn't build up a suitable DFA. - /// - /// When building a DFA accept state during ATN simulation, we evaluate - /// any predicates and return the sole semantically valid alternative. If - /// there is more than 1 alternative, we report an ambiguity. If there are - /// 0 alternatives, we throw an exception. Alternatives without predicates - /// act like they have true predicates. The simple way to think about it - /// is to strip away all alternatives with false predicates and choose the - /// minimum alternative that remains. - /// - /// When we start in the DFA and reach an accept state that's predicated, - /// we test those and return the minimum semantically viable - /// alternative. If no alternatives are viable, we throw an exception. - /// - /// During full LL ATN simulation, closure always evaluates predicates and - /// on-the-fly. This is crucial to reducing the configuration set size - /// during closure. It hits a landmine when parsing with the Java grammar, - /// for example, without this on-the-fly evaluation. - /// - /// SHARING DFA - /// - /// All instances of the same parser share the same decision DFAs through - /// a static field. Each instance gets its own ATN simulator but they - /// share the same decisionToDFA field. They also share a - /// PredictionContextCache object that makes sure that all - /// PredictionContext objects are shared among the DFA states. This makes - /// a big size difference. - /// - /// THREAD SAFETY - /// - /// The parser ATN simulator locks on the decisionDFA field when it adds a - /// new DFA object to that array. addDFAEdge locks on the DFA for the - /// current decision when setting the edges[] field. addDFAState locks on - /// the DFA for the current decision when looking up a DFA state to see if - /// it already exists. We must make sure that all requests to add DFA - /// states that are equivalent result in the same shared DFA object. This - /// is because lots of threads will be trying to update the DFA at - /// once. The addDFAState method also locks inside the DFA lock but this - /// time on the shared context cache when it rebuilds the configurations' - /// PredictionContext objects using cached subgraphs/nodes. No other - /// locking occurs, even during DFA simulation. This is safe as long as we - /// can guarantee that all threads referencing s.edge[t] get the same - /// physical target DFA state, or none. Once into the DFA, the DFA - /// simulation does not reference the dfa.state map. It follows the - /// edges[] field to new targets. The DFA simulator will either find - /// dfa.edges to be null, to be non-null and dfa.edges[t] null, or - /// dfa.edges[t] to be non-null. The addDFAEdge method could be racing to - /// set the field but in either case the DFA simulator works; if null, and - /// requests ATN simulation. It could also race trying to get - /// dfa.edges[t], but either way it will work because it's not doing a - /// test and set operation. - /// - /// Starting with SLL then failing to combined SLL/LL - /// - /// Sam pointed out that if SLL does not give a syntax error, then there - /// is no point in doing full LL, which is slower. We only have to try LL - /// if we get a syntax error. For maximum speed, Sam starts the parser - /// with pure SLL mode: - /// - /// parser.getInterpreter().setSLL(true); - /// - /// and with the bail error strategy: - /// - /// parser.setErrorHandler(new BailErrorStrategy()); - /// - /// If it does not get a syntax error, then we're done. If it does get a - /// syntax error, we need to retry with the combined SLL/LL strategy. - /// - /// The reason this works is as follows. If there are no SLL - /// conflicts then the grammar is SLL for sure, at least for that - /// input set. If there is an SLL conflict, the full LL analysis - /// must yield a set of ambiguous alternatives that is no larger - /// than the SLL set. If the LL set is a singleton, then the grammar - /// is LL but not SLL. If the LL set is the same size as the SLL - /// set, the decision is SLL. If the LL set has size > 1, then that - /// decision is truly ambiguous on the current input. If the LL set - /// is smaller, then the SLL conflict resolution might choose an - /// alternative that the full LL would rule out as a possibility - /// based upon better context information. If that's the case, then - /// the SLL parse will definitely get an error because the full LL - /// analysis says it's not viable. If SLL conflict resolution - /// chooses an alternative within the LL set, them both SLL and LL - /// would choose the same alternative because they both choose the - /// minimum of multiple conflicting alternatives. - /// - /// Let's say we have a set of SLL conflicting alternatives {1, 2, 3} and - /// a smaller LL set called s. If s is {2, 3}, then SLL parsing will get - /// an error because SLL will pursue alternative 1. If s is {1, 2} or {1, - /// 3} then both SLL and LL will choose the same alternative because - /// alternative one is the minimum of either set. If s is {2} or {3} then - /// SLL will get a syntax error. If s is {1} then SLL will succeed. - /// - /// Of course, if the input is invalid, then we will get an error for sure - /// in both SLL and LL parsing. Erroneous input will therefore require 2 - /// passes over the input. - /// - /// - + /** + * The embodiment of the adaptive LL(*), ALL(*), parsing strategy. + * + *

+ * The basic complexity of the adaptive strategy makes it harder to understand. + * We begin with ATN simulation to build paths in a DFA. Subsequent prediction + * requests go through the DFA first. If they reach a state without an edge for + * the current symbol, the algorithm fails over to the ATN simulation to + * complete the DFA path for the current input (until it finds a conflict state + * or uniquely predicting state).

+ * + *

+ * All of that is done without using the outer context because we want to create + * a DFA that is not dependent upon the rule invocation stack when we do a + * prediction. One DFA works in all contexts. We avoid using context not + * necessarily because it's slower, although it can be, but because of the DFA + * caching problem. The closure routine only considers the rule invocation stack + * created during prediction beginning in the decision rule. For example, if + * prediction occurs without invoking another rule's ATN, there are no context + * stacks in the configurations. When lack of context leads to a conflict, we + * don't know if it's an ambiguity or a weakness in the strong LL(*) parsing + * strategy (versus full LL(*)).

+ * + *

+ * When SLL yields a configuration set with conflict, we rewind the input and + * retry the ATN simulation, this time using full outer context without adding + * to the DFA. Configuration context stacks will be the full invocation stacks + * from the start rule. If we get a conflict using full context, then we can + * definitively say we have a true ambiguity for that input sequence. If we + * don't get a conflict, it implies that the decision is sensitive to the outer + * context. (It is not context-sensitive in the sense of context-sensitive + * grammars.)

+ * + *

+ * The next time we reach this DFA state with an SLL conflict, through DFA + * simulation, we will again retry the ATN simulation using full context mode. + * This is slow because we can't save the results and have to "interpret" the + * ATN each time we get that input.

+ * + *

+ * CACHING FULL CONTEXT PREDICTIONS

+ * + *

+ * We could cache results from full context to predicted alternative easily and + * that saves a lot of time but doesn't work in presence of predicates. The set + * of visible predicates from the ATN start state changes depending on the + * context, because closure can fall off the end of a rule. I tried to cache + * tuples (stack context, semantic context, predicted alt) but it was slower + * than interpreting and much more complicated. Also required a huge amount of + * memory. The goal is not to create the world's fastest parser anyway. I'd like + * to keep this algorithm simple. By launching multiple threads, we can improve + * the speed of parsing across a large number of files.

+ * + *

+ * There is no strict ordering between the amount of input used by SLL vs LL, + * which makes it really hard to build a cache for full context. Let's say that + * we have input A B C that leads to an SLL conflict with full context X. That + * implies that using X we might only use A B but we could also use A B C D to + * resolve conflict. Input A B C D could predict alternative 1 in one position + * in the input and A B C E could predict alternative 2 in another position in + * input. The conflicting SLL configurations could still be non-unique in the + * full context prediction, which would lead us to requiring more input than the + * original A B C. To make a prediction cache work, we have to track the exact + * input used during the previous prediction. That amounts to a cache that maps + * X to a specific DFA for that context.

+ * + *

+ * Something should be done for left-recursive expression predictions. They are + * likely LL(1) + pred eval. Easier to do the whole SLL unless error and retry + * with full LL thing Sam does.

+ * + *

+ * AVOIDING FULL CONTEXT PREDICTION

+ * + *

+ * We avoid doing full context retry when the outer context is empty, we did not + * dip into the outer context by falling off the end of the decision state rule, + * or when we force SLL mode.

+ * + *

+ * As an example of the not dip into outer context case, consider as super + * constructor calls versus function calls. One grammar might look like + * this:

+ * + * 
+   * ctorBody
+   *   : '{' superCall? stat* '}'
+   *   ;
+   *
+ * + *

+ * Or, you might see something like

+ * + * 
+   * stat
+   *   : superCall ';'
+   *   | expression ';'
+   *   | ...
+   *   ;
+   *
+ * + *

+ * In both cases I believe that no closure operations will dip into the outer + * context. In the first case ctorBody in the worst case will stop at the '}'. + * In the 2nd case it should stop at the ';'. Both cases should stay within the + * entry rule and not dip into the outer context.

+ * + *

+ * PREDICATES

+ * + *

+ * Predicates are always evaluated if present in either SLL or LL both. SLL and + * LL simulation deals with predicates differently. SLL collects predicates as + * it performs closure operations like ANTLR v3 did. It delays predicate + * evaluation until it reaches and accept state. This allows us to cache the SLL + * ATN simulation whereas, if we had evaluated predicates on-the-fly during + * closure, the DFA state configuration sets would be different and we couldn't + * build up a suitable DFA.

+ * + *

+ * When building a DFA accept state during ATN simulation, we evaluate any + * predicates and return the sole semantically valid alternative. If there is + * more than 1 alternative, we report an ambiguity. If there are 0 alternatives, + * we throw an exception. Alternatives without predicates act like they have + * true predicates. The simple way to think about it is to strip away all + * alternatives with false predicates and choose the minimum alternative that + * remains.

+ * + *

+ * When we start in the DFA and reach an accept state that's predicated, we test + * those and return the minimum semantically viable alternative. If no + * alternatives are viable, we throw an exception.

+ * + *

+ * During full LL ATN simulation, closure always evaluates predicates and + * on-the-fly. This is crucial to reducing the configuration set size during + * closure. It hits a landmine when parsing with the Java grammar, for example, + * without this on-the-fly evaluation.

+ * + *

+ * SHARING DFA

+ * + *

+ * All instances of the same parser share the same decision DFAs through a + * static field. Each instance gets its own ATN simulator but they share the + * same {@link #decisionToDFA} field. They also share a + * {@link PredictionContextCache} object that makes sure that all + * {@link PredictionContext} objects are shared among the DFA states. This makes + * a big size difference.

+ * + *

+ * THREAD SAFETY

+ * + *

+ * The {@link ParserATNSimulator} locks on the {@link #decisionToDFA} field when + * it adds a new DFA object to that array. {@link #addDFAEdge} + * locks on the DFA for the current decision when setting the + * {@link DFAState#edges} field. {@link #addDFAState} locks on + * the DFA for the current decision when looking up a DFA state to see if it + * already exists. We must make sure that all requests to add DFA states that + * are equivalent result in the same shared DFA object. This is because lots of + * threads will be trying to update the DFA at once. The + * {@link #addDFAState} method also locks inside the DFA lock + * but this time on the shared context cache when it rebuilds the + * configurations' {@link PredictionContext} objects using cached + * subgraphs/nodes. No other locking occurs, even during DFA simulation. This is + * safe as long as we can guarantee that all threads referencing + * {@code s.edge[t]} get the same physical target {@link DFAState}, or + * {@code null}. Once into the DFA, the DFA simulation does not reference the + * {@link DFA#states} map. It follows the {@link DFAState#edges} field to new + * targets. The DFA simulator will either find {@link DFAState#edges} to be + * {@code null}, to be non-{@code null} and {@code dfa.edges[t]} null, or + * {@code dfa.edges[t]} to be non-null. The + * {@link #addDFAEdge} method could be racing to set the field + * but in either case the DFA simulator works; if {@code null}, and requests ATN + * simulation. It could also race trying to get {@code dfa.edges[t]}, but either + * way it will work because it's not doing a test and set operation.

+ * + *

+ * Starting with SLL then failing to combined SLL/LL (Two-Stage + * Parsing)

+ * + *

+ * Sam pointed out that if SLL does not give a syntax error, then there is no + * point in doing full LL, which is slower. We only have to try LL if we get a + * syntax error. For maximum speed, Sam starts the parser set to pure SLL + * mode with the {@link BailErrorStrategy}:

+ * + * 
+   * parser.{@link Parser#getInterpreter() getInterpreter()}.{@link #setPredictionMode setPredictionMode}{@code (}{@link PredictionMode#SLL}{@code )};
+   * parser.{@link Parser#setErrorHandler setErrorHandler}(new {@link BailErrorStrategy}());
+   *
+ * + *

+ * If it does not get a syntax error, then we're done. If it does get a syntax + * error, we need to retry with the combined SLL/LL strategy.

+ * + *

+ * The reason this works is as follows. If there are no SLL conflicts, then the + * grammar is SLL (at least for that input set). If there is an SLL conflict, + * the full LL analysis must yield a set of viable alternatives which is a + * subset of the alternatives reported by SLL. If the LL set is a singleton, + * then the grammar is LL but not SLL. If the LL set is the same size as the SLL + * set, the decision is SLL. If the LL set has size > 1, then that decision + * is truly ambiguous on the current input. If the LL set is smaller, then the + * SLL conflict resolution might choose an alternative that the full LL would + * rule out as a possibility based upon better context information. If that's + * the case, then the SLL parse will definitely get an error because the full LL + * analysis says it's not viable. If SLL conflict resolution chooses an + * alternative within the LL set, them both SLL and LL would choose the same + * alternative because they both choose the minimum of multiple conflicting + * alternatives.

+ * + *

+ * Let's say we have a set of SLL conflicting alternatives {@code {1, 2, 3}} and + * a smaller LL set called s. If s is {@code {2, 3}}, then SLL + * parsing will get an error because SLL will pursue alternative 1. If + * s is {@code {1, 2}} or {@code {1, 3}} then both SLL and LL will + * choose the same alternative because alternative one is the minimum of either + * set. If s is {@code {2}} or {@code {3}} then SLL will get a syntax + * error. If s is {@code {1}} then SLL will succeed.

+ * + *

+ * Of course, if the input is invalid, then we will get an error for sure in + * both SLL and LL parsing. Erroneous input will therefore require 2 passes over + * the input.

+ */ class ParserATNSimulator : public ATNSimulator { public: static const bool debug = false; @@ -253,14 +282,15 @@ namespace atn { Parser *const parser; public: - std::vector &_decisionToDFA; + std::vector &decisionToDFA; /// /// SLL, LL, or LL + exact ambig detection? private: PredictionMode mode; - //Mutex to manage synchronized access for multithreading in the parser atn simulator - std::mutex mtx; + + // Mutex to manage synchronized access for multithreading in the parser atn simulator. + std::mutex _lock; /// /// Each prediction operation uses a cache for merge of prediction contexts. @@ -277,20 +307,20 @@ namespace atn { // LAME globals to avoid parameters!!!!! I need these down deep in predTransition TokenStream *_input; int _startIndex; - ParserRuleContext::Ref _outerContext; + Ref _outerContext; + dfa::DFA *_dfa; // Reference into the decisionToDFA vector. - /// - /// Testing only! public: - ParserATNSimulator(const ATN &atn, std::vector& decisionToDFA, - std::shared_ptr sharedContextCache); + /// Testing only! + ParserATNSimulator(const ATN &atn, std::vector &decisionToDFA, + Ref sharedContextCache); ParserATNSimulator(Parser *parser, const ATN &atn, std::vector &decisionToDFA, - std::shared_ptr sharedContextCache); + Ref sharedContextCache); virtual void reset() override; - - virtual int adaptivePredict(TokenStream *input, int decision, ParserRuleContext::Ref outerContext); + virtual void clearDFA() override; + virtual int adaptivePredict(TokenStream *input, int decision, Ref outerContext); /// /// Performs ATN simulation to compute a predicted alternative based @@ -325,7 +355,7 @@ namespace atn { /// protected: virtual int execATN(dfa::DFA &dfa, dfa::DFAState *s0, TokenStream *input, size_t startIndex, - ParserRuleContext::Ref outerContext); + Ref outerContext); /// /// Get an existing target state for an edge in the DFA. If the target state @@ -355,10 +385,10 @@ namespace atn { virtual void predicateDFAState(dfa::DFAState *dfaState, DecisionState *decisionState); // comes back with reach.uniqueAlt set to a valid alt - virtual int execATNWithFullContext(dfa::DFA &dfa, dfa::DFAState *D, std::shared_ptr s0, - TokenStream *input, size_t startIndex, ParserRuleContext::Ref outerContext); // how far we got before failing over + virtual int execATNWithFullContext(dfa::DFA &dfa, dfa::DFAState *D, Ref s0, + TokenStream *input, size_t startIndex, Ref outerContext); // how far we got before failing over - virtual std::shared_ptr computeReachSet(std::shared_ptr closure, ssize_t t, bool fullCtx); + virtual Ref computeReachSet(Ref closure, ssize_t t, bool fullCtx); /// /// Return a configuration set containing only the configurations from @@ -379,19 +409,248 @@ namespace atn { /// {@code configs} if all configurations in {@code configs} are in a /// rule stop state, otherwise return a new configuration set containing only /// the configurations from {@code configs} which are in a rule stop state - virtual std::shared_ptr removeAllConfigsNotInRuleStopState(std::shared_ptr configs, bool lookToEndOfRule); + virtual Ref removeAllConfigsNotInRuleStopState(Ref configs, bool lookToEndOfRule); - virtual std::shared_ptr computeStartState(ATNState *p, RuleContext::Ref ctx, bool fullCtx); + virtual Ref computeStartState(ATNState *p, Ref ctx, bool fullCtx); + /* parrt internal source braindump that doesn't mess up + * external API spec. + + applyPrecedenceFilter is an optimization to avoid highly + nonlinear prediction of expressions and other left recursive + rules. The precedence predicates such as {3>=prec}? Are highly + context-sensitive in that they can only be properly evaluated + in the context of the proper prec argument. Without pruning, + these predicates are normal predicates evaluated when we reach + conflict state (or unique prediction). As we cannot evaluate + these predicates out of context, the resulting conflict leads + to full LL evaluation and nonlinear prediction which shows up + very clearly with fairly large expressions. + + Example grammar: + + e : e '*' e + | e '+' e + | INT + ; + + We convert that to the following: + + e[int prec] + : INT + ( {3>=prec}? '*' e[4] + | {2>=prec}? '+' e[3] + )* + ; + + The (..)* loop has a decision for the inner block as well as + an enter or exit decision, which is what concerns us here. At + the 1st + of input 1+2+3, the loop entry sees both predicates + and the loop exit also sees both predicates by falling off the + edge of e. This is because we have no stack information with + SLL and find the follow of e, which will hit the return states + inside the loop after e[4] and e[3], which brings it back to + the enter or exit decision. In this case, we know that we + cannot evaluate those predicates because we have fallen off + the edge of the stack and will in general not know which prec + parameter is the right one to use in the predicate. + + Because we have special information, that these are precedence + predicates, we can resolve them without failing over to full + LL despite their context sensitive nature. We make an + assumption that prec[-1] <= prec[0], meaning that the current + precedence level is greater than or equal to the precedence + level of recursive invocations above us in the stack. For + example, if predicate {3>=prec}? is true of the current prec, + then one option is to enter the loop to match it now. The + other option is to exit the loop and the left recursive rule + to match the current operator in rule invocation further up + the stack. But, we know that all of those prec are lower or + the same value and so we can decide to enter the loop instead + of matching it later. That means we can strip out the other + configuration for the exit branch. + + So imagine we have (14,1,$,{2>=prec}?) and then + (14,2,$-dipsIntoOuterContext,{2>=prec}?). The optimization + allows us to collapse these two configurations. We know that + if {2>=prec}? is true for the current prec parameter, it will + also be true for any prec from an invoking e call, indicated + by dipsIntoOuterContext. As the predicates are both true, we + have the option to evaluate them early in the decision start + state. We do this by stripping both predicates and choosing to + enter the loop as it is consistent with the notion of operator + precedence. It's also how the full LL conflict resolution + would work. + + The solution requires a different DFA start state for each + precedence level. + + The basic filter mechanism is to remove configurations of the + form (p, 2, pi) if (p, 1, pi) exists for the same p and pi. In + other words, for the same ATN state and predicate context, + remove any configuration associated with an exit branch if + there is a configuration associated with the enter branch. + + It's also the case that the filter evaluates precedence + predicates and resolves conflicts according to precedence + levels. For example, for input 1+2+3 at the first +, we see + prediction filtering + + [(11,1,[$],{3>=prec}?), (14,1,[$],{2>=prec}?), (5,2,[$],up=1), + (11,2,[$],up=1), (14,2,[$],up=1)],hasSemanticContext=true,dipsIntoOuterContext + + to + + [(11,1,[$]), (14,1,[$]), (5,2,[$],up=1)],dipsIntoOuterContext + + This filters because {3>=prec}? evals to true and collapses + (11,1,[$],{3>=prec}?) and (11,2,[$],up=1) since early conflict + resolution based upon rules of operator precedence fits with + our usual match first alt upon conflict. + + We noticed a problem where a recursive call resets precedence + to 0. Sam's fix: each config has flag indicating if it has + returned from an expr[0] call. then just don't filter any + config with that flag set. flag is carried along in + closure(). so to avoid adding field, set bit just under sign + bit of dipsIntoOuterContext (SUPPRESS_PRECEDENCE_FILTER). + With the change you filter "unless (p, 2, pi) was reached + after leaving the rule stop state of the LR rule containing + state p, corresponding to a rule invocation with precedence + level 0" + */ + + /** + * This method transforms the start state computed by + * {@link #computeStartState} to the special start state used by a + * precedence DFA for a particular precedence value. The transformation + * process applies the following changes to the start state's configuration + * set. + * + *
    + *
  1. Evaluate the precedence predicates for each configuration using + * {@link SemanticContext#evalPrecedence}.
    2. + *
  2. When {@link ATNConfig#isPrecedenceFilterSuppressed} is {@code false}, + * remove all configurations which predict an alternative greater than 1, + * for which another configuration that predicts alternative 1 is in the + * same ATN state with the same prediction context. This transformation is + * valid for the following reasons: + *
      + *
    • The closure block cannot contain any epsilon transitions which bypass + * the body of the closure, so all states reachable via alternative 1 are + * part of the precedence alternatives of the transformed left-recursive + * rule.
      • + *
    • The "primary" portion of a left recursive rule cannot contain an + * epsilon transition, so the only way an alternative other than 1 can exist + * in a state that is also reachable via alternative 1 is by nesting calls + * to the left-recursive rule, with the outer calls not being at the + * preferred precedence level. The + * {@link ATNConfig#isPrecedenceFilterSuppressed} property marks ATN + * configurations which do not meet this condition, and therefore are not + * eligible for elimination during the filtering process.
      • + *
    + *
    3. + *
+ * + *

+ * The prediction context must be considered by this filter to address + * situations like the following. + *

+ * + * 
+     * grammar TA;
+     * prog: statement* EOF;
+     * statement: letterA | statement letterA 'b' ;
+     * letterA: 'a';
+     *
+ * + *

+ * If the above grammar, the ATN state immediately before the token + * reference {@code 'a'} in {@code letterA} is reachable from the left edge + * of both the primary and closure blocks of the left-recursive rule + * {@code statement}. The prediction context associated with each of these + * configurations distinguishes between them, and prevents the alternative + * which stepped out to {@code prog} (and then back in to {@code statement} + * from being eliminated by the filter. + *

+ * + * @param configs The configuration set computed by + * {@link #computeStartState} as the start state for the DFA. + * @return The transformed configuration set representing the start state + * for a precedence DFA at a particular precedence level (determined by + * calling {@link Parser#getPrecedence}). + */ + Ref applyPrecedenceFilter(Ref configs); + virtual ATNState *getReachableTarget(Transition *trans, int ttype); - virtual std::vector getPredsForAmbigAlts(const antlrcpp::BitSet &ambigAlts, - std::shared_ptr configs, size_t nalts); + virtual std::vector> getPredsForAmbigAlts(const antlrcpp::BitSet &ambigAlts, + Ref configs, size_t nalts); virtual std::vector getPredicatePredictions(const antlrcpp::BitSet &ambigAlts, - std::vector altToPred); + std::vector> altToPred); - virtual int getAltThatFinishedDecisionEntryRule(std::shared_ptr configs); + /** + * This method is used to improve the localization of error messages by + * choosing an alternative rather than throwing a + * {@link NoViableAltException} in particular prediction scenarios where the + * {@link #ERROR} state was reached during ATN simulation. + * + *

+ * The default implementation of this method uses the following + * algorithm to identify an ATN configuration which successfully parsed the + * decision entry rule. Choosing such an alternative ensures that the + * {@link ParserRuleContext} returned by the calling rule will be complete + * and valid, and the syntax error will be reported later at a more + * localized location.

+ * + *
    + *
  • If a syntactically valid path or paths reach the end of the decision rule and + * they are semantically valid if predicated, return the min associated alt.
    • + *
  • Else, if a semantically invalid but syntactically valid path exist + * or paths exist, return the minimum associated alt. + *
    • + *
  • Otherwise, return {@link ATN#INVALID_ALT_NUMBER}.
    • + *
+ * + *

+ * In some scenarios, the algorithm described above could predict an + * alternative which will result in a {@link FailedPredicateException} in + * the parser. Specifically, this could occur if the only configuration + * capable of successfully parsing to the end of the decision rule is + * blocked by a semantic predicate. By choosing this alternative within + * {@link #adaptivePredict} instead of throwing a + * {@link NoViableAltException}, the resulting + * {@link FailedPredicateException} in the parser will identify the specific + * predicate which is preventing the parser from successfully parsing the + * decision rule, which helps developers identify and correct logic errors + * in semantic predicates. + *

+ * + * @param configs The ATN configurations which were valid immediately before + * the {@link #ERROR} state was reached + * @param outerContext The is the \gamma_0 initial parser context from the paper + * or the parser stack at the instant before prediction commences. + * + * @return The value to return from {@link #adaptivePredict}, or + * {@link ATN#INVALID_ALT_NUMBER} if a suitable alternative was not + * identified and {@link #adaptivePredict} should report an error instead. + */ + int getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule(Ref configs, Ref outerContext); + + virtual int getAltThatFinishedDecisionEntryRule(Ref configs); + + /** Walk the list of configurations and split them according to + * those that have preds evaluating to true/false. If no pred, assume + * true pred and include in succeeded set. Returns Pair of sets. + * + * Create a new set so as not to alter the incoming parameter. + * + * Assumption: the input stream has been restored to the starting point + * prediction, which is where predicates need to evaluate. + */ + std::pair, Ref> splitAccordingToSemanticValidity(Ref configs, + Ref outerContext); /// /// Look through a list of predicate/alt pairs, returning alts for the @@ -401,42 +660,85 @@ namespace atn { /// includes pairs with null predicates. /// virtual antlrcpp::BitSet evalSemanticContext(std::vector predPredictions, - ParserRuleContext::Ref outerContext, bool complete); + Ref outerContext, bool complete); + /** + * Evaluate a semantic context within a specific parser context. + * + *

+ * This method might not be called for every semantic context evaluated + * during the prediction process. In particular, we currently do not + * evaluate the following but it may change in the future:

+ * + *
    + *
  • Precedence predicates (represented by + * {@link SemanticContext.PrecedencePredicate}) are not currently evaluated + * through this method.
    • + *
  • Operator predicates (represented by {@link SemanticContext.AND} and + * {@link SemanticContext.OR}) are evaluated as a single semantic + * context, rather than evaluating the operands individually. + * Implementations which require evaluation results from individual + * predicates should override this method to explicitly handle evaluation of + * the operands within operator predicates.
    • + *
+ * + * @param pred The semantic context to evaluate + * @param parserCallStack The parser context in which to evaluate the + * semantic context + * @param alt The alternative which is guarded by {@code pred} + * @param fullCtx {@code true} if the evaluation is occurring during LL + * prediction; otherwise, {@code false} if the evaluation is occurring + * during SLL prediction + * + * @since 4.3 + */ + virtual bool evalSemanticContext(Ref pred, Ref parserCallStack, int alt, bool fullCtx); + /* TO_DO: If we are doing predicates, there is no point in pursuing closure operations if we reach a DFA state that uniquely predicts alternative. We will not be caching that DFA state and it is a waste to pursue the closure. Might have to advance when we do ambig detection thought :( */ + virtual void closure(Ref config, Ref configs, std::set> &closureBusy, + bool collectPredicates, bool fullCtx, bool treatEofAsEpsilon); - virtual void closure(ATNConfig::Ref config, std::shared_ptr configs, std::set &closureBusy, - bool collectPredicates, bool fullCtx); - - virtual void closureCheckingStopState(ATNConfig::Ref config, std::shared_ptr configs, - std::set &closureBusy, bool collectPredicates, bool fullCtx, int depth); + virtual void closureCheckingStopState(Ref config, Ref configs, + std::set> &closureBusy, bool collectPredicates, bool fullCtx, int depth, bool treatEofAsEpsilon); /// Do the actual work of walking epsilon edges. - virtual void closure_(ATNConfig::Ref config, std::shared_ptr configs, std::set &closureBusy, - bool collectPredicates, bool fullCtx, int depth); + virtual void closure_(Ref config, Ref configs, std::set> &closureBusy, + bool collectPredicates, bool fullCtx, int depth, bool treatEofAsEpsilon); public: virtual std::wstring getRuleName(size_t index); protected: - virtual ATNConfig::Ref getEpsilonTarget(ATNConfig::Ref config, Transition *t, bool collectPredicates, bool inContext, bool fullCtx); - virtual ATNConfig::Ref actionTransition(ATNConfig::Ref config, ActionTransition *t); + virtual Ref getEpsilonTarget(Ref config, Transition *t, bool collectPredicates, + bool inContext, bool fullCtx, bool treatEofAsEpsilon); + virtual Ref actionTransition(Ref config, ActionTransition *t); public: - virtual ATNConfig::Ref precedenceTransition(ATNConfig::Ref config, PrecedencePredicateTransition *pt, bool collectPredicates, bool inContext, bool fullCtx); + virtual Ref precedenceTransition(Ref config, PrecedencePredicateTransition *pt, + bool collectPredicates, bool inContext, bool fullCtx); protected: - virtual ATNConfig::Ref predTransition(ATNConfig::Ref config, PredicateTransition *pt, bool collectPredicates, bool inContext, bool fullCtx); + virtual Ref predTransition(Ref config, PredicateTransition *pt, bool collectPredicates, + bool inContext, bool fullCtx); - virtual ATNConfig::Ref ruleTransition(ATNConfig::Ref config, RuleTransition *t); + virtual Ref ruleTransition(Ref config, RuleTransition *t); - virtual antlrcpp::BitSet getConflictingAlts(std::shared_ptr configs); + /** + * Gets a {@link BitSet} containing the alternatives in {@code configs} + * which are part of one or more conflicting alternative subsets. + * + * @param configs The {@link ATNConfigSet} to analyze. + * @return The alternatives in {@code configs} which are part of one or more + * conflicting alternative subsets. If {@code configs} does not contain any + * conflicting subsets, this method returns an empty {@link BitSet}. + */ + virtual antlrcpp::BitSet getConflictingAlts(Ref configs); /// /// Sam pointed out a problem with the previous definition, v3, of @@ -475,7 +777,7 @@ namespace atn { /// that we still need to pursue. /// - virtual antlrcpp::BitSet getConflictingAltsOrUniqueAlt(std::shared_ptr configs); + virtual antlrcpp::BitSet getConflictingAltsOrUniqueAlt(Ref configs); public: virtual std::wstring getTokenName(ssize_t t); @@ -490,10 +792,10 @@ namespace atn { virtual void dumpDeadEndConfigs(NoViableAltException &nvae); protected: - virtual NoViableAltException noViableAlt(TokenStream *input, ParserRuleContext::Ref outerContext, - std::shared_ptr configs, size_t startIndex); + virtual NoViableAltException noViableAlt(TokenStream *input, Ref outerContext, + Ref configs, size_t startIndex); - static int getUniqueAlt(std::shared_ptr configs); + static int getUniqueAlt(Ref configs); /// /// Add an edge to the DFA, if possible. This method calls @@ -533,21 +835,25 @@ namespace atn { virtual dfa::DFAState *addDFAState(dfa::DFA &dfa, dfa::DFAState *D); virtual void reportAttemptingFullContext(dfa::DFA &dfa, const antlrcpp::BitSet &conflictingAlts, - std::shared_ptr configs, size_t startIndex, size_t stopIndex); + Ref configs, size_t startIndex, size_t stopIndex); - virtual void reportContextSensitivity(dfa::DFA &dfa, int prediction, std::shared_ptr configs, + virtual void reportContextSensitivity(dfa::DFA &dfa, int prediction, Ref configs, size_t startIndex, size_t stopIndex); - /// - /// If context sensitive parsing, we know it's ambiguity not conflict - virtual void reportAmbiguity(dfa::DFA &dfa, dfa::DFAState *D, size_t startIndex, size_t stopIndex, bool exact, - const antlrcpp::BitSet &ambigAlts, std::shared_ptr configs); + /// If context sensitive parsing, we know it's ambiguity not conflict. + virtual void reportAmbiguity(dfa::DFA &dfa, + dfa::DFAState *D, // the DFA state from execATN() that had SLL conflicts + size_t startIndex, size_t stopIndex, + bool exact, + const antlrcpp::BitSet &ambigAlts, + Ref configs); // configs that LL not SLL considered conflicting public: void setPredictionMode(PredictionMode mode); - PredictionMode getPredictionMode(); + Parser* getParser(); + private: void InitializeInstanceFields(); }; diff --git a/runtime/Cpp/runtime/atn/PrecedencePredicateTransition.cpp b/runtime/Cpp/runtime/atn/PrecedencePredicateTransition.cpp index f9b17fe2f..d2f43050f 100755 --- a/runtime/Cpp/runtime/atn/PrecedencePredicateTransition.cpp +++ b/runtime/Cpp/runtime/atn/PrecedencePredicateTransition.cpp @@ -48,7 +48,7 @@ bool PrecedencePredicateTransition::matches(int symbol, int minVocabSymbol, int return false; } -std::shared_ptr PrecedencePredicateTransition::getPredicate() const { +Ref PrecedencePredicateTransition::getPredicate() const { return std::make_shared(precedence); } diff --git a/runtime/Cpp/runtime/atn/PrecedencePredicateTransition.h b/runtime/Cpp/runtime/atn/PrecedencePredicateTransition.h index d10c8c8d4..ed88ade4d 100755 --- a/runtime/Cpp/runtime/atn/PrecedencePredicateTransition.h +++ b/runtime/Cpp/runtime/atn/PrecedencePredicateTransition.h @@ -49,7 +49,7 @@ namespace atn { virtual int getSerializationType() const override; virtual bool isEpsilon() const override; virtual bool matches(int symbol, int minVocabSymbol, int maxVocabSymbol) const override; - std::shared_ptr getPredicate() const; + Ref getPredicate() const; virtual std::wstring toString() const override; }; diff --git a/runtime/Cpp/runtime/atn/PredicateEvalInfo.cpp b/runtime/Cpp/runtime/atn/PredicateEvalInfo.cpp new file mode 100755 index 000000000..e1730cfbd --- /dev/null +++ b/runtime/Cpp/runtime/atn/PredicateEvalInfo.cpp @@ -0,0 +1,43 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "SemanticContext.h" + +#include "PredicateEvalInfo.h" + +using namespace org::antlr::v4::runtime; +using namespace org::antlr::v4::runtime::atn; + +PredicateEvalInfo::PredicateEvalInfo(int decision, TokenStream *input, int startIndex, int stopIndex, + Ref semctx, bool evalResult, int predictedAlt, bool fullCtx) + : DecisionEventInfo(decision, Ref(), input, startIndex, stopIndex, fullCtx), + semctx(semctx), predictedAlt(predictedAlt), evalResult(evalResult) { +} diff --git a/runtime/Cpp/runtime/atn/PredicateEvalInfo.h b/runtime/Cpp/runtime/atn/PredicateEvalInfo.h new file mode 100755 index 000000000..652c752d5 --- /dev/null +++ b/runtime/Cpp/runtime/atn/PredicateEvalInfo.h @@ -0,0 +1,94 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2014 Terence Parr + * Copyright (c) 2014 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "DecisionEventInfo.h" + +namespace org { +namespace antlr { +namespace v4 { +namespace runtime { +namespace atn { + + /// + /// This class represents profiling event information for semantic predicate + /// evaluations which occur during prediction. + /// + /// + class PredicateEvalInfo : public DecisionEventInfo { + public: + /// The semantic context which was evaluated. + const Ref semctx; + + /// + /// The alternative number for the decision which is guarded by the semantic + /// context . Note that other ATN + /// configurations may predict the same alternative which are guarded by + /// other semantic contexts and/or . + /// + const int predictedAlt; + + /// The result of evaluating the semantic context . + const bool evalResult; + + /// + /// Constructs a new instance of the class with the + /// specified detailed predicate evaluation information. + /// + /// The decision number + /// The input token stream + /// The start index for the current prediction + /// The index at which the predicate evaluation was + /// triggered. Note that the input stream may be reset to other positions for + /// the actual evaluation of individual predicates. + /// The semantic context which was evaluated + /// The results of evaluating the semantic context + /// The alternative number for the decision which is + /// guarded by the semantic context {@code semctx}. See + /// for more information. + /// {@code true} if the semantic context was + /// evaluated during LL prediction; otherwise, {@code false} if the semantic + /// context was evaluated during SLL prediction + /// + /// + /// + PredicateEvalInfo(int decision, TokenStream *input, int startIndex, int stopIndex, Ref semctx, + bool evalResult, int predictedAlt, bool fullCtx); + }; + +} // namespace atn +} // namespace runtime +} // namespace v4 +} // namespace antlr +} // namespace org diff --git a/runtime/Cpp/runtime/atn/PredicateTransition.cpp b/runtime/Cpp/runtime/atn/PredicateTransition.cpp index 55c5c39cf..2fec17b4a 100755 --- a/runtime/Cpp/runtime/atn/PredicateTransition.cpp +++ b/runtime/Cpp/runtime/atn/PredicateTransition.cpp @@ -48,7 +48,7 @@ bool PredicateTransition::matches(int symbol, int minVocabSymbol, int maxVocabSy return false; } -std::shared_ptr PredicateTransition::getPredicate() const { +Ref PredicateTransition::getPredicate() const { return std::make_shared(ruleIndex, predIndex, isCtxDependent); } diff --git a/runtime/Cpp/runtime/atn/PredicateTransition.h b/runtime/Cpp/runtime/atn/PredicateTransition.h index 1f03b013f..da8e7e2a4 100755 --- a/runtime/Cpp/runtime/atn/PredicateTransition.h +++ b/runtime/Cpp/runtime/atn/PredicateTransition.h @@ -58,7 +58,7 @@ namespace atn { virtual bool isEpsilon() const override; virtual bool matches(int symbol, int minVocabSymbol, int maxVocabSymbol) const override; - std::shared_ptr getPredicate() const; + Ref getPredicate() const; virtual std::wstring toString() const override; diff --git a/runtime/Cpp/runtime/atn/PredictionContext.cpp b/runtime/Cpp/runtime/atn/PredictionContext.cpp index 23d0078f6..10c855b46 100755 --- a/runtime/Cpp/runtime/atn/PredictionContext.cpp +++ b/runtime/Cpp/runtime/atn/PredictionContext.cpp @@ -47,7 +47,7 @@ using namespace org::antlr::v4::runtime::atn; using namespace antlrcpp; int PredictionContext::globalNodeCount = 0; -const PredictionContext::Ref PredictionContext::EMPTY = std::make_shared(); +const Ref PredictionContext::EMPTY = std::make_shared(); const int PredictionContext::EMPTY_RETURN_STATE; const int PredictionContext::INITIAL_HASH; @@ -57,7 +57,7 @@ PredictionContext::PredictionContext(size_t cachedHashCode) : id(globalNodeCount PredictionContext::~PredictionContext() { } -PredictionContext::Ref PredictionContext::fromRuleContext(const ATN &atn, RuleContext::Ref outerContext) { +Ref PredictionContext::fromRuleContext(const ATN &atn, Ref outerContext) { if (!outerContext) { outerContext = RuleContext::EMPTY; } @@ -69,7 +69,7 @@ PredictionContext::Ref PredictionContext::fromRuleContext(const ATN &atn, RuleCo } // If we have a parent, convert it to a PredictionContext graph - PredictionContext::Ref parent = PredictionContext::fromRuleContext(atn, outerContext->parent.lock()); + Ref parent = PredictionContext::fromRuleContext(atn, outerContext->parent.lock()); ATNState *state = atn.states.at((size_t)outerContext->invokingState); RuleTransition *transition = (RuleTransition *)state->transition(0); @@ -121,8 +121,8 @@ size_t PredictionContext::calculateHashCode(const std::vector PredictionContext::merge(Ref a, + Ref b, bool rootIsWildcard, PredictionContextMergeCache *mergeCache) { assert(a && b); @@ -158,8 +158,8 @@ PredictionContext::Ref PredictionContext::merge(PredictionContext::Ref a, std::dynamic_pointer_cast(b), rootIsWildcard, mergeCache); } -PredictionContext::Ref PredictionContext::mergeSingletons(SingletonPredictionContext::Ref a, - SingletonPredictionContext::Ref b, bool rootIsWildcard, PredictionContextMergeCache *mergeCache) { +Ref PredictionContext::mergeSingletons(Ref a, + Ref b, bool rootIsWildcard, PredictionContextMergeCache *mergeCache) { if (mergeCache != nullptr) { // Can be null if not given to the ATNState from which this call originates. auto iterator = mergeCache->find({ a.get(), b.get() }); @@ -172,7 +172,7 @@ PredictionContext::Ref PredictionContext::mergeSingletons(SingletonPredictionCon } } - PredictionContext::Ref rootMerge = mergeRoot(a, b, rootIsWildcard); + Ref rootMerge = mergeRoot(a, b, rootIsWildcard); if (rootMerge) { if (mergeCache != nullptr) { (*mergeCache)[{ a.get(), b.get() }] = rootMerge; @@ -180,10 +180,10 @@ PredictionContext::Ref PredictionContext::mergeSingletons(SingletonPredictionCon return rootMerge; } - PredictionContext::Ref parentA = a->parent.lock(); - PredictionContext::Ref parentB = b->parent.lock(); + Ref parentA = a->parent.lock(); + Ref parentB = b->parent.lock(); if (a->returnState == b->returnState) { // a == b - PredictionContext::Ref parent = merge(parentA, parentB, rootIsWildcard, mergeCache); + Ref parent = merge(parentA, parentB, rootIsWildcard, mergeCache); // If parent is same as existing a or b parent or reduced to a parent, return it. if (parent == parentA) { // ax + bx = ax, if a=b @@ -197,7 +197,7 @@ PredictionContext::Ref PredictionContext::mergeSingletons(SingletonPredictionCon // merge parents x and y, giving array node with x,y then remainders // of those graphs. dup a, a' points at merged array // new joined parent so create new singleton pointing to it, a' - PredictionContext::Ref a_ = SingletonPredictionContext::create(parent, a->returnState); + Ref a_ = SingletonPredictionContext::create(parent, a->returnState); if (mergeCache != nullptr) { (*mergeCache)[{ a.get(), b.get() }] = a_; } @@ -216,7 +216,7 @@ PredictionContext::Ref PredictionContext::mergeSingletons(SingletonPredictionCon payloads[1] = a->returnState; } std::vector> parents = { singleParent, singleParent }; - PredictionContext::Ref a_ = std::make_shared(parents, payloads); + Ref a_ = std::make_shared(parents, payloads); if (mergeCache != nullptr) { (*mergeCache)[{ a.get(), b.get() }] = a_; } @@ -226,7 +226,7 @@ PredictionContext::Ref PredictionContext::mergeSingletons(SingletonPredictionCon // parents differ and can't merge them. Just pack together // into array; can't merge. // ax + by = [ax,by] - PredictionContext::Ref a_; + Ref a_; if (a->returnState > b->returnState) { // sort by payload std::vector payloads = { b->returnState, a->returnState }; std::vector> parents = { b->parent, a->parent }; @@ -244,8 +244,8 @@ PredictionContext::Ref PredictionContext::mergeSingletons(SingletonPredictionCon } } -PredictionContext::Ref PredictionContext::mergeRoot(SingletonPredictionContext::Ref a, SingletonPredictionContext::Ref b, - bool rootIsWildcard) { +Ref PredictionContext::mergeRoot(Ref a, Ref b, + bool rootIsWildcard) { if (rootIsWildcard) { if (a == EMPTY) { // * + b = * return EMPTY; @@ -260,21 +260,21 @@ PredictionContext::Ref PredictionContext::mergeRoot(SingletonPredictionContext:: if (a == EMPTY) { // $ + x = [$,x] std::vector payloads = { b->returnState, EMPTY_RETURN_STATE }; std::vector> parents = { b->parent, EMPTY }; - PredictionContext::Ref joined = std::make_shared(parents, payloads); + Ref joined = std::make_shared(parents, payloads); return joined; } if (b == EMPTY) { // x + $ = [$,x] ($ is always first if present) std::vector payloads = { a->returnState, EMPTY_RETURN_STATE }; std::vector> parents = { a->parent, EMPTY }; - PredictionContext::Ref joined = std::make_shared(parents, payloads); + Ref joined = std::make_shared(parents, payloads); return joined; } } return nullptr; } -PredictionContext::Ref PredictionContext::mergeArrays(std::shared_ptr a, - std::shared_ptr b, bool rootIsWildcard, PredictionContextMergeCache *mergeCache) { +Ref PredictionContext::mergeArrays(Ref a, + Ref b, bool rootIsWildcard, PredictionContextMergeCache *mergeCache) { if (mergeCache != nullptr) { auto iterator = mergeCache->find({ a.get(), b.get() }); @@ -297,8 +297,8 @@ PredictionContext::Ref PredictionContext::mergeArrays(std::shared_ptrreturnStates.size() && j < b->returnStates.size()) { - PredictionContext::Ref a_parent = a->parents[i].lock(); - PredictionContext::Ref b_parent = b->parents[j].lock(); + Ref a_parent = a->parents[i].lock(); + Ref b_parent = b->parents[j].lock(); if (a->returnStates[i] == b->returnStates[j]) { // same payload (stack tops are equal), must yield merged singleton int payload = a->returnStates[i]; @@ -310,7 +310,7 @@ PredictionContext::Ref PredictionContext::mergeArrays(std::shared_ptr a'[x,y] - PredictionContext::Ref mergedParent = merge(a_parent, b_parent, rootIsWildcard, mergeCache); + Ref mergedParent = merge(a_parent, b_parent, rootIsWildcard, mergeCache); mergedParents[k] = mergedParent; mergedReturnStates[k] = payload; } @@ -347,7 +347,7 @@ PredictionContext::Ref PredictionContext::mergeArrays(std::shared_ptr a_ = SingletonPredictionContext::create(mergedParents[0].lock(), mergedReturnStates[0]); if (mergeCache != nullptr) { (*mergeCache)[{ a.get(), b.get() }] = a_; } @@ -357,7 +357,7 @@ PredictionContext::Ref PredictionContext::mergeArrays(std::shared_ptr M = std::make_shared(mergedParents, mergedReturnStates); + Ref M = std::make_shared(mergedParents, mergedReturnStates); // if we created same array as a or b, return that instead // TO_DO: track whether this is possible above during merge sort for speed @@ -384,10 +384,10 @@ PredictionContext::Ref PredictionContext::mergeArrays(std::shared_ptr> &parents) { - std::set uniqueParents; + std::set> uniqueParents; for (size_t p = 0; p < parents.size(); p++) { - PredictionContext::Ref parent = parents[p].lock(); + Ref parent = parents[p].lock(); // ml: it's assumed that the == operator of PredictionContext kicks in here. if (uniqueParents.find(parent) == uniqueParents.end()) { // don't replace uniqueParents.insert(parent); @@ -402,7 +402,7 @@ bool PredictionContext::combineCommonParents(std::vector context) { if (context == nullptr) { return L""; } @@ -410,8 +410,8 @@ std::wstring PredictionContext::toDOTString(PredictionContext::Ref context) { std::wstringstream ss; ss << L"digraph G {\n" << L"rankdir=LR;\n"; - std::vector nodes = getAllContextNodes(context); - std::sort(nodes.begin(), nodes.end(), [](PredictionContext::Ref o1, PredictionContext::Ref o2) { + std::vector> nodes = getAllContextNodes(context); + std::sort(nodes.begin(), nodes.end(), [](Ref o1, Ref o2) { return o1->id - o2->id; }); @@ -426,7 +426,7 @@ std::wstring PredictionContext::toDOTString(PredictionContext::Ref context) { ss << L" [label=\"" << returnState << L"\"];\n"; continue; } - std::shared_ptr arr = std::static_pointer_cast(current); + Ref arr = std::static_pointer_cast(current); ss << L" s" << arr->id << L" [shape=box, label=\"" << L"["; bool first = true; for (auto inv : arr->returnStates) { @@ -466,8 +466,8 @@ std::wstring PredictionContext::toDOTString(PredictionContext::Ref context) { } // The "visited" map is just a temporary structure to control the retrieval process (which is recursive). -PredictionContext::Ref PredictionContext::getCachedContext(PredictionContext::Ref context, - std::shared_ptr contextCache, std::map &visited) { +Ref PredictionContext::getCachedContext(Ref context, + Ref contextCache, std::map, Ref> &visited) { if (context->isEmpty()) { return context; } @@ -511,7 +511,7 @@ PredictionContext::Ref PredictionContext::getCachedContext(PredictionContext::Re return context; } - PredictionContext::Ref updated; + Ref updated; if (parents.empty()) { updated = EMPTY; } else if (parents.size() == 1) { @@ -527,17 +527,16 @@ PredictionContext::Ref PredictionContext::getCachedContext(PredictionContext::Re return updated; } -std::vector PredictionContext::getAllContextNodes(PredictionContext::Ref context) { - std::vector nodes; - std::map visited; +std::vector> PredictionContext::getAllContextNodes(Ref context) { + std::vector> nodes; + std::map, Ref> visited; getAllContextNodes_(context, nodes, visited); return nodes; } -void PredictionContext::getAllContextNodes_(PredictionContext::Ref context, - std::vector &nodes, - std::map &visited) { +void PredictionContext::getAllContextNodes_(Ref context, std::vector> &nodes, + std::map, Ref> &visited) { if (visited.find(context) != visited.end()) { return; // Already done. @@ -564,7 +563,7 @@ std::vector PredictionContext::toStrings(Recognizer *recognizer, i return toStrings(recognizer, EMPTY, currentState); } -std::vector PredictionContext::toStrings(Recognizer *recognizer, PredictionContext::Ref stop, int currentState) { +std::vector PredictionContext::toStrings(Recognizer *recognizer, Ref stop, int currentState) { std::vector result; diff --git a/runtime/Cpp/runtime/atn/PredictionContext.h b/runtime/Cpp/runtime/atn/PredictionContext.h index 711173dce..7c9a2e423 100755 --- a/runtime/Cpp/runtime/atn/PredictionContext.h +++ b/runtime/Cpp/runtime/atn/PredictionContext.h @@ -41,19 +41,17 @@ namespace v4 { namespace runtime { namespace atn { - // Cannot use PredictionContext::Ref here as this declared below first. - typedef std::set> PredictionContextCache; + // Cannot use PredictionContext> here as this declared below first. + typedef std::set> PredictionContextCache; // For the keys we use raw pointers, as we don't need to access them. - typedef std::map, std::shared_ptr> PredictionContextMergeCache; + typedef std::map, Ref> PredictionContextMergeCache; class PredictionContext { public: - typedef std::shared_ptr Ref; - /// Represents $ in local context prediction, which means wildcard. /// *+x = *. - static const Ref EMPTY; + static const Ref EMPTY; /// Represents $ in an array in full context mode, when $ /// doesn't mean wildcard: $ + x = [$,x]. Here, @@ -97,7 +95,7 @@ namespace atn { public: /// Convert a RuleContext tree to a PredictionContext graph. /// Return EMPTY if outerContext is empty. - static PredictionContext::Ref fromRuleContext(const ATN &atn, RuleContext::Ref outerContext); + static Ref fromRuleContext(const ATN &atn, Ref outerContext); virtual size_t size() const = 0; virtual std::weak_ptr getParent(size_t index) const = 0; @@ -118,8 +116,8 @@ namespace atn { public: // dispatch - static PredictionContext::Ref merge(PredictionContext::Ref a, - PredictionContext::Ref b, bool rootIsWildcard, PredictionContextMergeCache *mergeCache); + static Ref merge(Ref a, + Ref b, bool rootIsWildcard, PredictionContextMergeCache *mergeCache); /// /// Merge two instances. @@ -155,8 +153,8 @@ namespace atn { /// {@code true} if this is a local-context merge, /// otherwise false to indicate a full-context merge /// - static PredictionContext::Ref mergeSingletons(std::shared_ptr a, - std::shared_ptr b, bool rootIsWildcard, PredictionContextMergeCache *mergeCache); + static Ref mergeSingletons(Ref a, + Ref b, bool rootIsWildcard, PredictionContextMergeCache *mergeCache); /// /// Handle case where at least one of {@code a} or {@code b} is @@ -207,8 +205,8 @@ namespace atn { /// the second /// {@code true} if this is a local-context merge, /// otherwise false to indicate a full-context merge - static PredictionContext::Ref mergeRoot(std::shared_ptr a, - std::shared_ptr b, bool rootIsWildcard); + static Ref mergeRoot(Ref a, + Ref b, bool rootIsWildcard); /// /// Merge two instances. @@ -239,8 +237,8 @@ namespace atn { /// .
/// /// - static PredictionContext::Ref mergeArrays(std::shared_ptr a, - std::shared_ptr b, bool rootIsWildcard, PredictionContextMergeCache *mergeCache); + static Ref mergeArrays(Ref a, + Ref b, bool rootIsWildcard, PredictionContextMergeCache *mergeCache); protected: /// Make pass over all M parents; merge any equal() ones. @@ -248,22 +246,22 @@ namespace atn { static bool combineCommonParents(std::vector> &parents); public: - static std::wstring toDOTString(PredictionContext::Ref context); + static std::wstring toDOTString(Ref context); - static PredictionContext::Ref getCachedContext(PredictionContext::Ref context, - std::shared_ptr contextCache, - std::map &visited); + static Ref getCachedContext(Ref context, + Ref contextCache, + std::map, Ref> &visited); // ter's recursive version of Sam's getAllNodes() - static std::vector getAllContextNodes(PredictionContext::Ref context); - static void getAllContextNodes_(PredictionContext::Ref context, - std::vector &nodes, std::map &visited); + static std::vector> getAllContextNodes(Ref context); + static void getAllContextNodes_(Ref context, + std::vector> &nodes, std::map, Ref> &visited); virtual std::wstring toString() const; virtual std::wstring toString(Recognizer *recog) const; std::vector toStrings(Recognizer *recognizer, int currentState); - std::vector toStrings(Recognizer *recognizer, PredictionContext::Ref stop, int currentState); + std::vector toStrings(Recognizer *recognizer, Ref stop, int currentState); }; } // namespace atn diff --git a/runtime/Cpp/runtime/atn/PredictionMode.cpp b/runtime/Cpp/runtime/atn/PredictionMode.cpp index 540b4d896..63916c935 100755 --- a/runtime/Cpp/runtime/atn/PredictionMode.cpp +++ b/runtime/Cpp/runtime/atn/PredictionMode.cpp @@ -33,6 +33,7 @@ #include "ATNConfigSet.h" #include "ATNConfig.h" #include "MurmurHash.h" +#include "SemanticContext.h" #include "PredictionMode.h" @@ -60,7 +61,7 @@ struct AltAndContextConfigComparer { } }; -bool PredictionModeClass::hasSLLConflictTerminatingPrediction(PredictionMode* mode, std::shared_ptr configs) { +bool PredictionModeClass::hasSLLConflictTerminatingPrediction(PredictionMode* mode, Ref configs) { /* Configs in rule stop states indicate reaching the end of the decision * rule (local context) or end of start rule (full context). If all * configs meet this condition, then none of the configurations is able @@ -77,9 +78,9 @@ bool PredictionModeClass::hasSLLConflictTerminatingPrediction(PredictionMode* mo // since we'll often fail over anyway. if (configs->hasSemanticContext) { // dup configs, tossing out semantic predicates - std::shared_ptr dup = std::make_shared(true); + Ref dup = std::make_shared(true); for (auto config : configs->configs) { - ATNConfig::Ref c = std::make_shared(config, SemanticContext::NONE); + Ref c = std::make_shared(config, SemanticContext::NONE); dup->add(c); } configs = dup; @@ -94,9 +95,9 @@ bool PredictionModeClass::hasSLLConflictTerminatingPrediction(PredictionMode* mo return heuristic; } -bool PredictionModeClass::hasConfigInRuleStopState(std::shared_ptr configs) { +bool PredictionModeClass::hasConfigInRuleStopState(Ref configs) { for (auto c : configs->configs) { - if (is(c->state)) { + if (is(c->state)) { return true; } } @@ -104,7 +105,7 @@ bool PredictionModeClass::hasConfigInRuleStopState(std::shared_ptr return false; } -bool PredictionModeClass::allConfigsInRuleStopStates(std::shared_ptr configs) { +bool PredictionModeClass::allConfigsInRuleStopStates(Ref configs) { for (auto config : configs->configs) { if (!is(config->state)) { return false; @@ -171,8 +172,8 @@ antlrcpp::BitSet PredictionModeClass::getAlts(const std::vector PredictionModeClass::getConflictingAltSubsets(std::shared_ptr configs) { - std::unordered_map configToAlts; +std::vector PredictionModeClass::getConflictingAltSubsets(Ref configs) { + std::unordered_map, antlrcpp::BitSet, AltAndContextConfigHasher, AltAndContextConfigComparer> configToAlts; for (auto config : configs->configs) { configToAlts[config].set((size_t)config->alt); } @@ -183,7 +184,7 @@ std::vector PredictionModeClass::getConflictingAltSubsets(std: return values; } -std::map PredictionModeClass::getStateToAltMap(std::shared_ptr configs) { +std::map PredictionModeClass::getStateToAltMap(Ref configs) { std::map m; for (auto c : configs->configs) { m[c->state].set((size_t)c->alt); @@ -191,7 +192,7 @@ std::map PredictionModeClass::getStateToAltMap(std: return m; } -bool PredictionModeClass::hasStateAssociatedWithOneAlt(std::shared_ptr configs) { +bool PredictionModeClass::hasStateAssociatedWithOneAlt(Ref configs) { std::map x = getStateToAltMap(configs); for (std::map::iterator it = x.begin(); it != x.end(); it++){ if (it->second.count() == 1) return true; diff --git a/runtime/Cpp/runtime/atn/PredictionMode.h b/runtime/Cpp/runtime/atn/PredictionMode.h index 1df81c27d..a959cde9e 100755 --- a/runtime/Cpp/runtime/atn/PredictionMode.h +++ b/runtime/Cpp/runtime/atn/PredictionMode.h @@ -204,7 +204,7 @@ namespace atn { /// the configurations to strip out all of the predicates so that a standard /// will merge everything ignoring predicates. /// - static bool hasSLLConflictTerminatingPrediction(PredictionMode *mode, std::shared_ptr configs); + static bool hasSLLConflictTerminatingPrediction(PredictionMode *mode, Ref configs); /// /// Checks if any configuration in {@code configs} is in a @@ -216,7 +216,7 @@ namespace atn { /// the configuration set to test /// {@code true} if any configuration in {@code configs} is in a /// , otherwise {@code false} - static bool hasConfigInRuleStopState(std::shared_ptr configs); + static bool hasConfigInRuleStopState(Ref configs); /// /// Checks if all configurations in {@code configs} are in a @@ -228,7 +228,7 @@ namespace atn { /// the configuration set to test /// {@code true} if all configurations in {@code configs} are in a /// , otherwise {@code false} - static bool allConfigsInRuleStopStates(std::shared_ptr configs); + static bool allConfigsInRuleStopStates(Ref configs); /// /// Full LL prediction termination. @@ -485,7 +485,7 @@ namespace atn { /// alt and not pred /// /// - static std::vector getConflictingAltSubsets(std::shared_ptr configs); + static std::vector getConflictingAltSubsets(Ref configs); /// /// Get a map from state to alt subset from a configuration set. For each @@ -496,9 +496,9 @@ namespace atn { /// cref="ATNConfig#alt alt"/> /// /// - static std::map getStateToAltMap(std::shared_ptr configs); + static std::map getStateToAltMap(Ref configs); - static bool hasStateAssociatedWithOneAlt(std::shared_ptr configs); + static bool hasStateAssociatedWithOneAlt(Ref configs); static int getSingleViableAlt(const std::vector &altsets); }; diff --git a/runtime/Cpp/runtime/atn/ProfilingATNSimulator.cpp b/runtime/Cpp/runtime/atn/ProfilingATNSimulator.cpp new file mode 100755 index 000000000..8d6996d28 --- /dev/null +++ b/runtime/Cpp/runtime/atn/ProfilingATNSimulator.cpp @@ -0,0 +1,204 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2013 Terence Parr + * Copyright (c) 2013 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "PredicateEvalInfo.h" +#include "LookaheadEventInfo.h" +#include "Parser.h" +#include "ATNConfigSet.h" +#include "CPPUtils.h" + +#include "ProfilingATNSimulator.h" + +using namespace org::antlr::v4::runtime::atn; +using namespace org::antlr::v4::runtime::dfa; +using namespace antlrcpp; + +using namespace std::chrono; + +ProfilingATNSimulator::ProfilingATNSimulator(Parser *parser) + : ParserATNSimulator(parser, parser->getInterpreter()->atn, + parser->getInterpreter()->decisionToDFA, + parser->getInterpreter()->getSharedContextCache()) { + for (size_t i = 0; i < atn.decisionToState.size(); i++) { + _decisions.push_back(DecisionInfo(i)); + } +} + +int ProfilingATNSimulator::adaptivePredict(TokenStream *input, int decision, Ref outerContext) { + auto onExit = finally([this](){ + _currentDecision = -1; + }); + + _sllStopIndex = -1; + _llStopIndex = -1; + _currentDecision = decision; + high_resolution_clock::time_point start = high_resolution_clock::now(); // expensive but useful info + int alt = ParserATNSimulator::adaptivePredict(input, decision, outerContext); + high_resolution_clock::time_point stop = high_resolution_clock::now(); + _decisions[decision].timeInPrediction += duration_cast(stop - start).count(); + _decisions[decision].invocations++; + + long long SLL_k = _sllStopIndex - _startIndex + 1; + _decisions[decision].SLL_TotalLook += SLL_k; + _decisions[decision].SLL_MinLook = _decisions[decision].SLL_MinLook == 0 ? SLL_k : std::min(_decisions[decision].SLL_MinLook, SLL_k); + if (SLL_k > _decisions[decision].SLL_MaxLook) { + _decisions[decision].SLL_MaxLook = SLL_k; + _decisions[decision].SLL_MaxLookEvent = std::make_shared(decision, nullptr, alt, input, _startIndex, _sllStopIndex, false); + } + + if (_llStopIndex >= 0) { + long long LL_k = _llStopIndex - _startIndex + 1; + _decisions[decision].LL_TotalLook += LL_k; + _decisions[decision].LL_MinLook = _decisions[decision].LL_MinLook == 0 ? LL_k : std::min(_decisions[decision].LL_MinLook, LL_k); + if (LL_k > _decisions[decision].LL_MaxLook) { + _decisions[decision].LL_MaxLook = LL_k; + _decisions[decision].LL_MaxLookEvent = std::make_shared(decision, nullptr, alt, input, _startIndex, _llStopIndex, true); + } + } + + return alt; +} + +DFAState* ProfilingATNSimulator::getExistingTargetState(DFAState *previousD, ssize_t t) { + // this method is called after each time the input position advances + // during SLL prediction + _sllStopIndex = (int)_input->index(); + + DFAState *existingTargetState = ParserATNSimulator::getExistingTargetState(previousD, t); + if (existingTargetState != nullptr) { + _decisions[_currentDecision].SLL_DFATransitions++; // count only if we transition over a DFA state + if (existingTargetState == ERROR.get()) { + _decisions[_currentDecision].errors.push_back( + ErrorInfo(_currentDecision, previousD->configs, _input, _startIndex, _sllStopIndex, false) + ); + } + } + + _currentState = existingTargetState; + return existingTargetState; +} + +DFAState* ProfilingATNSimulator::computeTargetState(DFA &dfa, DFAState *previousD, ssize_t t) { + DFAState *state = ParserATNSimulator::computeTargetState(dfa, previousD, t); + _currentState = state; + return state; +} + +Ref ProfilingATNSimulator::computeReachSet(Ref closure, ssize_t t, bool fullCtx) { + if (fullCtx) { + // this method is called after each time the input position advances + // during full context prediction + _llStopIndex = (int)_input->index(); + } + + Ref reachConfigs = ParserATNSimulator::computeReachSet(closure, t, fullCtx); + if (fullCtx) { + _decisions[_currentDecision].LL_ATNTransitions++; // count computation even if error + if (reachConfigs != nullptr) { + } else { // no reach on current lookahead symbol. ERROR. + // TODO: does not handle delayed errors per getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule() + _decisions[_currentDecision].errors.push_back(ErrorInfo(_currentDecision, closure, _input, _startIndex, _llStopIndex, true)); + } + } else { + ++_decisions[_currentDecision].SLL_ATNTransitions; + if (reachConfigs != nullptr) { + } else { // no reach on current lookahead symbol. ERROR. + _decisions[_currentDecision].errors.push_back(ErrorInfo(_currentDecision, closure, _input, _startIndex, _sllStopIndex, false)); + } + } + return reachConfigs; +} + +bool ProfilingATNSimulator::evalSemanticContext(Ref pred, Ref parserCallStack, + int alt, bool fullCtx) { + bool result = ParserATNSimulator::evalSemanticContext(pred, parserCallStack, alt, fullCtx); + if (!(std::dynamic_pointer_cast(pred) != nullptr)) { + bool fullContext = _llStopIndex >= 0; + int stopIndex = fullContext ? _llStopIndex : _sllStopIndex; + _decisions[_currentDecision].predicateEvals.push_back( + PredicateEvalInfo(_currentDecision, _input, _startIndex, stopIndex, pred, result, alt, fullCtx)); + } + + return result; +} + +void ProfilingATNSimulator::reportAttemptingFullContext(DFA &dfa, const BitSet &conflictingAlts, Ref configs, + size_t startIndex, size_t stopIndex) { + if (conflictingAlts.count() > 0) { + conflictingAltResolvedBySLL = conflictingAlts.nextSetBit(0); + } else { + conflictingAltResolvedBySLL = configs->getAlts().nextSetBit(0); + } + _decisions[_currentDecision].LL_Fallback++; + ParserATNSimulator::reportAttemptingFullContext(dfa, conflictingAlts, configs, startIndex, stopIndex); +} + +void ProfilingATNSimulator::reportContextSensitivity(DFA &dfa, int prediction, Ref configs, + size_t startIndex, size_t stopIndex) { + if (prediction != conflictingAltResolvedBySLL) { + _decisions[_currentDecision].contextSensitivities.push_back( + ContextSensitivityInfo(_currentDecision, configs, _input, startIndex, stopIndex) + ); + } + ParserATNSimulator::reportContextSensitivity(dfa, prediction, configs, startIndex, stopIndex); +} + +void ProfilingATNSimulator::reportAmbiguity(DFA &dfa, DFAState *D, size_t startIndex, size_t stopIndex, bool exact, + const BitSet &ambigAlts, Ref configs) { + int prediction; + if (ambigAlts.count() > 0) { + prediction = ambigAlts.nextSetBit(0); + } else { + prediction = configs->getAlts().nextSetBit(0); + } + if (configs->fullCtx && prediction != conflictingAltResolvedBySLL) { + // Even though this is an ambiguity we are reporting, we can + // still detect some context sensitivities. Both SLL and LL + // are showing a conflict, hence an ambiguity, but if they resolve + // to different minimum alternatives we have also identified a + // context sensitivity. + _decisions[_currentDecision].contextSensitivities.push_back( + ContextSensitivityInfo(_currentDecision, configs, _input, startIndex, stopIndex) + ); + } + _decisions[_currentDecision].ambiguities.push_back( + AmbiguityInfo(_currentDecision, configs, ambigAlts, _input, startIndex, stopIndex, configs->fullCtx) + ); + ParserATNSimulator::reportAmbiguity(dfa, D, startIndex, stopIndex, exact, ambigAlts, configs); +} + +std::vector ProfilingATNSimulator::getDecisionInfo() const { + return _decisions; +} + +DFAState* ProfilingATNSimulator::getCurrentState() const { + return _currentState; +} diff --git a/runtime/Cpp/runtime/atn/ProfilingATNSimulator.h b/runtime/Cpp/runtime/atn/ProfilingATNSimulator.h new file mode 100755 index 000000000..88d425768 --- /dev/null +++ b/runtime/Cpp/runtime/atn/ProfilingATNSimulator.h @@ -0,0 +1,93 @@ +/* + * [The "BSD license"] + * Copyright (c) 2016 Mike Lischke + * Copyright (c) 2013 Terence Parr + * Copyright (c) 2013 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "ParserATNSimulator.h" +#include "DecisionInfo.h" + +namespace org { +namespace antlr { +namespace v4 { +namespace runtime { +namespace atn { + + class ProfilingATNSimulator : public ParserATNSimulator { + public: + ProfilingATNSimulator(Parser *parser); + + virtual int adaptivePredict(TokenStream *input, int decision, Ref outerContext) override; + + virtual std::vector getDecisionInfo() const; + virtual dfa::DFAState* getCurrentState() const; + + protected: + std::vector _decisions; + int _numDecisions = 0; + + int _sllStopIndex = 0; + int _llStopIndex = 0; + + int _currentDecision = 0; + dfa::DFAState *_currentState; + + /// + /// At the point of LL failover, we record how SLL would resolve the conflict so that + /// we can determine whether or not a decision / input pair is context-sensitive. + /// If LL gives a different result than SLL's predicted alternative, we have a + /// context sensitivity for sure. The converse is not necessarily true, however. + /// It's possible that after conflict resolution chooses minimum alternatives, + /// SLL could get the same answer as LL. Regardless of whether or not the result indicates + /// an ambiguity, it is not treated as a context sensitivity because LL prediction + /// was not required in order to produce a correct prediction for this decision and input sequence. + /// It may in fact still be a context sensitivity but we don't know by looking at the + /// minimum alternatives for the current input. + /// + int conflictingAltResolvedBySLL = 0; + + virtual dfa::DFAState* getExistingTargetState(dfa::DFAState *previousD, ssize_t t) override; + virtual dfa::DFAState* computeTargetState(dfa::DFA &dfa, dfa::DFAState *previousD, ssize_t t) override; + virtual Ref computeReachSet(Ref closure, ssize_t t, bool fullCtx) override; + virtual bool evalSemanticContext(Ref pred, Ref parserCallStack, + int alt, bool fullCtx) override; + virtual void reportAttemptingFullContext(dfa::DFA &dfa, const antlrcpp::BitSet &conflictingAlts, Ref configs, + size_t startIndex, size_t stopIndex) override; + virtual void reportContextSensitivity(dfa::DFA &dfa, int prediction, Ref configs, + size_t startIndex, size_t stopIndex) override; + virtual void reportAmbiguity(dfa::DFA &dfa, dfa::DFAState *D, size_t startIndex, size_t stopIndex, bool exact, + const antlrcpp::BitSet &ambigAlts, Ref configs) override; + }; + +} // namespace atn +} // namespace runtime +} // namespace v4 +} // namespace antlr +} // namespace org diff --git a/runtime/Cpp/runtime/atn/SemanticContext.cpp b/runtime/Cpp/runtime/atn/SemanticContext.cpp index bc6ed7e5c..c96b92188 100755 --- a/runtime/Cpp/runtime/atn/SemanticContext.cpp +++ b/runtime/Cpp/runtime/atn/SemanticContext.cpp @@ -38,14 +38,24 @@ using namespace org::antlr::v4::runtime; using namespace org::antlr::v4::runtime::atn; using namespace antlrcpp; +//------------------ Predicate ----------------------------------------------------------------------------------------- + SemanticContext::Predicate::Predicate() : Predicate(-1, -1, false) { } SemanticContext::Predicate::Predicate(int ruleIndex, int predIndex, bool isCtxDependent) - : ruleIndex(ruleIndex), predIndex(predIndex), isCtxDependent(isCtxDependent) { +: ruleIndex(ruleIndex), predIndex(predIndex), isCtxDependent(isCtxDependent) { } -size_t SemanticContext::Predicate::hashCode() { + +bool SemanticContext::Predicate::eval(Recognizer *parser, Ref parserCallStack) { + Ref localctx; + if (isCtxDependent) + localctx = parserCallStack; + return parser->sempred(localctx, ruleIndex, predIndex); +} + +size_t SemanticContext::Predicate::hashCode() const { size_t hashCode = misc::MurmurHash::initialize(); hashCode = misc::MurmurHash::update(hashCode, (size_t)ruleIndex); hashCode = misc::MurmurHash::update(hashCode, (size_t)predIndex); @@ -73,17 +83,32 @@ std::wstring SemanticContext::Predicate::toString() const { return std::wstring(L"{") + std::to_wstring(ruleIndex) + std::wstring(L":") + std::to_wstring(predIndex) + std::wstring(L"}?"); } +//------------------ PrecedencePredicate ------------------------------------------------------------------------------- + SemanticContext::PrecedencePredicate::PrecedencePredicate() : precedence(0) { } SemanticContext::PrecedencePredicate::PrecedencePredicate(int precedence) : precedence(precedence) { } +bool SemanticContext::PrecedencePredicate::eval(Recognizer *parser, Ref parserCallStack) { + return parser->precpred(parserCallStack, precedence); +} + +Ref SemanticContext::PrecedencePredicate::evalPrecedence(Recognizer *parser, Ref parserCallStack) { + if (parser->precpred(parserCallStack, precedence)) { + return SemanticContext::NONE; + } + else { + return nullptr; + } +} + int SemanticContext::PrecedencePredicate::compareTo(PrecedencePredicate *o) { return precedence - o->precedence; } -size_t SemanticContext::PrecedencePredicate::hashCode() { +size_t SemanticContext::PrecedencePredicate::hashCode() const { size_t hashCode = 1; hashCode = 31 * hashCode + (size_t)precedence; return hashCode; @@ -102,14 +127,14 @@ bool SemanticContext::PrecedencePredicate::operator == (const SemanticContext &o } std::wstring SemanticContext::PrecedencePredicate::toString() const { - return L"Precedence: " + std::to_wstring(precedence); + return L"{" + std::to_wstring(precedence) + L">=prec}?"; } +//------------------ AND ----------------------------------------------------------------------------------------------- -SemanticContext::AND::AND(SemanticContext::Ref a, SemanticContext::Ref b) { - +SemanticContext::AND::AND(Ref a, Ref b) { if (is(a)) { - const std::vector op = ((AND*)a.get())->opnds; + const std::vector> op = ((AND*)a.get())->opnds; for (auto var : op) { opnds.push_back(var); } @@ -118,7 +143,7 @@ SemanticContext::AND::AND(SemanticContext::Ref a, SemanticContext::Ref b) { } if (is(b)) { - const std::vector op = ((AND*)b.get())->opnds; + const std::vector> op = ((AND*)b.get())->opnds; for (auto var : op) { opnds.push_back(var); } @@ -126,11 +151,11 @@ SemanticContext::AND::AND(SemanticContext::Ref a, SemanticContext::Ref b) { opnds.push_back(b); } - std::vector> precedencePredicates = filterPrecedencePredicates(opnds); + std::vector> precedencePredicates = filterPrecedencePredicates(opnds); if (!precedencePredicates.empty()) { // interested in the transition with the lowest precedence - auto predicate = [](std::shared_ptr a, std::shared_ptr b) { + auto predicate = [](Ref a, Ref b) { return a->precedence < b->precedence; }; auto reduced = std::min_element(precedencePredicates.begin(), precedencePredicates.end(), predicate); @@ -139,6 +164,10 @@ SemanticContext::AND::AND(SemanticContext::Ref a, SemanticContext::Ref b) { } +std::vector> SemanticContext::AND::getOperands() const { + return opnds; +} + bool SemanticContext::AND::operator == (const SemanticContext &other) const { const AND *context = dynamic_cast(&other); if (context == nullptr) @@ -151,12 +180,50 @@ bool SemanticContext::AND::operator == (const SemanticContext &other) const { return opnds == context->opnds; } - -size_t SemanticContext::AND::hashCode() { +size_t SemanticContext::AND::hashCode() const { return misc::MurmurHash::hashCode(opnds, typeid(AND).hash_code()); } +bool SemanticContext::AND::eval(Recognizer *parser, Ref parserCallStack) { + for (auto opnd : opnds) { + if (!opnd->eval(parser, parserCallStack)) { + return false; + } + } + return true; +} +Ref SemanticContext::AND::evalPrecedence(Recognizer *parser, Ref parserCallStack) { + bool differs = false; + std::vector> operands; + for (auto context : opnds) { + Ref evaluated = context->evalPrecedence(parser, parserCallStack); + differs |= (evaluated != context); + if (evaluated == nullptr) { + // The AND context is false if any element is false. + return nullptr; + } else if (evaluated != NONE) { + // Reduce the result by skipping true elements. + operands.push_back(evaluated); + } + } + + if (!differs) { + return shared_from_this(); + } + + if (operands.empty()) { + // All elements were true, so the AND context is true. + return NONE; + } + + Ref result = operands[0]; + for (size_t i = 1; i < operands.size(); ++i) { + result = SemanticContext::And(result, operands[i]); + } + + return result; +} std::wstring SemanticContext::AND::toString() const { std::wstring tmp; @@ -166,10 +233,11 @@ std::wstring SemanticContext::AND::toString() const { return tmp; } -SemanticContext::OR::OR(SemanticContext::Ref a, SemanticContext::Ref b){ +//------------------ OR ------------------------------------------------------------------------------------------------ +SemanticContext::OR::OR(Ref a, Ref b) { if (is(a)) { - const std::vector op = ((OR*)a.get())->opnds; + const std::vector> op = ((OR*)a.get())->opnds; for (auto var : op) { opnds.push_back(var); } @@ -178,7 +246,7 @@ SemanticContext::OR::OR(SemanticContext::Ref a, SemanticContext::Ref b){ } if (is(b)) { - const std::vector op = ((OR*)b.get())->opnds; + const std::vector> op = ((OR*)b.get())->opnds; for (auto var : op) { opnds.push_back(var); } @@ -186,10 +254,10 @@ SemanticContext::OR::OR(SemanticContext::Ref a, SemanticContext::Ref b){ opnds.push_back(b); } - std::vector> precedencePredicates = filterPrecedencePredicates(opnds); + std::vector> precedencePredicates = filterPrecedencePredicates(opnds); if (!precedencePredicates.empty()) { // interested in the transition with the highest precedence - auto predicate = [](std::shared_ptr a, std::shared_ptr b) { + auto predicate = [](Ref a, Ref b) { return a->precedence > b->precedence; }; auto reduced = std::min_element(precedencePredicates.begin(), precedencePredicates.end(), predicate); @@ -197,6 +265,10 @@ SemanticContext::OR::OR(SemanticContext::Ref a, SemanticContext::Ref b){ } } +std::vector> SemanticContext::OR::getOperands() const { + return opnds; +} + bool SemanticContext::OR::operator == (const SemanticContext &other) const { const OR *context = dynamic_cast(&other); if (context == nullptr) @@ -209,19 +281,51 @@ bool SemanticContext::OR::operator == (const SemanticContext &other) const { return opnds == context->opnds; } -size_t SemanticContext::OR::hashCode() { +size_t SemanticContext::OR::hashCode() const { return misc::MurmurHash::hashCode(opnds, typeid(OR).hash_code()); } -bool SemanticContext::OR::eval(Recognizer *parser, RuleContext::Ref outerContext) { +bool SemanticContext::OR::eval(Recognizer *parser, Ref parserCallStack) { for (auto opnd : opnds) { - if (opnd->eval(parser, outerContext)) { + if (opnd->eval(parser, parserCallStack)) { return true; } } return false; } +Ref SemanticContext::OR::evalPrecedence(Recognizer *parser, Ref parserCallStack) { + bool differs = false; + std::vector> operands; + for (auto context : opnds) { + Ref evaluated = context->evalPrecedence(parser, parserCallStack); + differs |= (evaluated != context); + if (evaluated == NONE) { + // The OR context is true if any element is true. + return NONE; + } else if (evaluated != nullptr) { + // Reduce the result by skipping false elements. + operands.push_back(evaluated); + } + } + + if (!differs) { + return shared_from_this(); + } + + if (operands.empty()) { + // All elements were false, so the OR context is false. + return nullptr; + } + + Ref result = operands[0]; + for (size_t i = 1; i < operands.size(); ++i) { + result = SemanticContext::Or(result, operands[i]); + } + + return result; +} + std::wstring SemanticContext::OR::toString() const { std::wstring tmp; for(auto var : opnds) { @@ -230,9 +334,15 @@ std::wstring SemanticContext::OR::toString() const { return tmp; } -const SemanticContext::Ref SemanticContext::NONE = std::make_shared(-1, -1, false); +//------------------ SemanticContext ----------------------------------------------------------------------------------- -SemanticContext::Ref SemanticContext::And(SemanticContext::Ref a, SemanticContext::Ref b) { +const Ref SemanticContext::NONE = std::make_shared(-1, -1, false); + +Ref SemanticContext::evalPrecedence(Recognizer *parser, Ref parserCallStack) { + return shared_from_this(); +} + +Ref SemanticContext::And(Ref a, Ref b) { if (!a || a == NONE) { return b; } @@ -241,7 +351,7 @@ SemanticContext::Ref SemanticContext::And(SemanticContext::Ref a, SemanticContex return a; } - std::shared_ptr result = std::make_shared(a, b); + Ref result = std::make_shared(a, b); if (result->opnds.size() == 1) { return result->opnds[0]; } @@ -249,7 +359,7 @@ SemanticContext::Ref SemanticContext::And(SemanticContext::Ref a, SemanticContex return result; } -SemanticContext::Ref SemanticContext::Or(SemanticContext::Ref a, SemanticContext::Ref b) { +Ref SemanticContext::Or(Ref a, Ref b) { if (!a) { return b; } @@ -261,7 +371,7 @@ SemanticContext::Ref SemanticContext::Or(SemanticContext::Ref a, SemanticContext return NONE; } - std::shared_ptr result = std::make_shared(a, b); + Ref result = std::make_shared(a, b); if (result->opnds.size() == 1) { return result->opnds[0]; } diff --git a/runtime/Cpp/runtime/atn/SemanticContext.h b/runtime/Cpp/runtime/atn/SemanticContext.h index fe462855b..5d348b829 100755 --- a/runtime/Cpp/runtime/atn/SemanticContext.h +++ b/runtime/Cpp/runtime/atn/SemanticContext.h @@ -46,13 +46,15 @@ namespace atn { /// /// I have scoped the AND, OR, and Predicate subclasses of /// SemanticContext within the scope of this outer class. - class SemanticContext { + class SemanticContext : public std::enable_shared_from_this { public: - typedef std::shared_ptr Ref; + /** + * The default {@link SemanticContext}, which is semantically equivalent to + * a predicate of the form {@code {true}?}. + */ + static const Ref NONE; - static const SemanticContext::Ref NONE; - - virtual size_t hashCode() = 0; + virtual size_t hashCode() const = 0; virtual std::wstring toString() const = 0; virtual bool operator == (const SemanticContext &other) const = 0; @@ -69,22 +71,43 @@ namespace atn { /// prediction, so we passed in the outer context here in case of context /// dependent predicate evaluation. /// - virtual bool eval(Recognizer *parser, RuleContext::Ref outerContext) = 0; + virtual bool eval(Recognizer *parser, Ref parserCallStack) = 0; - static SemanticContext::Ref And(SemanticContext::Ref a, SemanticContext::Ref b); + /** + * Evaluate the precedence predicates for the context and reduce the result. + * + * @param parser The parser instance. + * @param parserCallStack + * @return The simplified semantic context after precedence predicates are + * evaluated, which will be one of the following values. + *
    + *
  • {@link #NONE}: if the predicate simplifies to {@code true} after + * precedence predicates are evaluated.
    • + *
  • {@code null}: if the predicate simplifies to {@code false} after + * precedence predicates are evaluated.
    • + *
  • {@code this}: if the semantic context is not changed as a result of + * precedence predicate evaluation.
    • + *
  • A non-{@code null} {@link SemanticContext}: the new simplified + * semantic context after precedence predicates are evaluated.
    • + *
+ */ + virtual Ref evalPrecedence(Recognizer *parser, Ref parserCallStack); + + static Ref And(Ref a, Ref b); /// See also: ParserATNSimulator::getPredsForAmbigAlts. - static SemanticContext::Ref Or(SemanticContext::Ref a, SemanticContext::Ref b); + static Ref Or(Ref a, Ref b); class Predicate; class PrecedencePredicate; + class Operator; class AND; class OR; private: - template // where T1 : SemanticContext::Ref - static std::vector> filterPrecedencePredicates(const std::vector &collection) { - std::vector> result; + template // where T1 : SemanticContext> + static std::vector> filterPrecedencePredicates(const std::vector &collection) { + std::vector> result; for (auto context : collection) { if (antlrcpp::is(context)) { result.push_back(std::dynamic_pointer_cast(context)); @@ -96,10 +119,7 @@ namespace atn { }; - class SemanticContext::Predicate : public SemanticContext { - friend class SemanticContext; - public: const int ruleIndex; const int predIndex; @@ -111,18 +131,12 @@ namespace atn { public: Predicate(int ruleIndex, int predIndex, bool isCtxDependent); - virtual bool eval(Recognizer *parser, RuleContext::Ref outerContext) override { - RuleContext::Ref localctx; - if (isCtxDependent) - localctx = outerContext; - return parser->sempred(localctx, ruleIndex, predIndex); - } - - virtual size_t hashCode() override; + virtual bool eval(Recognizer *parser, Ref parserCallStack) override; + virtual size_t hashCode() const override; virtual bool operator == (const SemanticContext &other) const override; virtual std::wstring toString() const override; }; - + class SemanticContext::PrecedencePredicate : public SemanticContext { public: const int precedence; @@ -133,50 +147,80 @@ namespace atn { public: PrecedencePredicate(int precedence); - virtual bool eval(Recognizer *parser, RuleContext::Ref outerContext) override { - return parser->precpred(outerContext, precedence); - } - + virtual bool eval(Recognizer *parser, Ref parserCallStack) override; + virtual Ref evalPrecedence(Recognizer *parser, Ref parserCallStack) override; virtual int compareTo(PrecedencePredicate *o); - virtual size_t hashCode() override; + virtual size_t hashCode() const override; virtual bool operator == (const SemanticContext &other) const override; virtual std::wstring toString() const override; }; - class SemanticContext::AND : public SemanticContext { + /** + * This is the base class for semantic context "operators", which operate on + * a collection of semantic context "operands". + * + * @since 4.3 + */ + class SemanticContext::Operator : public SemanticContext { public: - std::vector opnds; + /** + * Gets the operands for the semantic context operator. + * + * @return a collection of {@link SemanticContext} operands for the + * operator. + * + * @since 4.3 + */ - AND(SemanticContext::Ref a, SemanticContext::Ref b); - - virtual bool operator == (const SemanticContext &other) const override; - virtual size_t hashCode() override; - - virtual bool eval(Recognizer *parser, RuleContext::Ref outerContext) override { - for (auto opnd : opnds) { - if (!opnd->eval(parser, outerContext)) { - return false; - } - } - return true; - } - - - virtual std::wstring toString() const override; + virtual std::vector> getOperands() const = 0; }; - - class SemanticContext::OR : public SemanticContext { + + /** + * A semantic context which is true whenever none of the contained contexts + * is false. + */ + class SemanticContext::AND : public SemanticContext::Operator { public: - std::vector opnds; + std::vector> opnds; - OR(SemanticContext::Ref a, SemanticContext::Ref b); + AND(Ref a, Ref b) ; + virtual std::vector> getOperands() const override; virtual bool operator == (const SemanticContext &other) const override; - virtual size_t hashCode() override; - virtual bool eval(Recognizer *parser, RuleContext::Ref outerContext) override; + virtual size_t hashCode() const override; + + /** + * The evaluation of predicates by this context is short-circuiting, but + * unordered.

+ */ + virtual bool eval(Recognizer *parser, Ref parserCallStack) override; + virtual Ref evalPrecedence(Recognizer *parser, Ref parserCallStack) override; virtual std::wstring toString() const override; }; + /** + * A semantic context which is true whenever at least one of the contained + * contexts is true. + */ + class SemanticContext::OR : public SemanticContext::Operator { + public: + std::vector> opnds; + + OR(Ref a, Ref b); + + virtual std::vector> getOperands() const override; + virtual bool operator == (const SemanticContext &other) const override; + virtual size_t hashCode() const override; + + /** + * The evaluation of predicates by this context is short-circuiting, but + * unordered. + */ + virtual bool eval(Recognizer *parser, Ref parserCallStack) override; + virtual Ref evalPrecedence(Recognizer *parser, Ref parserCallStack) override; + virtual std::wstring toString() const override; + }; + } // namespace atn } // namespace runtime } // namespace v4 diff --git a/runtime/Cpp/runtime/atn/SingletonPredictionContext.cpp b/runtime/Cpp/runtime/atn/SingletonPredictionContext.cpp index 2c1841116..b11c6bff5 100755 --- a/runtime/Cpp/runtime/atn/SingletonPredictionContext.cpp +++ b/runtime/Cpp/runtime/atn/SingletonPredictionContext.cpp @@ -41,7 +41,8 @@ SingletonPredictionContext::SingletonPredictionContext(std::weak_ptr parent, int returnState) { +Ref SingletonPredictionContext::create(std::weak_ptr parent, + int returnState) { if (returnState == EMPTY_RETURN_STATE && parent.expired()) { // someone can pass in the bits of an array ctx that mean $ return std::dynamic_pointer_cast(EMPTY); diff --git a/runtime/Cpp/runtime/atn/SingletonPredictionContext.h b/runtime/Cpp/runtime/atn/SingletonPredictionContext.h index 8c62a441e..32226fec4 100755 --- a/runtime/Cpp/runtime/atn/SingletonPredictionContext.h +++ b/runtime/Cpp/runtime/atn/SingletonPredictionContext.h @@ -41,14 +41,12 @@ namespace atn { class SingletonPredictionContext : public PredictionContext { public: - typedef std::shared_ptr Ref; - const std::weak_ptr parent; const int returnState; SingletonPredictionContext(std::weak_ptr parent, int returnState); - static SingletonPredictionContext::Ref create(std::weak_ptr parent, int returnState); + static Ref create(std::weak_ptr parent, int returnState); virtual size_t size() const override; virtual std::weak_ptr getParent(size_t index) const override; diff --git a/runtime/Cpp/runtime/dfa/DFA.cpp b/runtime/Cpp/runtime/dfa/DFA.cpp index 3e892b187..11cd24fe8 100755 --- a/runtime/Cpp/runtime/dfa/DFA.cpp +++ b/runtime/Cpp/runtime/dfa/DFA.cpp @@ -32,17 +32,44 @@ #include "DFAState.h" #include "DFASerializer.h" #include "LexerDFASerializer.h" +#include "CPPUtils.h" +#include "StarLoopEntryState.h" +#include "ATNConfigSet.h" #include "DFA.h" using namespace org::antlr::v4::runtime; using namespace org::antlr::v4::runtime::dfa; +using namespace antlrcpp; DFA::DFA(atn::DecisionState *atnStartState) : DFA(atnStartState, 0) { } DFA::DFA(atn::DecisionState *atnStartState, int decision) : atnStartState(atnStartState), decision(decision), s0(nullptr) { + + _precedenceDfa = false; + if (is(atnStartState)) { + if (static_cast(atnStartState)->isPrecedenceDecision) { + _precedenceDfa = true; + DFAState *precedenceState = new DFAState(std::make_shared()); // TODO: mem leak + precedenceState->isAcceptState = false; + precedenceState->requiresFullContext = false; + s0 = precedenceState; + } + } +} + +DFA::DFA(DFA &&other) : atnStartState(std::move(other.atnStartState)), decision(std::move(other.decision)) { + states = std::move(other.states); + s0 = std::move(other.s0); + _precedenceDfa = std::move(other._precedenceDfa); +} + +DFA::DFA(const DFA &other) : atnStartState(other.atnStartState), decision(other.decision) { + states = other.states; + s0 = other.s0; + _precedenceDfa = other._precedenceDfa; } DFA::~DFA() { @@ -51,7 +78,46 @@ DFA::~DFA() { } } -std::vector DFA::getStates() { +bool DFA::isPrecedenceDfa() const { + return _precedenceDfa; +} + +DFAState* DFA::getPrecedenceStartState(int precedence) const { + if (!isPrecedenceDfa()) { + throw IllegalStateException("Only precedence DFAs may contain a precedence start state."); + } + + // s0.edges is never null for a precedence DFA + if (precedence < 0 || precedence >= (int)s0->edges.size()) { + return nullptr; + } + + return s0->edges[precedence]; +} + +void DFA::setPrecedenceStartState(int precedence, DFAState *startState) { + if (!isPrecedenceDfa()) { + throw IllegalStateException("Only precedence DFAs may contain a precedence start state."); + } + + if (precedence < 0) { + return; + } + + // synchronization on s0 here is ok. when the DFA is turned into a + // precedence DFA, s0 will be initialized once and not updated again + std::unique_lock lock(_lock); + { + // s0.edges is never null for a precedence DFA + if (precedence >= (int)s0->edges.size()) { + s0->edges.resize(precedence + 1); + } + + s0->edges[precedence] = startState; + } +} + +std::vector DFA::getStates() const { std::vector result; for (auto state : states) result.push_back(state.first); @@ -63,11 +129,6 @@ std::vector DFA::getStates() { return result; } -std::wstring DFA::toString() { - std::vector tokenNames; - return toString(tokenNames); -} - std::wstring DFA::toString(const std::vector &tokenNames) { if (s0 == nullptr) { return L""; @@ -77,6 +138,15 @@ std::wstring DFA::toString(const std::vector &tokenNames) { return serializer.toString(); } +std::wstring DFA::toString(Ref vocabulary) const { + if (s0 == nullptr) { + return L""; + } + + DFASerializer serializer(this, vocabulary); + return serializer.toString(); +} + std::wstring DFA::toLexerString() { if (s0 == nullptr) { return L""; @@ -85,3 +155,4 @@ std::wstring DFA::toLexerString() { return serializer.toString(); } + diff --git a/runtime/Cpp/runtime/dfa/DFA.h b/runtime/Cpp/runtime/dfa/DFA.h index 014aab0e0..6ee7d777b 100755 --- a/runtime/Cpp/runtime/dfa/DFA.h +++ b/runtime/Cpp/runtime/dfa/DFA.h @@ -50,15 +50,66 @@ namespace dfa { DFA(atn::DecisionState *atnStartState); DFA(atn::DecisionState *atnStartState, int decision); + DFA(const DFA &other); + DFA(DFA &&other); ~DFA(); + /** + * Gets whether this DFA is a precedence DFA. Precedence DFAs use a special + * start state {@link #s0} which is not stored in {@link #states}. The + * {@link DFAState#edges} array for this start state contains outgoing edges + * supplying individual start states corresponding to specific precedence + * values. + * + * @return {@code true} if this is a precedence DFA; otherwise, + * {@code false}. + * @see Parser#getPrecedence() + */ + bool isPrecedenceDfa() const; + + /** + * Get the start state for a specific precedence value. + * + * @param precedence The current precedence. + * @return The start state corresponding to the specified precedence, or + * {@code null} if no start state exists for the specified precedence. + * + * @throws IllegalStateException if this is not a precedence DFA. + * @see #isPrecedenceDfa() + */ + DFAState* getPrecedenceStartState(int precedence) const; + + /** + * Set the start state for a specific precedence value. + * + * @param precedence The current precedence. + * @param startState The start state corresponding to the specified + * precedence. + * + * @throws IllegalStateException if this is not a precedence DFA. + * @see #isPrecedenceDfa() + */ + void setPrecedenceStartState(int precedence, DFAState *startState); + /// Return a list of all states in this DFA, ordered by state number. - virtual std::vector getStates(); + virtual std::vector getStates() const; - virtual std::wstring toString(); + /** + * @deprecated Use {@link #toString(Vocabulary)} instead. + */ virtual std::wstring toString(const std::vector& tokenNames); + std::wstring toString(Ref vocabulary) const; + virtual std::wstring toLexerString(); + private: + /** + * {@code true} if this DFA is for a precedence decision; otherwise, + * {@code false}. This is the backing field for {@link #isPrecedenceDfa}. + */ + bool _precedenceDfa; + + std::mutex _lock; // To synchronize access to s0. }; } // namespace atn diff --git a/runtime/Cpp/runtime/dfa/DFASerializer.cpp b/runtime/Cpp/runtime/dfa/DFASerializer.cpp index e710ec784..43d89a2b9 100755 --- a/runtime/Cpp/runtime/dfa/DFASerializer.cpp +++ b/runtime/Cpp/runtime/dfa/DFASerializer.cpp @@ -1,8 +1,9 @@ /* * [The "BSD license"] * Copyright (c) 2016 Mike Lischke - * Copyright (c) 2013 Terence Parr * Copyright (c) 2013 Dan McLaughlin + * Copyright (c) 2013 Terence Parr + * Copyright (c) 2012 Sam Harwell * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,21 +32,26 @@ #include "DFA.h" #include "DFAState.h" +#include "VocabularyImpl.h" #include "DFASerializer.h" using namespace org::antlr::v4::runtime::dfa; -DFASerializer::DFASerializer(DFA *dfa, const std::vector& tnames) : dfa(dfa), tokenNames(tnames) { +DFASerializer::DFASerializer(const DFA *dfa, const std::vector& tokenNames) + : DFASerializer(dfa, VocabularyImpl::fromTokenNames(tokenNames)) { } -std::wstring DFASerializer::toString() { - if (dfa->s0 == nullptr) { +DFASerializer::DFASerializer(const DFA *dfa, Ref vocabulary) : _dfa(dfa), _vocabulary(vocabulary) { +} + +std::wstring DFASerializer::toString() const { + if (_dfa->s0 == nullptr) { return L""; } std::wstringstream ss; - std::vector states = dfa->getStates(); + std::vector states = _dfa->getStates(); for (auto s : states) { for (size_t i = 0; i < s->edges.size(); i++) { DFAState *t = s->edges[i]; @@ -60,20 +66,11 @@ std::wstring DFASerializer::toString() { return ss.str(); } -std::wstring DFASerializer::getEdgeLabel(size_t i) { - std::wstring label; - if (i == 0) { - return L"EOF"; - } - if (!tokenNames.empty()) { - label = tokenNames[i - 1]; - } else { - label = std::to_wstring(i - 1); - } - return label; +std::wstring DFASerializer::getEdgeLabel(size_t i) const { + return _vocabulary->getDisplayName((int)i - 1); } -std::wstring DFASerializer::getStateString(DFAState *s) { +std::wstring DFASerializer::getStateString(DFAState *s) const { size_t n = (size_t)s->stateNumber; const std::wstring baseStateStr = (s->isAcceptState ? L":" : L"") + std::wstring(L"s") + std::to_wstring(n) + (s->requiresFullContext ? L"^" : L""); diff --git a/runtime/Cpp/runtime/dfa/DFASerializer.h b/runtime/Cpp/runtime/dfa/DFASerializer.h index 843849e30..3dab005b0 100755 --- a/runtime/Cpp/runtime/dfa/DFASerializer.h +++ b/runtime/Cpp/runtime/dfa/DFASerializer.h @@ -1,8 +1,9 @@ /* * [The "BSD license"] * Copyright (c) 2016 Mike Lischke - * Copyright (c) 2013 Terence Parr * Copyright (c) 2013 Dan McLaughlin + * Copyright (c) 2012 Terence Parr + * Copyright (c) 2012 Sam Harwell * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,6 +32,8 @@ #pragma once +#include "Vocabulary.h" + namespace org { namespace antlr { namespace v4 { @@ -40,16 +43,18 @@ namespace dfa { /// A DFA walker that knows how to dump them to serialized strings. class DFASerializer { public: - DFA *const dfa; - const std::vector& tokenNames; + DFASerializer(const DFA *dfa, const std::vector& tnames); + DFASerializer(const DFA *dfa, Ref vocabulary); - DFASerializer(DFA *dfa, const std::vector& tnames); - - virtual std::wstring toString(); + virtual std::wstring toString() const; protected: - virtual std::wstring getEdgeLabel(size_t i); - virtual std::wstring getStateString(DFAState *s); + virtual std::wstring getEdgeLabel(size_t i) const; + virtual std::wstring getStateString(DFAState *s) const; + + private: + const DFA *_dfa; + Ref const _vocabulary; }; } // namespace atn diff --git a/runtime/Cpp/runtime/dfa/DFAState.cpp b/runtime/Cpp/runtime/dfa/DFAState.cpp index eec4f8be0..a4157efe5 100755 --- a/runtime/Cpp/runtime/dfa/DFAState.cpp +++ b/runtime/Cpp/runtime/dfa/DFAState.cpp @@ -39,7 +39,7 @@ using namespace org::antlr::v4::runtime::dfa; using namespace org::antlr::v4::runtime::atn; -DFAState::PredPrediction::PredPrediction(SemanticContext::Ref pred, int alt) : pred(pred) { +DFAState::PredPrediction::PredPrediction(Ref pred, int alt) : pred(pred) { InitializeInstanceFields(); this->alt = alt; } @@ -60,7 +60,7 @@ DFAState::DFAState(int state) : DFAState() { stateNumber = state; } -DFAState::DFAState(std::shared_ptr configs) : DFAState() { +DFAState::DFAState(Ref configs) : DFAState() { this->configs = configs; } @@ -119,7 +119,5 @@ void DFAState::InitializeInstanceFields() { stateNumber = -1; isAcceptState = false; prediction = 0; - lexerRuleIndex = -1; - lexerActionIndex = -1; requiresFullContext = false; } diff --git a/runtime/Cpp/runtime/dfa/DFAState.h b/runtime/Cpp/runtime/dfa/DFAState.h index 5061bcaf3..8eecd5b06 100755 --- a/runtime/Cpp/runtime/dfa/DFAState.h +++ b/runtime/Cpp/runtime/dfa/DFAState.h @@ -66,9 +66,9 @@ namespace dfa { public: class PredPrediction { public: - std::shared_ptr pred; // never null; at least SemanticContext.NONE + Ref pred; // never null; at least SemanticContext.NONE int alt; - PredPrediction(std::shared_ptr pred, int alt); + PredPrediction(Ref pred, int alt); virtual std::wstring toString(); private: @@ -78,7 +78,7 @@ namespace dfa { public: int stateNumber; - std::shared_ptr configs; + Ref configs; /// /// {@code edges[symbol]} points to target of symbol. Shift up by 1 so (-1) @@ -95,8 +95,7 @@ namespace dfa { /// int prediction; - int lexerRuleIndex; // if accept, exec action in what rule? - int lexerActionIndex; // if accept, exec what action? + Ref lexerActionExecutor; /// /// Indicates that this state was created during SLL prediction that @@ -124,7 +123,7 @@ namespace dfa { /// Map a predicate to a predicted alternative. DFAState(); DFAState(int state); - DFAState(std::shared_ptr configs); + DFAState(Ref configs); ~DFAState(); /// @@ -133,7 +132,7 @@ namespace dfa { /// virtual std::set getAltSet(); - virtual size_t hashCode() ; + virtual size_t hashCode(); /// Two DFAState instances are equal if their ATN configuration sets /// are the same. This method is used to see if a state already exists. diff --git a/runtime/Cpp/runtime/dfa/LexerDFASerializer.cpp b/runtime/Cpp/runtime/dfa/LexerDFASerializer.cpp index 9b2cb24eb..48a0f3e28 100755 --- a/runtime/Cpp/runtime/dfa/LexerDFASerializer.cpp +++ b/runtime/Cpp/runtime/dfa/LexerDFASerializer.cpp @@ -36,6 +36,6 @@ using namespace org::antlr::v4::runtime::dfa; LexerDFASerializer::LexerDFASerializer(DFA *dfa) : DFASerializer(dfa, lexerTokenNames_) { } -std::wstring LexerDFASerializer::getEdgeLabel(size_t i) { +std::wstring LexerDFASerializer::getEdgeLabel(size_t i) const { return std::wstring(L"'") + static_cast(i) + std::wstring(L"'"); } diff --git a/runtime/Cpp/runtime/dfa/LexerDFASerializer.h b/runtime/Cpp/runtime/dfa/LexerDFASerializer.h index cb8feb82d..b975c126a 100755 --- a/runtime/Cpp/runtime/dfa/LexerDFASerializer.h +++ b/runtime/Cpp/runtime/dfa/LexerDFASerializer.h @@ -45,7 +45,7 @@ namespace dfa { protected: std::vector lexerTokenNames_; - virtual std::wstring getEdgeLabel(size_t i) override; + virtual std::wstring getEdgeLabel(size_t i) const override; }; } // namespace atn diff --git a/runtime/Cpp/runtime/misc/MurmurHash.h b/runtime/Cpp/runtime/misc/MurmurHash.h index bbbe1b291..ad3ae6eb6 100755 --- a/runtime/Cpp/runtime/misc/MurmurHash.h +++ b/runtime/Cpp/runtime/misc/MurmurHash.h @@ -72,7 +72,7 @@ namespace misc { /// the seed for the MurmurHash algorithm /// the hash code of the data template // where T is C array type - static size_t hashCode(const std::vector> &data, size_t seed) { + static size_t hashCode(const std::vector> &data, size_t seed) { size_t hash = initialize(seed); for (auto entry : data) { hash = update(hash, (size_t)entry.get()); diff --git a/runtime/Cpp/runtime/support/Arrays.h b/runtime/Cpp/runtime/support/Arrays.h index 42f1da222..4d8bf6156 100644 --- a/runtime/Cpp/runtime/support/Arrays.h +++ b/runtime/Cpp/runtime/support/Arrays.h @@ -97,7 +97,7 @@ namespace antlrcpp { } template - static std::wstring toString(const std::vector> &source) { + static std::wstring toString(const std::vector> &source) { std::wstring result = L"["; bool firstEntry = true; for (auto &value : source) { diff --git a/runtime/Cpp/runtime/support/BitSet.h b/runtime/Cpp/runtime/support/BitSet.h index 2d5788b58..9f088f401 100644 --- a/runtime/Cpp/runtime/support/BitSet.h +++ b/runtime/Cpp/runtime/support/BitSet.h @@ -34,7 +34,7 @@ namespace antlrcpp { class BitSet : public std::bitset<1024> { public: - int nextSetBit(size_t pos) { + int nextSetBit(size_t pos) const { for (size_t i = pos; i < size(); i++){ if (test(i)) { return (int)i; diff --git a/runtime/Cpp/runtime/support/CPPUtils.h b/runtime/Cpp/runtime/support/CPPUtils.h index cf339b3fb..36c1f79a1 100644 --- a/runtime/Cpp/runtime/support/CPPUtils.h +++ b/runtime/Cpp/runtime/support/CPPUtils.h @@ -67,7 +67,7 @@ namespace antlrcpp { } template - bool is(std::shared_ptr obj) { // For shared pointers. + bool is(Ref obj) { // For shared pointers. return dynamic_cast(obj.get()) != nullptr; } @@ -79,7 +79,7 @@ namespace antlrcpp { ss << typeid(o).name() << "@" << std::hex << (size_t)&o; return ss.str(); } - + } // namespace antlrcpp namespace std { @@ -96,7 +96,7 @@ namespace std { } template - bool operator == (const std::shared_ptr &lhs, const std::shared_ptr &rhs) { + bool operator == (const Ref &lhs, const Ref &rhs) { if (!lhs && !rhs) return true; diff --git a/runtime/Cpp/runtime/support/Declarations.h b/runtime/Cpp/runtime/support/Declarations.h index 408c1a8e3..a26d735d2 100644 --- a/runtime/Cpp/runtime/support/Declarations.h +++ b/runtime/Cpp/runtime/support/Declarations.h @@ -122,11 +122,14 @@ namespace org { class EmptyPredictionContext; class EpsilonTransition; class LL1Analyzer; + class LexerAction; + class LexerActionExecutor; class LexerATNConfig; class LexerATNSimulator; class LoopEndState; class NotSetTransition; class OrderedATNConfigSet; + class ParseInfo; class ParserATNSimulator; class PlusBlockStartState; class PlusLoopbackState; @@ -154,6 +157,7 @@ namespace org { class DFASerializer; class DFAState; class LexerDFASerializer; + class Vocabulary; } namespace tree { template class AbstractParseTreeVisitor; diff --git a/runtime/Cpp/runtime/tree/AbstractParseTreeVisitor.h b/runtime/Cpp/runtime/tree/AbstractParseTreeVisitor.h index 995de214b..dadc281d4 100755 --- a/runtime/Cpp/runtime/tree/AbstractParseTreeVisitor.h +++ b/runtime/Cpp/runtime/tree/AbstractParseTreeVisitor.h @@ -72,7 +72,7 @@ namespace tree { break; } - std::shared_ptr c = node->getChild(i); + Ref c = node->getChild(i); T childResult = c->accept(this); result = aggregateResult(result, childResult); } diff --git a/runtime/Cpp/runtime/tree/ErrorNodeImpl.cpp b/runtime/Cpp/runtime/tree/ErrorNodeImpl.cpp index 508b19c42..c3333da92 100755 --- a/runtime/Cpp/runtime/tree/ErrorNodeImpl.cpp +++ b/runtime/Cpp/runtime/tree/ErrorNodeImpl.cpp @@ -37,5 +37,5 @@ using namespace org::antlr::v4::runtime; using namespace org::antlr::v4::runtime::misc; using namespace org::antlr::v4::runtime::tree; -ErrorNodeImpl::ErrorNodeImpl(Token::Ref token) : TerminalNodeImpl(token) { +ErrorNodeImpl::ErrorNodeImpl(Ref token) : TerminalNodeImpl(token) { } diff --git a/runtime/Cpp/runtime/tree/ErrorNodeImpl.h b/runtime/Cpp/runtime/tree/ErrorNodeImpl.h index 45d40e9d3..d53c2a215 100755 --- a/runtime/Cpp/runtime/tree/ErrorNodeImpl.h +++ b/runtime/Cpp/runtime/tree/ErrorNodeImpl.h @@ -50,7 +50,7 @@ namespace tree { /// class ErrorNodeImpl : public virtual TerminalNodeImpl, public virtual ErrorNode { public: - ErrorNodeImpl(Token::Ref token); + ErrorNodeImpl(Ref token); template T accept(ParseTreeVisitor *visitor) { diff --git a/runtime/Cpp/runtime/tree/ParseTree.h b/runtime/Cpp/runtime/tree/ParseTree.h index 79d0b0eb2..7bb0eabae 100755 --- a/runtime/Cpp/runtime/tree/ParseTree.h +++ b/runtime/Cpp/runtime/tree/ParseTree.h @@ -51,7 +51,7 @@ namespace tree { // the following methods narrow the return type; they are not additional methods public: std::weak_ptr getParent() { return std::dynamic_pointer_cast(getParentReference().lock()); }; - virtual std::shared_ptr getChild(size_t i) { return std::dynamic_pointer_cast(getChildReference(i)); }; + virtual Ref getChild(size_t i) { return std::dynamic_pointer_cast(getChildReference(i)); }; /// /// The needs a double dispatch method. diff --git a/runtime/Cpp/runtime/tree/ParseTreeListener.h b/runtime/Cpp/runtime/tree/ParseTreeListener.h index 60885990d..97c1d3577 100755 --- a/runtime/Cpp/runtime/tree/ParseTreeListener.h +++ b/runtime/Cpp/runtime/tree/ParseTreeListener.h @@ -39,10 +39,10 @@ namespace tree { class ParseTreeListener { public: - virtual void visitTerminal(std::shared_ptr node) = 0; - virtual void visitErrorNode(std::shared_ptr node) = 0; - virtual void enterEveryRule(std::shared_ptr ctx) = 0; - virtual void exitEveryRule(std::shared_ptr ctx) = 0; + virtual void visitTerminal(Ref node) = 0; + virtual void visitErrorNode(Ref node) = 0; + virtual void enterEveryRule(Ref ctx) = 0; + virtual void exitEveryRule(Ref ctx) = 0; bool operator == (const ParseTreeListener &other) { return this == &other; diff --git a/runtime/Cpp/runtime/tree/ParseTreeWalker.cpp b/runtime/Cpp/runtime/tree/ParseTreeWalker.cpp index d515a78b2..de2b569f5 100755 --- a/runtime/Cpp/runtime/tree/ParseTreeWalker.cpp +++ b/runtime/Cpp/runtime/tree/ParseTreeWalker.cpp @@ -39,9 +39,9 @@ using namespace org::antlr::v4::runtime::tree; using namespace antlrcpp; -const std::shared_ptr ParseTreeWalker::DEFAULT = std::make_shared(); +const Ref ParseTreeWalker::DEFAULT = std::make_shared(); -void ParseTreeWalker::walk(std::shared_ptr listener, std::shared_ptr t) { +void ParseTreeWalker::walk(Ref listener, Ref t) { if (is(t)) { listener->visitErrorNode(std::dynamic_pointer_cast(t)); return; @@ -49,7 +49,7 @@ void ParseTreeWalker::walk(std::shared_ptr listener, std::sha listener->visitTerminal(std::dynamic_pointer_cast(t)); return; } - std::shared_ptr r = std::dynamic_pointer_cast(t); + Ref r = std::dynamic_pointer_cast(t); enterRule(listener, r); std::size_t n = r->getChildCount(); for (std::size_t i = 0; i < n; i++) { @@ -58,14 +58,14 @@ void ParseTreeWalker::walk(std::shared_ptr listener, std::sha exitRule(listener, r); } -void ParseTreeWalker::enterRule(std::shared_ptr listener, std::shared_ptr r) { - ParserRuleContext::Ref ctx = std::dynamic_pointer_cast(r->getRuleContext()); +void ParseTreeWalker::enterRule(Ref listener, Ref r) { + Ref ctx = std::dynamic_pointer_cast(r->getRuleContext()); listener->enterEveryRule(ctx); ctx->enterRule(listener); } -void ParseTreeWalker::exitRule(std::shared_ptr listener, std::shared_ptr r) { - ParserRuleContext::Ref ctx = std::dynamic_pointer_cast(r->getRuleContext()); +void ParseTreeWalker::exitRule(Ref listener, Ref r) { + Ref ctx = std::dynamic_pointer_cast(r->getRuleContext()); ctx->exitRule(listener); listener->exitEveryRule(ctx); } diff --git a/runtime/Cpp/runtime/tree/ParseTreeWalker.h b/runtime/Cpp/runtime/tree/ParseTreeWalker.h index c769c40af..73508b975 100755 --- a/runtime/Cpp/runtime/tree/ParseTreeWalker.h +++ b/runtime/Cpp/runtime/tree/ParseTreeWalker.h @@ -39,9 +39,9 @@ namespace tree { class ParseTreeWalker { public: - static const std::shared_ptr DEFAULT; + static const Ref DEFAULT; - virtual void walk(std::shared_ptr listener, std::shared_ptr t); + virtual void walk(Ref listener, Ref t); /// /// The discovery of a rule node, involves sending two events: the generic @@ -50,9 +50,9 @@ namespace tree { /// the rule specific. We to them in reverse order upon finishing the node. /// protected: - virtual void enterRule(std::shared_ptr listener, std::shared_ptr r); + virtual void enterRule(Ref listener, Ref r); - virtual void exitRule(std::shared_ptr listener, std::shared_ptr r); + virtual void exitRule(Ref listener, Ref r); }; } // namespace tree diff --git a/runtime/Cpp/runtime/tree/RuleNode.h b/runtime/Cpp/runtime/tree/RuleNode.h index f2c39b30c..da4ce0fb7 100755 --- a/runtime/Cpp/runtime/tree/RuleNode.h +++ b/runtime/Cpp/runtime/tree/RuleNode.h @@ -41,8 +41,8 @@ namespace tree { class RuleNode : public ParseTree { public: - // Because of cross references (RuleNode <-> RuleContext) we cannot use RuleContext::Ref here. - virtual std::shared_ptr getRuleContext() = 0; + // Because of cross references (RuleNode <-> RuleContext) we cannot use RuleContext> here. + virtual Ref getRuleContext() = 0; }; } // namespace tree diff --git a/runtime/Cpp/runtime/tree/TerminalNode.h b/runtime/Cpp/runtime/tree/TerminalNode.h index fb0d3026e..5ba911678 100755 --- a/runtime/Cpp/runtime/tree/TerminalNode.h +++ b/runtime/Cpp/runtime/tree/TerminalNode.h @@ -31,7 +31,6 @@ #pragma once -#include "Token.h" #include "ParseTree.h" namespace org { @@ -42,7 +41,7 @@ namespace tree { class TerminalNode : public ParseTree { public: - virtual Token::Ref getSymbol() = 0; + virtual Ref getSymbol() = 0; }; } // namespace tree diff --git a/runtime/Cpp/runtime/tree/TerminalNodeImpl.cpp b/runtime/Cpp/runtime/tree/TerminalNodeImpl.cpp index f176f3447..81caa6b09 100755 --- a/runtime/Cpp/runtime/tree/TerminalNodeImpl.cpp +++ b/runtime/Cpp/runtime/tree/TerminalNodeImpl.cpp @@ -37,11 +37,11 @@ using namespace org::antlr::v4::runtime; using namespace org::antlr::v4::runtime::tree; -TerminalNodeImpl::TerminalNodeImpl(Token::Ref symbol) { +TerminalNodeImpl::TerminalNodeImpl(Ref symbol) { this->symbol = symbol; } -Token::Ref TerminalNodeImpl::getSymbol() { +Ref TerminalNodeImpl::getSymbol() { return symbol; } @@ -81,6 +81,6 @@ std::weak_ptr TerminalNodeImpl::getParentReference() { return parent; } -std::shared_ptr TerminalNodeImpl::getChildReference(size_t i) { - return std::shared_ptr(); +Ref TerminalNodeImpl::getChildReference(size_t i) { + return Ref(); } diff --git a/runtime/Cpp/runtime/tree/TerminalNodeImpl.h b/runtime/Cpp/runtime/tree/TerminalNodeImpl.h index 80d5a722d..edb728cb5 100755 --- a/runtime/Cpp/runtime/tree/TerminalNodeImpl.h +++ b/runtime/Cpp/runtime/tree/TerminalNodeImpl.h @@ -41,12 +41,12 @@ namespace tree { class TerminalNodeImpl : public virtual TerminalNode { public: - Token::Ref symbol; + Ref symbol; std::weak_ptr parent; - TerminalNodeImpl(Token::Ref symbol); + TerminalNodeImpl(Ref symbol); - virtual Token::Ref getSymbol() override; + virtual Ref getSymbol() override; virtual misc::Interval getSourceInterval() override; virtual std::size_t getChildCount() override; @@ -63,7 +63,7 @@ namespace tree { protected: virtual std::weak_ptr getParentReference() override; - virtual std::shared_ptr getChildReference(size_t i) override; + virtual Ref getChildReference(size_t i) override; }; } // namespace tree diff --git a/runtime/Cpp/runtime/tree/Tree.h b/runtime/Cpp/runtime/tree/Tree.h index 40cfb59a4..b6a763a76 100755 --- a/runtime/Cpp/runtime/tree/Tree.h +++ b/runtime/Cpp/runtime/tree/Tree.h @@ -62,7 +62,7 @@ namespace tree { /// /// If there are children, get the {@code i}th value indexed from 0. - std::shared_ptr getChild(size_t i) { return getChildReference(i); }; + Ref getChild(size_t i) { return getChildReference(i); }; /// /// How many children are there? If there is none, then this @@ -80,7 +80,7 @@ namespace tree { protected: virtual std::weak_ptr getParentReference() = 0; - virtual std::shared_ptr getChildReference(size_t i) = 0; + virtual Ref getChildReference(size_t i) = 0; }; } // namespace tree diff --git a/runtime/Cpp/runtime/tree/Trees.cpp b/runtime/Cpp/runtime/tree/Trees.cpp index e1f9fe0d0..7f1472c11 100755 --- a/runtime/Cpp/runtime/tree/Trees.cpp +++ b/runtime/Cpp/runtime/tree/Trees.cpp @@ -42,17 +42,17 @@ using namespace org::antlr::v4::runtime::tree; using namespace antlrcpp; -std::wstring Trees::toStringTree(std::shared_ptr t) { +std::wstring Trees::toStringTree(Ref t) { return toStringTree(t, nullptr); } -std::wstring Trees::toStringTree(std::shared_ptr t, Parser *recog) { +std::wstring Trees::toStringTree(Ref t, Parser *recog) { if (recog == nullptr) return toStringTree(t, {}); return toStringTree(t, recog->getRuleNames()); } -std::wstring Trees::toStringTree(std::shared_ptr t, const std::vector &ruleNames) { +std::wstring Trees::toStringTree(Ref t, const std::vector &ruleNames) { std::wstring temp = antlrcpp::escapeWhitespace(Trees::getNodeText(t, ruleNames), false); if (t->getChildCount() == 0) { return temp; @@ -70,11 +70,11 @@ std::wstring Trees::toStringTree(std::shared_ptr t, const std::vector t, Parser *recog) { +std::wstring Trees::getNodeText(Ref t, Parser *recog) { return getNodeText(t, recog->getRuleNames()); } -std::wstring Trees::getNodeText(std::shared_ptr t, const std::vector &ruleNames) { +std::wstring Trees::getNodeText(Ref t, const std::vector &ruleNames) { if (ruleNames.size() > 0) { if (is(t)) { ssize_t ruleIndex = (std::static_pointer_cast(t))->getRuleContext()->getRuleIndex(); @@ -85,7 +85,7 @@ std::wstring Trees::getNodeText(std::shared_ptr t, const std::vector(t)) { return t->toString(); } else if (is(t)) { - Token::Ref symbol = (std::static_pointer_cast(t))->getSymbol(); + Ref symbol = (std::static_pointer_cast(t))->getSymbol(); if (symbol != nullptr) { std::wstring s = symbol->getText(); return s; @@ -104,15 +104,15 @@ std::wstring Trees::getNodeText(std::shared_ptr t, const std::vector> Trees::getChildren(std::shared_ptr t) { - std::vector> kids; +std::vector> Trees::getChildren(Ref t) { + std::vector> kids; for (size_t i = 0; i < t->getChildCount(); i++) { kids.push_back(t->getChild(i)); } return kids; } -std::vector> Trees::getAncestors(std::shared_ptr t) { +std::vector> Trees::getAncestors(Ref t) { std::vector> ancestors; while (!t->getParent().expired()) { t = t->getParent().lock(); @@ -122,15 +122,15 @@ std::vector> Trees::getAncestors(std::shared_ptr t) { } template -static void _findAllNodes(std::shared_ptr t, int index, bool findTokens, std::vector &nodes) { +static void _findAllNodes(Ref t, int index, bool findTokens, std::vector &nodes) { // check this node (the root) first if (findTokens && is(t)) { - std::shared_ptr tnode = std::dynamic_pointer_cast(t); + Ref tnode = std::dynamic_pointer_cast(t); if (tnode->getSymbol()->getType() == index) { nodes.push_back(t); } } else if (!findTokens && is(t)) { - ParserRuleContext::Ref ctx = std::dynamic_pointer_cast(t); + Ref ctx = std::dynamic_pointer_cast(t); if (ctx->getRuleIndex() == index) { nodes.push_back(t); } @@ -141,22 +141,22 @@ static void _findAllNodes(std::shared_ptr t, int index, bool findToke } } -std::vector> Trees::findAllTokenNodes(std::shared_ptr t, int ttype) { +std::vector> Trees::findAllTokenNodes(Ref t, int ttype) { return findAllNodes(t, ttype, true); } -std::vector> Trees::findAllRuleNodes(std::shared_ptr t, int ruleIndex) { +std::vector> Trees::findAllRuleNodes(Ref t, int ruleIndex) { return findAllNodes(t, ruleIndex, false); } -std::vector> Trees::findAllNodes(std::shared_ptr t, int index, bool findTokens) { - std::vector> nodes; - _findAllNodes>(t, index, findTokens, nodes); +std::vector> Trees::findAllNodes(Ref t, int index, bool findTokens) { + std::vector> nodes; + _findAllNodes>(t, index, findTokens, nodes); return nodes; } -std::vector> Trees::descendants(std::shared_ptr t) { - std::vector> nodes; +std::vector> Trees::descendants(Ref t) { + std::vector> nodes; nodes.push_back(t); std::size_t n = t->getChildCount(); for (size_t i = 0 ; i < n ; i++) { diff --git a/runtime/Cpp/runtime/tree/Trees.h b/runtime/Cpp/runtime/tree/Trees.h index 7466a19d7..3f4daea7d 100755 --- a/runtime/Cpp/runtime/tree/Trees.h +++ b/runtime/Cpp/runtime/tree/Trees.h @@ -48,31 +48,31 @@ namespace tree { /// Print out a whole tree in LISP form. getNodeText is used on the /// node payloads to get the text for the nodes. Detect /// parse trees and extract data appropriately. - static std::wstring toStringTree(std::shared_ptr t); + static std::wstring toStringTree(Ref t); /// Print out a whole tree in LISP form. getNodeText is used on the /// node payloads to get the text for the nodes. Detect /// parse trees and extract data appropriately. - static std::wstring toStringTree(std::shared_ptr t, Parser *recog); + static std::wstring toStringTree(Ref t, Parser *recog); /// Print out a whole tree in LISP form. getNodeText is used on the /// node payloads to get the text for the nodes. Detect /// parse trees and extract data appropriately. - static std::wstring toStringTree(std::shared_ptr t, const std::vector &ruleNames); - static std::wstring getNodeText(std::shared_ptr t, Parser *recog); - static std::wstring getNodeText(std::shared_ptr t, const std::vector &ruleNames); + static std::wstring toStringTree(Ref t, const std::vector &ruleNames); + static std::wstring getNodeText(Ref t, Parser *recog); + static std::wstring getNodeText(Ref t, const std::vector &ruleNames); /// Return ordered list of all children of this node. - static std::vector> getChildren(std::shared_ptr t); + static std::vector> getChildren(Ref t); /// Return a list of all ancestors of this node. The first node of /// list is the root and the last is the parent of this node. - static std::vector> getAncestors(std::shared_ptr t); - static std::vector> findAllTokenNodes(std::shared_ptr t, int ttype); - static std::vector> findAllRuleNodes(std::shared_ptr t, int ruleIndex); - static std::vector> findAllNodes(std::shared_ptr t, int index, bool findTokens); + static std::vector> getAncestors(Ref t); + static std::vector> findAllTokenNodes(Ref t, int ttype); + static std::vector> findAllRuleNodes(Ref t, int ruleIndex); + static std::vector> findAllNodes(Ref t, int index, bool findTokens); - static std::vector> descendants(std::shared_ptr t); + static std::vector> descendants(Ref t); private: Trees(); diff --git a/runtime/Cpp/runtime/tree/pattern/ParseTreeMatch.cpp b/runtime/Cpp/runtime/tree/pattern/ParseTreeMatch.cpp index eb7ea62dc..0b352f462 100755 --- a/runtime/Cpp/runtime/tree/pattern/ParseTreeMatch.cpp +++ b/runtime/Cpp/runtime/tree/pattern/ParseTreeMatch.cpp @@ -36,16 +36,16 @@ using namespace org::antlr::v4::runtime::tree; using namespace org::antlr::v4::runtime::tree::pattern; -ParseTreeMatch::ParseTreeMatch(std::shared_ptr tree, const ParseTreePattern &pattern, - const std::map>> &labels, - std::shared_ptr mismatchedNode) +ParseTreeMatch::ParseTreeMatch(Ref tree, const ParseTreePattern &pattern, + const std::map>> &labels, + Ref mismatchedNode) : _tree(tree), _pattern(pattern), _labels(labels), _mismatchedNode(mismatchedNode) { if (tree == nullptr) { throw IllegalArgumentException("tree cannot be null"); } } -std::shared_ptr ParseTreeMatch::get(const std::wstring &label) { +Ref ParseTreeMatch::get(const std::wstring &label) { auto iterator = _labels.find(label); if (iterator == _labels.end()) { return nullptr; @@ -54,20 +54,20 @@ std::shared_ptr ParseTreeMatch::get(const std::wstring &label) { return iterator->second.back(); // return last if multiple } -std::vector> ParseTreeMatch::getAll(const std::wstring &label) { +std::vector> ParseTreeMatch::getAll(const std::wstring &label) { auto iterator = _labels.find(label); if (iterator == _labels.end()) { - return std::vector>(); + return std::vector>(); } return iterator->second; } -std::map>>& ParseTreeMatch::getLabels() { +std::map>>& ParseTreeMatch::getLabels() { return _labels; } -std::shared_ptr ParseTreeMatch::getMismatchedNode() { +Ref ParseTreeMatch::getMismatchedNode() { return _mismatchedNode; } @@ -79,7 +79,7 @@ const ParseTreePattern& ParseTreeMatch::getPattern() { return _pattern; } -std::shared_ptr ParseTreeMatch::getTree() { +Ref ParseTreeMatch::getTree() { return _tree; } diff --git a/runtime/Cpp/runtime/tree/pattern/ParseTreeMatch.h b/runtime/Cpp/runtime/tree/pattern/ParseTreeMatch.h index 61bf9bbfd..7adfb2d54 100755 --- a/runtime/Cpp/runtime/tree/pattern/ParseTreeMatch.h +++ b/runtime/Cpp/runtime/tree/pattern/ParseTreeMatch.h @@ -42,16 +42,16 @@ namespace pattern { class ParseTreeMatch { private: /// This is the backing field for getTree(). - std::shared_ptr _tree; + Ref _tree; /// This is the backing field for getPattern(). const ParseTreePattern &_pattern; /// This is the backing field for getLabels(). - std::map>> _labels; + std::map>> _labels; /// This is the backing field for getMismatchedNode(). - std::shared_ptr _mismatchedNode; + Ref _mismatchedNode; /// /// Constructs a new instance of from the specified @@ -68,9 +68,9 @@ namespace pattern { /// if {@code pattern} is {@code null} /// if {@code labels} is {@code null} public: - ParseTreeMatch(std::shared_ptr tree, const ParseTreePattern &pattern, - const std::map>> &labels, - std::shared_ptr mismatchedNode); + ParseTreeMatch(Ref tree, const ParseTreePattern &pattern, + const std::map>> &labels, + Ref mismatchedNode); /// /// Get the last node associated with a specific {@code label}. @@ -87,7 +87,7 @@ namespace pattern { /// /// The last to match a tag with the specified /// label, or {@code null} if no parse tree matched a tag with the label. - virtual std::shared_ptr get(const std::wstring &label); + virtual Ref get(const std::wstring &label); /// /// Return all nodes matching a rule or token tag with the specified label. @@ -111,7 +111,7 @@ namespace pattern { /// A collection of all nodes matching tags with /// the specified {@code label}. If no nodes matched the label, an empty list /// is returned. - virtual std::vector> getAll(const std::wstring &label); + virtual std::vector> getAll(const std::wstring &label); /// /// Return a mapping from label → [list of nodes]. @@ -122,14 +122,14 @@ namespace pattern { /// /// A mapping from labels to parse tree nodes. If the parse tree /// pattern did not contain any rule or token tags, this map will be empty. - virtual std::map>>& getLabels(); + virtual std::map>>& getLabels(); /// /// Get the node at which we first detected a mismatch. /// /// the node at which we first detected a mismatch, or {@code null} /// if the match was successful. - virtual std::shared_ptr getMismatchedNode(); + virtual Ref getMismatchedNode(); /// /// Gets a value indicating whether the match operation succeeded. @@ -148,7 +148,7 @@ namespace pattern { /// Get the parse tree we are trying to match to a pattern. /// /// The we are trying to match to a pattern. - virtual std::shared_ptr getTree(); + virtual Ref getTree(); /// /// {@inheritDoc} diff --git a/runtime/Cpp/runtime/tree/pattern/ParseTreePattern.cpp b/runtime/Cpp/runtime/tree/pattern/ParseTreePattern.cpp index 7609472b4..fdb514518 100755 --- a/runtime/Cpp/runtime/tree/pattern/ParseTreePattern.cpp +++ b/runtime/Cpp/runtime/tree/pattern/ParseTreePattern.cpp @@ -39,15 +39,15 @@ using namespace org::antlr::v4::runtime::tree; using namespace org::antlr::v4::runtime::tree::pattern; ParseTreePattern::ParseTreePattern(ParseTreePatternMatcher *matcher, const std::wstring &pattern, int patternRuleIndex, - std::shared_ptr patternTree) + Ref patternTree) : patternRuleIndex(patternRuleIndex), pattern(pattern), patternTree(patternTree), matcher(matcher) { } -ParseTreeMatch ParseTreePattern::match(std::shared_ptr tree) { +ParseTreeMatch ParseTreePattern::match(Ref tree) { return matcher->match(tree, *this); } -bool ParseTreePattern::matches(std::shared_ptr tree) { +bool ParseTreePattern::matches(Ref tree) { return matcher->match(tree, *this).succeeded(); } @@ -77,6 +77,6 @@ int ParseTreePattern::getPatternRuleIndex() const { return patternRuleIndex; } -std::shared_ptr ParseTreePattern::getPatternTree() const { +Ref ParseTreePattern::getPatternTree() const { return patternTree; } diff --git a/runtime/Cpp/runtime/tree/pattern/ParseTreePattern.h b/runtime/Cpp/runtime/tree/pattern/ParseTreePattern.h index 2a06cc38e..1a3293ce1 100755 --- a/runtime/Cpp/runtime/tree/pattern/ParseTreePattern.h +++ b/runtime/Cpp/runtime/tree/pattern/ParseTreePattern.h @@ -54,7 +54,7 @@ namespace pattern { /// tree pattern. /// The tree pattern in form. ParseTreePattern(ParseTreePatternMatcher *matcher, const std::wstring &pattern, int patternRuleIndex, - std::shared_ptr patternTree); + Ref patternTree); /// /// Match a specific parse tree against this tree pattern. @@ -63,7 +63,7 @@ namespace pattern { /// A object describing the result of the /// match operation. The method can be /// used to determine whether or not the match was successful. - virtual ParseTreeMatch match(std::shared_ptr tree); + virtual ParseTreeMatch match(Ref tree); /// /// Determine whether or not a parse tree matches this tree pattern. @@ -71,7 +71,7 @@ namespace pattern { /// The parse tree to match against this tree pattern. /// {@code true} if {@code tree} is a match for the current tree /// pattern; otherwise, {@code false}. - virtual bool matches(std::shared_ptr tree); + virtual bool matches(Ref tree); /// Find all nodes using XPath and then try to match those subtrees against /// this tree pattern. @@ -113,7 +113,7 @@ namespace pattern { /// of type or . /// /// The tree pattern as a . - virtual std::shared_ptr getPatternTree() const; + virtual Ref getPatternTree() const; private: const int patternRuleIndex; @@ -124,7 +124,7 @@ namespace pattern { const std::wstring pattern; /// This is the backing field for . - std::shared_ptr patternTree; + Ref patternTree; /// /// This is the backing field for . diff --git a/runtime/Cpp/runtime/tree/pattern/ParseTreePatternMatcher.cpp b/runtime/Cpp/runtime/tree/pattern/ParseTreePatternMatcher.cpp index 63844d67f..04a93816e 100755 --- a/runtime/Cpp/runtime/tree/pattern/ParseTreePatternMatcher.cpp +++ b/runtime/Cpp/runtime/tree/pattern/ParseTreePatternMatcher.cpp @@ -77,30 +77,30 @@ void ParseTreePatternMatcher::setDelimiters(const std::wstring &start, const std _escape = escapeLeft; } -bool ParseTreePatternMatcher::matches(std::shared_ptr tree, const std::wstring &pattern, int patternRuleIndex) { +bool ParseTreePatternMatcher::matches(Ref tree, const std::wstring &pattern, int patternRuleIndex) { ParseTreePattern p = compile(pattern, patternRuleIndex); return matches(tree, p); } -bool ParseTreePatternMatcher::matches(std::shared_ptr tree, const ParseTreePattern &pattern) { - std::map>> labels; - std::shared_ptr mismatchedNode = matchImpl(tree, pattern.getPatternTree(), labels); +bool ParseTreePatternMatcher::matches(Ref tree, const ParseTreePattern &pattern) { + std::map>> labels; + Ref mismatchedNode = matchImpl(tree, pattern.getPatternTree(), labels); return mismatchedNode == nullptr; } -ParseTreeMatch ParseTreePatternMatcher::match(std::shared_ptr tree, const std::wstring &pattern, int patternRuleIndex) { +ParseTreeMatch ParseTreePatternMatcher::match(Ref tree, const std::wstring &pattern, int patternRuleIndex) { ParseTreePattern p = compile(pattern, patternRuleIndex); return match(tree, p); } -ParseTreeMatch ParseTreePatternMatcher::match(std::shared_ptr tree, const ParseTreePattern &pattern) { - std::map>> labels; - std::shared_ptr mismatchedNode = matchImpl(tree, pattern.getPatternTree(), labels); +ParseTreeMatch ParseTreePatternMatcher::match(Ref tree, const ParseTreePattern &pattern) { + std::map>> labels; + Ref mismatchedNode = matchImpl(tree, pattern.getPatternTree(), labels); return ParseTreeMatch(tree, pattern, labels, mismatchedNode); } ParseTreePattern ParseTreePatternMatcher::compile(const std::wstring &pattern, int patternRuleIndex) { - std::vector tokenList = tokenize(pattern); + std::vector> tokenList = tokenize(pattern); ListTokenSource *tokenSrc = new ListTokenSource(tokenList); /* mem-check: deleted in finally block */ CommonTokenStream *tokens = new CommonTokenStream(tokenSrc); /* mem-check: deleted in finally block */ @@ -113,7 +113,7 @@ ParseTreePattern ParseTreePatternMatcher::compile(const std::wstring &pattern, i _parser->getRuleNames(), _parser->getATNWithBypassAlts(), tokens); try { - ParserRuleContext::Ref context = parserInterp.parse(patternRuleIndex); + Ref context = parserInterp.parse(patternRuleIndex); return ParseTreePattern(this, pattern, patternRuleIndex, context); } catch (std::exception &e) { #if defined(_MSC_FULL_VER) && _MSC_FULL_VER < 190023026 @@ -134,8 +134,8 @@ Parser* ParseTreePatternMatcher::getParser() { return _parser; } -std::shared_ptr ParseTreePatternMatcher::matchImpl(std::shared_ptr tree, - std::shared_ptr patternTree, std::map>> &labels) { +Ref ParseTreePatternMatcher::matchImpl(Ref tree, + Ref patternTree, std::map>> &labels) { if (tree == nullptr) { throw IllegalArgumentException("tree cannot be null"); } @@ -146,14 +146,14 @@ std::shared_ptr ParseTreePatternMatcher::matchImpl(std::shared_ptr, x and y, or x and x; or could be mismatched types if (is(tree) && is(patternTree)) { - std::shared_ptr t1 = std::static_pointer_cast(tree); - std::shared_ptr t2 = std::static_pointer_cast(patternTree); + Ref t1 = std::static_pointer_cast(tree); + Ref t2 = std::static_pointer_cast(patternTree); - std::shared_ptr mismatchedNode; + Ref mismatchedNode; // both are tokens and they have same type if (t1->getSymbol()->getType() == t2->getSymbol()->getType()) { if (is(t2->getSymbol())) { // x and - std::shared_ptr tokenTagToken = std::dynamic_pointer_cast(t2->getSymbol()); + Ref tokenTagToken = std::dynamic_pointer_cast(t2->getSymbol()); // track label->list-of-nodes for both token name and label (if any) labels[tokenTagToken->getTokenName()].push_back(tree); @@ -178,12 +178,12 @@ std::shared_ptr ParseTreePatternMatcher::matchImpl(std::shared_ptr(tree) && is(patternTree)) { - ParserRuleContext::Ref r1 = std::dynamic_pointer_cast(tree); - ParserRuleContext::Ref r2 = std::dynamic_pointer_cast(patternTree); - std::shared_ptr mismatchedNode; + Ref r1 = std::dynamic_pointer_cast(tree); + Ref r2 = std::dynamic_pointer_cast(patternTree); + Ref mismatchedNode; // (expr ...) and - std::shared_ptr ruleTagToken = getRuleTagToken(r2); + Ref ruleTagToken = getRuleTagToken(r2); if (ruleTagToken != nullptr) { //ParseTreeMatch *m = nullptr; // unused? if (r1->RuleContext::getRuleContext()->getRuleIndex() == r2->RuleContext::getRuleContext()->getRuleIndex()) { @@ -212,7 +212,7 @@ std::shared_ptr ParseTreePatternMatcher::matchImpl(std::shared_ptrgetChildCount(); for (size_t i = 0; i < n; i++) { - std::shared_ptr childMatch = matchImpl(r1->getChild(i), patternTree->getChild(i), labels); + Ref childMatch = matchImpl(r1->getChild(i), patternTree->getChild(i), labels); if (childMatch) { return childMatch; } @@ -225,11 +225,11 @@ std::shared_ptr ParseTreePatternMatcher::matchImpl(std::shared_ptr ParseTreePatternMatcher::getRuleTagToken(std::shared_ptr t) { +Ref ParseTreePatternMatcher::getRuleTagToken(Ref t) { if (is(t)) { - std::shared_ptr r = std::dynamic_pointer_cast(t); + Ref r = std::dynamic_pointer_cast(t); if (r->getChildCount() == 1 && is(r->getChild(0))) { - std::shared_ptr c = std::dynamic_pointer_cast(r->getChild(0)); + Ref c = std::dynamic_pointer_cast(r->getChild(0)); if (is(c->getSymbol())) { return std::dynamic_pointer_cast(c->getSymbol()); } @@ -238,12 +238,12 @@ std::shared_ptr ParseTreePatternMatcher::getRuleTagToken(std::shar return nullptr; } -std::vector ParseTreePatternMatcher::tokenize(const std::wstring &pattern) { +std::vector> ParseTreePatternMatcher::tokenize(const std::wstring &pattern) { // split pattern into chunks: sea (raw input) and islands (, ) std::vector chunks = split(pattern); // create token stream from text and tags - std::vector tokens; + std::vector> tokens; for (auto chunk : chunks) { if (is(chunk)) { TagChunk &tagChunk = (TagChunk&)chunk; @@ -254,7 +254,7 @@ std::vector ParseTreePatternMatcher::tokenize(const std::wstring &pa throw IllegalArgumentException("Unknown token " + antlrcpp::ws2s(tagChunk.getTag()) + " in pattern: " + antlrcpp::ws2s(pattern)); } - std::shared_ptr t = std::make_shared(tagChunk.getTag(), (int)ttype, tagChunk.getLabel()); + Ref t = std::make_shared(tagChunk.getTag(), (int)ttype, tagChunk.getLabel()); tokens.push_back(t); } else if (islower(tagChunk.getTag()[0])) { ssize_t ruleIndex = _parser->getRuleIndex(tagChunk.getTag()); @@ -270,7 +270,7 @@ std::vector ParseTreePatternMatcher::tokenize(const std::wstring &pa TextChunk &textChunk = (TextChunk&)chunk; ANTLRInputStream input(textChunk.getText()); _lexer->setInputStream(&input); - Token::Ref t = _lexer->nextToken(); + Ref t = _lexer->nextToken(); while (t->getType() != EOF) { tokens.push_back(t); t = _lexer->nextToken(); diff --git a/runtime/Cpp/runtime/tree/pattern/ParseTreePatternMatcher.h b/runtime/Cpp/runtime/tree/pattern/ParseTreePatternMatcher.h index e1e5f3a69..dfff7381e 100755 --- a/runtime/Cpp/runtime/tree/pattern/ParseTreePatternMatcher.h +++ b/runtime/Cpp/runtime/tree/pattern/ParseTreePatternMatcher.h @@ -124,20 +124,20 @@ namespace pattern { /// /// Does {@code pattern} matched as rule {@code patternRuleIndex} match {@code tree}? - virtual bool matches(std::shared_ptr tree, const std::wstring &pattern, int patternRuleIndex); + virtual bool matches(Ref tree, const std::wstring &pattern, int patternRuleIndex); /// /// Does {@code pattern} matched as rule patternRuleIndex match tree? Pass in a /// compiled pattern instead of a string representation of a tree pattern. /// - virtual bool matches(std::shared_ptr tree, const ParseTreePattern &pattern); + virtual bool matches(Ref tree, const ParseTreePattern &pattern); /// /// Compare {@code pattern} matched as rule {@code patternRuleIndex} against /// {@code tree} and return a object that contains the /// matched elements, or the node at which the match failed. /// - virtual ParseTreeMatch match(std::shared_ptr tree, const std::wstring &pattern, int patternRuleIndex); + virtual ParseTreeMatch match(Ref tree, const std::wstring &pattern, int patternRuleIndex); /// /// Compare {@code pattern} matched against {@code tree} and return a @@ -145,7 +145,7 @@ namespace pattern { /// node at which the match failed. Pass in a compiled pattern instead of a /// string representation of a tree pattern. /// - virtual ParseTreeMatch match(std::shared_ptr tree, const ParseTreePattern &pattern); + virtual ParseTreeMatch match(Ref tree, const ParseTreePattern &pattern); /// /// For repeated use of a tree pattern, compile it to a @@ -167,7 +167,7 @@ namespace pattern { // ---- SUPPORT CODE ---- - virtual std::vector tokenize(const std::wstring &pattern); + virtual std::vector> tokenize(const std::wstring &pattern); /// Split " = ;" into 4 chunks for tokenizing by tokenize(). virtual std::vector split(const std::wstring &pattern); @@ -184,11 +184,11 @@ namespace pattern { /// a corresponding node in {@code patternTree}, or {@code null} if the match /// was successful. The specific node returned depends on the matching /// algorithm used by the implementation, and may be overridden. - virtual std::shared_ptr matchImpl(std::shared_ptr tree, - std::shared_ptr patternTree, std::map>> &labels); + virtual Ref matchImpl(Ref tree, + Ref patternTree, std::map>> &labels); /// Is t subtree? - virtual std::shared_ptr getRuleTagToken(std::shared_ptr t); + virtual Ref getRuleTagToken(Ref t); private: Lexer *_lexer;