From 10f8cac7f4b7accd7a647e1096c85241f0687805 Mon Sep 17 00:00:00 2001 From: Andreas Skaar Date: Mon, 8 Oct 2018 22:05:05 +0200 Subject: [PATCH 001/365] Update runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/SingletonPredictionContext.cs NullReferenceException in CSharp target #2304 --- .../CSharp/Antlr4.Runtime/Atn/SingletonPredictionContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/SingletonPredictionContext.cs b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/SingletonPredictionContext.cs index 2f8f1bb5d..e162dcaca 100644 --- a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/SingletonPredictionContext.cs +++ b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/SingletonPredictionContext.cs @@ -79,7 +79,7 @@ namespace Antlr4.Runtime.Atn return false; } Antlr4.Runtime.Atn.SingletonPredictionContext other = (Antlr4.Runtime.Atn.SingletonPredictionContext)o; - return returnState == other.returnState && parent.Equals(other.parent); + return returnState == other.returnState && (parent != null && parent.Equals(other.parent)); } public override string ToString() From 8b458f85c02fa93e9e216c7899a28776f00185c9 Mon Sep 17 00:00:00 2001 From: Andreas Skaar Date: Mon, 8 Oct 2018 22:15:05 +0200 Subject: [PATCH 002/365] Update contributors.txt sign of contributors.txt --- contributors.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contributors.txt b/contributors.txt index 3a223169c..8afdd597d 100644 --- a/contributors.txt +++ b/contributors.txt @@ -192,4 +192,5 @@ YYYY/MM/DD, github id, Full name, email 2018/06/16, EternalPhane, Zongyuan Zuo, eternalphane@gmail.com 2018/07/03, jgoppert, James Goppert, james.goppert@gmail.com 2018/07/27, Maksim Novikov, mnovikov.work@gmail.com -2018/08/03, ENDOH takanao, djmchl@gmail.com \ No newline at end of file +2018/08/03, ENDOH takanao, djmchl@gmail.com +2018/10/08, xsIceman, Andreas Skaar, andreas.skaar@gmail.com \ No newline at end of file From ec665c1b905ee2a1287a0f3e069f4fe46ed53a16 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Thu, 3 Jan 2019 10:39:59 +0100 Subject: [PATCH 003/365] A small fix in the C++ documentation. --- doc/cpp-target.md | 2 +- runtime/Cpp/demo/Mac/antlrcpp-demo.xcodeproj/project.pbxproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/cpp-target.md b/doc/cpp-target.md index 2933d606e..eec7cf88b 100644 --- a/doc/cpp-target.md +++ b/doc/cpp-target.md @@ -45,7 +45,7 @@ The generation step above created a listener and base listener class for you. Th #include "MyGrammarParser.h" #include "MyGrammarBaseListener.h" -using namespace org::antlr::v4::runtime; +using namespace antlr4; class TreeShapeListener : public MyGrammarBaseListener { public: diff --git a/runtime/Cpp/demo/Mac/antlrcpp-demo.xcodeproj/project.pbxproj b/runtime/Cpp/demo/Mac/antlrcpp-demo.xcodeproj/project.pbxproj index 77ad3f4f7..5f136b03e 100644 --- a/runtime/Cpp/demo/Mac/antlrcpp-demo.xcodeproj/project.pbxproj +++ b/runtime/Cpp/demo/Mac/antlrcpp-demo.xcodeproj/project.pbxproj @@ -332,7 +332,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "pushd ..\nif [ TParser.g4 -nt generated/TParser.cpp -o TLexer.g4 -nt generated/TLexer.cpp ]; then\n./generate.sh;\nfi\npopd"; + shellScript = "pushd ..\nif [ TParser.g4 -nt generated/TParser.cpp -o TLexer.g4 -nt generated/TLexer.cpp ]; then\n./generate.sh;\nfi\npopd\n"; showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ From 3cd2878b28cae0a04b3fa251d1af3c7155bbbd33 Mon Sep 17 00:00:00 2001 From: Wail Khemir Date: Thu, 3 Jan 2019 15:26:47 -0500 Subject: [PATCH 004/365] Fix code generation for catch statements in python --- contributors.txt | 3 ++- .../org/antlr/v4/tool/templates/codegen/Python2/Python2.stg | 3 +-- .../org/antlr/v4/tool/templates/codegen/Python3/Python3.stg | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/contributors.txt b/contributors.txt index f48052823..ebda19f64 100644 --- a/contributors.txt +++ b/contributors.txt @@ -210,4 +210,5 @@ YYYY/MM/DD, github id, Full name, email 2018/11/14, nxtstep, Adriaan (Arjan) Duz, codewithadriaan[et]gmail[dot]com 2018/11/15, amykyta3, Alex Mykyta, amykyta3@users.noreply.github.com 2018/11/29, hannemann-tamas, Ralf Hannemann-Tamas, ralf.ht@gmail.com -2018/12/20, WalterCouto, Walter Couto, WalterCouto@users.noreply.github.com \ No newline at end of file +2018/12/20, WalterCouto, Walter Couto, WalterCouto@users.noreply.github.com +2019/01/02, wkhemir, Wail Khemir, khemirwail@gmail.com diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg index b56b8cbdb..b3aadfd55 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg @@ -517,9 +517,8 @@ if not : >> ExceptionClause(e, catchArg, catchAction) ::= << -catch () { +except : -} >> // lexer actions are not associated with model objects diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg index 650d0002e..b2b23a19d 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg @@ -529,9 +529,8 @@ if not : >> ExceptionClause(e, catchArg, catchAction) ::= << -catch () { +except : -} >> // lexer actions are not associated with model objects From 4fec5f9f7cd60b11a28892ab9ec82c7cb314024f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Zancolo=CC=80?= Date: Wed, 16 Jan 2019 13:29:57 +0100 Subject: [PATCH 005/365] fix cpp cmake for emscripten Emscripten can do C++11. added it to the compilercheck in CMakeLists --- contributors.txt | 1 + runtime/Cpp/CMakeLists.txt | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/contributors.txt b/contributors.txt index c54c0279e..887bb7095 100644 --- a/contributors.txt +++ b/contributors.txt @@ -212,3 +212,4 @@ YYYY/MM/DD, github id, Full name, email 2018/11/29, hannemann-tamas, Ralf Hannemann-Tamas, ralf.ht@gmail.com 2018/12/20, WalterCouto, Walter Couto, WalterCouto@users.noreply.github.com 2018/12/23, youkaichao, Kaichao You, youkaichao@gmail.com +2019/01/16, kuegi, Markus Zancolo, markus.zancolo@roomle.com diff --git a/runtime/Cpp/CMakeLists.txt b/runtime/Cpp/CMakeLists.txt index 28b19a195..390078151 100644 --- a/runtime/Cpp/CMakeLists.txt +++ b/runtime/Cpp/CMakeLists.txt @@ -122,7 +122,9 @@ elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" AND ( CMAKE_SYSTEM_NAME MATCH endif() elseif(MSVC_VERSION GREATER 1800 OR MSVC_VERSION EQUAL 1800) # Visual Studio 2012+ supports c++11 features -else () +elseif(CMAKE_SYSTEM_NAME MATCHES "Emscripten") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++") +else() message(FATAL_ERROR "Your C++ compiler does not support C++11.") endif() From d69e7c4d0fed284c720e09f326a8332837cb1729 Mon Sep 17 00:00:00 2001 From: Gerasimos Dimitriadis Date: Sat, 23 Feb 2019 20:31:19 +0200 Subject: [PATCH 006/365] Avoid double deletion when NoViableAltException is thrown When ownership of the configurations set is passed to a NoViableAltException, it get deleted more than once if the exception happens to get copied. --- contributors.txt | 1 + .../Cpp/runtime/src/NoViableAltException.cpp | 20 +++++++++++++++---- .../Cpp/runtime/src/NoViableAltException.h | 7 +++---- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/contributors.txt b/contributors.txt index a35b20200..58c23e4ae 100644 --- a/contributors.txt +++ b/contributors.txt @@ -213,3 +213,4 @@ YYYY/MM/DD, github id, Full name, email 2018/12/20, WalterCouto, Walter Couto, WalterCouto@users.noreply.github.com 2018/12/23, youkaichao, Kaichao You, youkaichao@gmail.com 2019/02/06, ralucado, Cristina Raluca Vijulie, ralucris.v[at]gmail[dot]com +2019/02/23, gedimitr, Gerasimos Dimitriadis, gedimitr@gmail.com diff --git a/runtime/Cpp/runtime/src/NoViableAltException.cpp b/runtime/Cpp/runtime/src/NoViableAltException.cpp index ced7f827f..273c208c7 100755 --- a/runtime/Cpp/runtime/src/NoViableAltException.cpp +++ b/runtime/Cpp/runtime/src/NoViableAltException.cpp @@ -9,6 +9,20 @@ using namespace antlr4; +namespace { + +// Create a normal shared pointer if the configurations are to be deleted. If not, then +// the shared pointer is created with a deleter that does nothing. +Ref buildConfigsRef(atn::ATNConfigSet *configs, bool deleteConfigs) { + if (deleteConfigs) { + return Ref(configs); + } else { + return Ref(configs, [](atn::ATNConfigSet *){}); + } +} + +} + NoViableAltException::NoViableAltException(Parser *recognizer) : NoViableAltException(recognizer, recognizer->getTokenStream(), recognizer->getCurrentToken(), recognizer->getCurrentToken(), nullptr, recognizer->getContext(), false) { @@ -17,12 +31,10 @@ NoViableAltException::NoViableAltException(Parser *recognizer) NoViableAltException::NoViableAltException(Parser *recognizer, TokenStream *input,Token *startToken, Token *offendingToken, atn::ATNConfigSet *deadEndConfigs, ParserRuleContext *ctx, bool deleteConfigs) : RecognitionException("No viable alternative", recognizer, input, ctx, offendingToken), - _deadEndConfigs(deadEndConfigs), _startToken(startToken), _deleteConfigs(deleteConfigs) { + _deadEndConfigs(buildConfigsRef(deadEndConfigs, deleteConfigs)), _startToken(startToken) { } NoViableAltException::~NoViableAltException() { - if (_deleteConfigs) - delete _deadEndConfigs; } Token* NoViableAltException::getStartToken() const { @@ -30,5 +42,5 @@ Token* NoViableAltException::getStartToken() const { } atn::ATNConfigSet* NoViableAltException::getDeadEndConfigs() const { - return _deadEndConfigs; + return _deadEndConfigs.get(); } diff --git a/runtime/Cpp/runtime/src/NoViableAltException.h b/runtime/Cpp/runtime/src/NoViableAltException.h index 94d43c54c..b15039d0c 100755 --- a/runtime/Cpp/runtime/src/NoViableAltException.h +++ b/runtime/Cpp/runtime/src/NoViableAltException.h @@ -27,10 +27,9 @@ namespace antlr4 { private: /// Which configurations did we try at input.index() that couldn't match input.LT(1)? - atn::ATNConfigSet* _deadEndConfigs; - - // Flag that indicates if we own the dead end config set and have to delete it on destruction. - bool _deleteConfigs; + /// Shared pointer that conditionally deletes the configurations (based on flag + /// passed during construction) + Ref _deadEndConfigs; /// The token object at the start index; the input stream might /// not be buffering tokens so get a reference to it. (At the From 4cf4883514bcea58053baa10386a70ed39fd5068 Mon Sep 17 00:00:00 2001 From: Alexei Averchenko Date: Tue, 9 Apr 2019 09:20:04 +0300 Subject: [PATCH 007/365] Fixed -Wreorder warning --- runtime/Cpp/runtime/src/NoViableAltException.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/Cpp/runtime/src/NoViableAltException.cpp b/runtime/Cpp/runtime/src/NoViableAltException.cpp index ced7f827f..b00bc69e8 100755 --- a/runtime/Cpp/runtime/src/NoViableAltException.cpp +++ b/runtime/Cpp/runtime/src/NoViableAltException.cpp @@ -17,7 +17,7 @@ NoViableAltException::NoViableAltException(Parser *recognizer) NoViableAltException::NoViableAltException(Parser *recognizer, TokenStream *input,Token *startToken, Token *offendingToken, atn::ATNConfigSet *deadEndConfigs, ParserRuleContext *ctx, bool deleteConfigs) : RecognitionException("No viable alternative", recognizer, input, ctx, offendingToken), - _deadEndConfigs(deadEndConfigs), _startToken(startToken), _deleteConfigs(deleteConfigs) { + _deadEndConfigs(deadEndConfigs), _deleteConfigs(deleteConfigs), _startToken(startToken) { } NoViableAltException::~NoViableAltException() { From d0f0ef12fd99e430ccb75f7685ca6882ff89f57d Mon Sep 17 00:00:00 2001 From: Alexei Averchenko Date: Tue, 9 Apr 2019 09:24:08 +0300 Subject: [PATCH 008/365] Signed contributors.txt --- contributors.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contributors.txt b/contributors.txt index 54ff875dc..b3f0169db 100644 --- a/contributors.txt +++ b/contributors.txt @@ -215,3 +215,4 @@ YYYY/MM/DD, github id, Full name, email 2019/02/06, ralucado, Cristina Raluca Vijulie, ralucris.v[at]gmail[dot]com 2019/03/13, base698, Justin Thomas, justin.thomas1@gmail.com 2019/03/18, carlodri, Carlo Dri, carlo.dri@gmail.com +2019/04/09, a-square, Alexei Averchenko, echo 'bGV4LmF2ZXJAZ21haWwuY29tCg==' | base64 -D From 31f4dcc7cfe44e81ef00c6ee0aafb1581f30cadc Mon Sep 17 00:00:00 2001 From: WalterCouto Date: Wed, 17 Apr 2019 20:10:37 -0400 Subject: [PATCH 009/365] don't unnecessary fetch all tokens when grabbing an interval of text (match C# logic) When an exception is thrown, it calls getText to get the text of the tokens invloved but the current C++ and Java implemention first fetches the rest of the tokens instead of just using the tokens it already has. C#'s version is correct (fill() is called only when asking for the entire stream's text). --- runtime/Cpp/runtime/src/BufferedTokenStream.cpp | 5 +++-- .../Java/src/org/antlr/v4/runtime/BufferedTokenStream.java | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/runtime/Cpp/runtime/src/BufferedTokenStream.cpp b/runtime/Cpp/runtime/src/BufferedTokenStream.cpp index d2a88dd6c..241dfe5c4 100755 --- a/runtime/Cpp/runtime/src/BufferedTokenStream.cpp +++ b/runtime/Cpp/runtime/src/BufferedTokenStream.cpp @@ -272,7 +272,7 @@ ssize_t BufferedTokenStream::previousTokenOnChannel(size_t i, size_t channel) { } if (i == 0) - break; + return -1; i--; } return i; @@ -358,17 +358,18 @@ std::string BufferedTokenStream::getSourceName() const } std::string BufferedTokenStream::getText() { + fill(); return getText(misc::Interval(0U, size() - 1)); } std::string BufferedTokenStream::getText(const misc::Interval &interval) { lazyInit(); - fill(); size_t start = interval.a; size_t stop = interval.b; if (start == INVALID_INDEX || stop == INVALID_INDEX) { return ""; } + sync(stop); if (stop >= _tokens.size()) { stop = _tokens.size() - 1; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index e1dc4b197..907d59a3b 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -441,15 +441,17 @@ public class BufferedTokenStream implements TokenStream { @Override public String getText() { + fill(); return getText(Interval.of(0,size()-1)); } @Override public String getText(Interval interval) { + lazyInit(); int start = interval.a; int stop = interval.b; if ( start<0 || stop<0 ) return ""; - fill(); + sync(stop); if ( stop>=tokens.size() ) stop = tokens.size()-1; StringBuilder buf = new StringBuilder(); From 115274e77124e88a011fe68ebe91a91dad94f088 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Fri, 12 Jul 2019 15:15:17 -0700 Subject: [PATCH 010/365] Remove unnecessary Travis files. Remove .travis/before-install-linux-{cpp,java,python2,python3}.sh, and remove some lines from before-install-linux-{javascript,go}.sh. These contained apt commands for configuring the Mono project's GPG key, which is not used in any of these builds, and pointless apt-get update calls. --- .travis/before-install-linux-cpp.sh | 6 ------ .travis/before-install-linux-go.sh | 2 -- .travis/before-install-linux-java.sh | 6 ------ .travis/before-install-linux-javascript.sh | 1 - .travis/before-install-linux-python2.sh | 7 ------- .travis/before-install-linux-python3.sh | 5 ----- .travis/run-tests-python2.sh | 4 +++- .travis/run-tests-python3.sh | 4 +++- 8 files changed, 6 insertions(+), 29 deletions(-) delete mode 100755 .travis/before-install-linux-cpp.sh delete mode 100755 .travis/before-install-linux-java.sh delete mode 100755 .travis/before-install-linux-python2.sh delete mode 100755 .travis/before-install-linux-python3.sh diff --git a/.travis/before-install-linux-cpp.sh b/.travis/before-install-linux-cpp.sh deleted file mode 100755 index c496c2cbd..000000000 --- a/.travis/before-install-linux-cpp.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -sudo apt-get update -qq diff --git a/.travis/before-install-linux-go.sh b/.travis/before-install-linux-go.sh index 71adcfadd..16c828180 100755 --- a/.travis/before-install-linux-go.sh +++ b/.travis/before-install-linux-go.sh @@ -2,7 +2,5 @@ set -euo pipefail -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -sudo apt-get update -qq eval "$(sudo gimme 1.7.3)" ( go version ; go env ) || true diff --git a/.travis/before-install-linux-java.sh b/.travis/before-install-linux-java.sh deleted file mode 100755 index c496c2cbd..000000000 --- a/.travis/before-install-linux-java.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -sudo apt-get update -qq diff --git a/.travis/before-install-linux-javascript.sh b/.travis/before-install-linux-javascript.sh index c70757d53..28b59a6fc 100755 --- a/.travis/before-install-linux-javascript.sh +++ b/.travis/before-install-linux-javascript.sh @@ -2,7 +2,6 @@ set -euo pipefail -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF sudo apt-get update -qq curl -sL https://deb.nodesource.com/setup_0.12 | sudo -E bash - sudo apt-get install -qq nodejs diff --git a/.travis/before-install-linux-python2.sh b/.travis/before-install-linux-python2.sh deleted file mode 100755 index d79e21610..000000000 --- a/.travis/before-install-linux-python2.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -sudo apt-get update -qq -python --version diff --git a/.travis/before-install-linux-python3.sh b/.travis/before-install-linux-python3.sh deleted file mode 100755 index a3b091260..000000000 --- a/.travis/before-install-linux-python3.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -python3 --version diff --git a/.travis/run-tests-python2.sh b/.travis/run-tests-python2.sh index d2a22dc77..c5cd0ca99 100755 --- a/.travis/run-tests-python2.sh +++ b/.travis/run-tests-python2.sh @@ -2,8 +2,10 @@ set -euo pipefail +python --version + mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=python2.* test cd ../runtime/Python2/tests -python run.py \ No newline at end of file +python run.py diff --git a/.travis/run-tests-python3.sh b/.travis/run-tests-python3.sh index 9b528f35b..8b74928c5 100755 --- a/.travis/run-tests-python3.sh +++ b/.travis/run-tests-python3.sh @@ -2,8 +2,10 @@ set -euo pipefail +python3 --version + mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=python3.* test cd ../runtime/Python3/test -python3.6 run.py \ No newline at end of file +python3 run.py From d667b90bc9d30dee7940014c52740b4c5d73d0b9 Mon Sep 17 00:00:00 2001 From: Iman Hosseini Date: Sun, 28 Jul 2019 02:25:42 +0430 Subject: [PATCH 011/365] Fixed TextIO compatibility with 3.6+ Fixing this issue: https://github.com/antlr/antlr4/issues/2611 (and probably https://github.com/antlr/antlr4/issues/2193) TextIO has moved since Python 3.6 so it should be imported considering the change for new versions. --- .../org/antlr/v4/tool/templates/codegen/Python3/Python3.stg | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg index ecd6af0f5..701a09a28 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg @@ -51,8 +51,11 @@ ParserFile(file, parser, namedActions, contextSuperClass) ::= << # encoding: utf-8 from antlr4 import * from io import StringIO -from typing.io import TextIO import sys +if sys.version_info[1] > 5: + from typing import TextIO +else: + from typing.io import TextIO From e2ee6e86dfc385247b6efcaaff87dc6e8485af98 Mon Sep 17 00:00:00 2001 From: Eason Lin Date: Mon, 12 Aug 2019 18:01:09 +0800 Subject: [PATCH 012/365] [Golang] enable syntax highlighting --- doc/go-target.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/go-target.md b/doc/go-target.md index 4f7e64e05..695f1564f 100644 --- a/doc/go-target.md +++ b/doc/go-target.md @@ -65,7 +65,7 @@ Another common option to the ANTLR tool is `-visitor`, which generates a parse t We'll write a small main func to call the generated parser/lexer (assuming they are separate). This one writes out the encountered `ParseTreeContext`'s. Suppose the gen'ed parser code is in the `parser` directory relative to this code: -``` +```golang package main import ( From 0fa34a9621989985ef1002468f15843e75f60e1f Mon Sep 17 00:00:00 2001 From: Eason Lin Date: Mon, 12 Aug 2019 18:06:56 +0800 Subject: [PATCH 013/365] sign contributors.txt --- contributors.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contributors.txt b/contributors.txt index d40bacd5f..b917153d6 100644 --- a/contributors.txt +++ b/contributors.txt @@ -221,3 +221,4 @@ YYYY/MM/DD, github id, Full name, email 2019/07/11, olowo726, Olof Wolgast, olof@baah.se 2019/07/16, abhijithneilabraham, Abhijith Neil Abraham, abhijithneilabrahampk@gmail.com 2019/07/26, Braavos96, Eric Hettiaratchi, erichettiaratchi@gmail.com +2019/08/12, easonlin404, Eason Lin, easonlin404@gmail.com From d93857c8febaf9687be4b150817d0b8b276eb6e2 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Sun, 18 Aug 2019 12:27:50 +0200 Subject: [PATCH 014/365] Applied recommended settings after XCode upgrade. --- runtime/Cpp/runtime/antlrcpp.xcodeproj/project.pbxproj | 7 +++++-- .../xcshareddata/xcschemes/antlr4.xcscheme | 2 +- .../xcshareddata/xcschemes/antlr4_ios.xcscheme | 2 +- .../xcshareddata/xcschemes/antlr4_static.xcscheme | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/runtime/Cpp/runtime/antlrcpp.xcodeproj/project.pbxproj b/runtime/Cpp/runtime/antlrcpp.xcodeproj/project.pbxproj index d6822dfdd..a256e089a 100644 --- a/runtime/Cpp/runtime/antlrcpp.xcodeproj/project.pbxproj +++ b/runtime/Cpp/runtime/antlrcpp.xcodeproj/project.pbxproj @@ -2226,7 +2226,7 @@ 37D727A21867AF1E007B6D10 /* Project object */ = { isa = PBXProject; attributes = { - LastUpgradeCheck = 1010; + LastUpgradeCheck = 1030; ORGANIZATIONNAME = ANTLR; TargetAttributes = { 270C67EF1CDB4F1E00116E17 = { @@ -2239,10 +2239,11 @@ }; buildConfigurationList = 37D727A51867AF1E007B6D10 /* Build configuration list for PBXProject "antlrcpp" */; compatibilityVersion = "Xcode 3.2"; - developmentRegion = English; + developmentRegion = en; hasScannedForEncodings = 0; knownRegions = ( en, + Base, ); mainGroup = 37D727A11867AF1E007B6D10; productRefGroup = 37D727AB1867AF1E007B6D10 /* Products */; @@ -2839,6 +2840,7 @@ isa = XCBuildConfiguration; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_LOCALIZABILITY_NONLOCALIZED = YES; CLANG_CXX_LANGUAGE_STANDARD = "c++0x"; CLANG_CXX_LIBRARY = "libc++"; CLANG_ENABLE_OBJC_ARC = YES; @@ -2898,6 +2900,7 @@ isa = XCBuildConfiguration; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_LOCALIZABILITY_NONLOCALIZED = YES; CLANG_CXX_LANGUAGE_STANDARD = "c++0x"; CLANG_CXX_LIBRARY = "libc++"; CLANG_ENABLE_OBJC_ARC = YES; diff --git a/runtime/Cpp/runtime/antlrcpp.xcodeproj/xcshareddata/xcschemes/antlr4.xcscheme b/runtime/Cpp/runtime/antlrcpp.xcodeproj/xcshareddata/xcschemes/antlr4.xcscheme index 740698b37..f2e132436 100644 --- a/runtime/Cpp/runtime/antlrcpp.xcodeproj/xcshareddata/xcschemes/antlr4.xcscheme +++ b/runtime/Cpp/runtime/antlrcpp.xcodeproj/xcshareddata/xcschemes/antlr4.xcscheme @@ -1,6 +1,6 @@ Date: Mon, 19 Aug 2019 22:06:58 +0800 Subject: [PATCH 015/365] add cmd line option --- .travis/before-install-linux-dotnet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/before-install-linux-dotnet.sh b/.travis/before-install-linux-dotnet.sh index bfe4eff76..816044f40 100755 --- a/.travis/before-install-linux-dotnet.sh +++ b/.travis/before-install-linux-dotnet.sh @@ -6,5 +6,5 @@ set -euo pipefail sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ trusty main" > /etc/apt/sources.list.d/dotnetdev.list' sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 417A0893 sudo apt-get update -sudo apt-get install dotnet-dev-1.0.4 +sudo apt-get --allow-unauthenticated install dotnet-dev-1.0.4 From c4d319d11aeb37c95dcb1912ef40c12638329ee0 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Mon, 19 Aug 2019 22:07:14 +0800 Subject: [PATCH 016/365] temporarily disable other builds --- .travis.yml | 106 ++++++++++++++++++++++++++-------------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/.travis.yml b/.travis.yml index 880cf8907..db7edf9e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,8 +12,8 @@ cache: - $HOME/Library/Caches/Homebrew stages: - - smoke-test - - main-test +# - smoke-test +# - main-test - extended-test matrix: @@ -72,27 +72,27 @@ matrix: - g++-5 - uuid-dev - clang-3.7 - - os: osx - compiler: clang - osx_image: xcode10.1 - env: - - TARGET=cpp - - GROUP=LEXER - stage: extended-test - - os: osx - compiler: clang - osx_image: xcode10.1 - env: - - TARGET=cpp - - GROUP=PARSER - stage: extended-test - - os: osx - compiler: clang - osx_image: xcode10.1 - env: - - TARGET=cpp - - GROUP=RECURSION - stage: extended-test +# - os: osx +# compiler: clang +# osx_image: xcode10.1 +# env: +# - TARGET=cpp +# - GROUP=LEXER +# stage: extended-test +# - os: osx +# compiler: clang +# osx_image: xcode10.1 +# env: +# - TARGET=cpp +# - GROUP=PARSER +# stage: extended-test +# - os: osx +# compiler: clang +# osx_image: xcode10.1 +# env: +# - TARGET=cpp +# - GROUP=RECURSION +# stage: extended-test - os: osx compiler: clang osx_image: xcode10.1 @@ -114,36 +114,36 @@ matrix: - TARGET=swift - GROUP=RECURSION stage: main-test - - os: linux - dist: xenial - compiler: clang - env: - - TARGET=swift - - GROUP=ALL - stage: extended-test - - os: osx - osx_image: xcode10.1 - env: - - TARGET=dotnet - - GROUP=LEXER - stage: extended-test - - os: osx - osx_image: xcode10.1 - env: - - TARGET=dotnet - - GROUP=PARSER - stage: extended-test - - os: osx - osx_image: xcode10.1 - env: - - TARGET=dotnet - - GROUP=RECURSION - stage: extended-test - - os: linux - dist: trusty - jdk: openjdk7 - env: TARGET=java - stage: extended-test +# - os: linux +# dist: xenial +# compiler: clang +# env: +# - TARGET=swift +# - GROUP=ALL +# stage: extended-test +# - os: osx +# osx_image: xcode10.1 +# env: +# - TARGET=dotnet +# - GROUP=LEXER +# stage: extended-test +# - os: osx +# osx_image: xcode10.1 +# env: +# - TARGET=dotnet +# - GROUP=PARSER +# stage: extended-test +# - os: osx +# osx_image: xcode10.1 +# env: +# - TARGET=dotnet +# - GROUP=RECURSION +# stage: extended-test +# - os: linux +# dist: trusty +# jdk: openjdk7 +# env: TARGET=java +# stage: extended-test - os: linux jdk: openjdk8 env: TARGET=java From 7fa06602b9434017f771d29e5dfe9c2e6a4e6d6e Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Mon, 19 Aug 2019 22:37:23 +0800 Subject: [PATCH 017/365] @parrt fixed --- .travis.yml | 106 ++++++++++++++++++++++++++-------------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/.travis.yml b/.travis.yml index db7edf9e6..880cf8907 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,8 +12,8 @@ cache: - $HOME/Library/Caches/Homebrew stages: -# - smoke-test -# - main-test + - smoke-test + - main-test - extended-test matrix: @@ -72,27 +72,27 @@ matrix: - g++-5 - uuid-dev - clang-3.7 -# - os: osx -# compiler: clang -# osx_image: xcode10.1 -# env: -# - TARGET=cpp -# - GROUP=LEXER -# stage: extended-test -# - os: osx -# compiler: clang -# osx_image: xcode10.1 -# env: -# - TARGET=cpp -# - GROUP=PARSER -# stage: extended-test -# - os: osx -# compiler: clang -# osx_image: xcode10.1 -# env: -# - TARGET=cpp -# - GROUP=RECURSION -# stage: extended-test + - os: osx + compiler: clang + osx_image: xcode10.1 + env: + - TARGET=cpp + - GROUP=LEXER + stage: extended-test + - os: osx + compiler: clang + osx_image: xcode10.1 + env: + - TARGET=cpp + - GROUP=PARSER + stage: extended-test + - os: osx + compiler: clang + osx_image: xcode10.1 + env: + - TARGET=cpp + - GROUP=RECURSION + stage: extended-test - os: osx compiler: clang osx_image: xcode10.1 @@ -114,36 +114,36 @@ matrix: - TARGET=swift - GROUP=RECURSION stage: main-test -# - os: linux -# dist: xenial -# compiler: clang -# env: -# - TARGET=swift -# - GROUP=ALL -# stage: extended-test -# - os: osx -# osx_image: xcode10.1 -# env: -# - TARGET=dotnet -# - GROUP=LEXER -# stage: extended-test -# - os: osx -# osx_image: xcode10.1 -# env: -# - TARGET=dotnet -# - GROUP=PARSER -# stage: extended-test -# - os: osx -# osx_image: xcode10.1 -# env: -# - TARGET=dotnet -# - GROUP=RECURSION -# stage: extended-test -# - os: linux -# dist: trusty -# jdk: openjdk7 -# env: TARGET=java -# stage: extended-test + - os: linux + dist: xenial + compiler: clang + env: + - TARGET=swift + - GROUP=ALL + stage: extended-test + - os: osx + osx_image: xcode10.1 + env: + - TARGET=dotnet + - GROUP=LEXER + stage: extended-test + - os: osx + osx_image: xcode10.1 + env: + - TARGET=dotnet + - GROUP=PARSER + stage: extended-test + - os: osx + osx_image: xcode10.1 + env: + - TARGET=dotnet + - GROUP=RECURSION + stage: extended-test + - os: linux + dist: trusty + jdk: openjdk7 + env: TARGET=java + stage: extended-test - os: linux jdk: openjdk8 env: TARGET=java From 91a50749f99d7a5844f3ff3f25c79ebd65439bca Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Wed, 21 Aug 2019 10:34:21 +0200 Subject: [PATCH 018/365] Added cmake build informations as reported in antlr4/#2606 --- runtime/Cpp/cmake/Antlr4Package.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/Cpp/cmake/Antlr4Package.md b/runtime/Cpp/cmake/Antlr4Package.md index ac9480db2..984564203 100644 --- a/runtime/Cpp/cmake/Antlr4Package.md +++ b/runtime/Cpp/cmake/Antlr4Package.md @@ -14,7 +14,7 @@ given input file during build. The following table lists the parameters that can be used with the function: Argument# | Required | Default | Use -----------|-----------|---------|-- +----------|-----------|---------|--- 0 | Yes | n/a | Unique target name. It is used to generate CMake Variables to reference the various outputs of the generation 1 | Yes | n/a | Input file containing the lexer/parser definition 2 | Yes | n/a | Type of Rules contained in the input: LEXER, PARSER or BOTH @@ -24,10 +24,10 @@ Argument# | Required | Default | Use 7 | No | none | Additional files on which the input depends 8 | No | none | Library path to use during generation -The `ANTLR4_JAR_LOCATION` CMake variable must be set to the location where the `antlr-4*-complete.jar` generator is located. -You can download the file from [here](http://www.antlr.org/download.html). +The `ANTLR4_JAR_LOCATION` CMake variable must be set to the location where the `antlr-4*-complete.jar` generator is located. You can download the file from [here](http://www.antlr.org/download.html). -Additional option to the ANTLR4 generator can be passed in the `ANTLR4_GENERATED_OPTIONS` variable +Additional options to the ANTLR4 generator can be passed in the `ANTLR4_GENERATED_OPTIONS` variable. Add the installation prefix of `antlr4-runtime` to `CMAKE_PREFIX_PATH` or set + `antlr4-runtime_DIR` to a directory containing the files. The following CMake variables are available following a call to `antlr4_generate` From 32470dbab48dbe70e035e61cebee54f4df611b33 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 24 Aug 2019 01:44:36 +0800 Subject: [PATCH 019/365] align on other runtimes --- runtime/JavaScript/src/antlr4/dfa/DFAState.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/runtime/JavaScript/src/antlr4/dfa/DFAState.js b/runtime/JavaScript/src/antlr4/dfa/DFAState.js index e597a54c3..b80df0365 100644 --- a/runtime/JavaScript/src/antlr4/dfa/DFAState.js +++ b/runtime/JavaScript/src/antlr4/dfa/DFAState.js @@ -139,12 +139,6 @@ DFAState.prototype.toString = function() { DFAState.prototype.hashCode = function() { var hash = new Hash(); hash.update(this.configs); - if(this.isAcceptState) { - if (this.predicates !== null) - hash.update(this.predicates); - else - hash.update(this.prediction); - } return hash.finish(); }; From badee1ffe11673f72d091b55275a324e9920c80c Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 24 Aug 2019 01:44:53 +0800 Subject: [PATCH 020/365] fix minor issue in deserializer --- runtime/JavaScript/src/antlr4/atn/ATNDeserializer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/JavaScript/src/antlr4/atn/ATNDeserializer.js b/runtime/JavaScript/src/antlr4/atn/ATNDeserializer.js index 3d71d4242..295c07924 100644 --- a/runtime/JavaScript/src/antlr4/atn/ATNDeserializer.js +++ b/runtime/JavaScript/src/antlr4/atn/ATNDeserializer.js @@ -138,7 +138,7 @@ ATNDeserializer.prototype.deserialize = function(data) { ATNDeserializer.prototype.reset = function(data) { var adjust = function(c) { var v = c.charCodeAt(0); - return v>1 ? v-2 : v + 65533; + return v>1 ? v-2 : v + 65534; }; var temp = data.split("").map(adjust); // don't adjust the first value since that's the version number From 165dfa2b705d173ed833acaf5413295990c0d721 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 24 Aug 2019 01:45:31 +0800 Subject: [PATCH 021/365] use same hash code whether readOnly or not --- runtime/JavaScript/src/antlr4/atn/ATNConfigSet.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/runtime/JavaScript/src/antlr4/atn/ATNConfigSet.js b/runtime/JavaScript/src/antlr4/atn/ATNConfigSet.js index 31b249d47..5a45f797e 100644 --- a/runtime/JavaScript/src/antlr4/atn/ATNConfigSet.js +++ b/runtime/JavaScript/src/antlr4/atn/ATNConfigSet.js @@ -175,7 +175,7 @@ ATNConfigSet.prototype.equals = function(other) { ATNConfigSet.prototype.hashCode = function() { var hash = new Hash(); - this.updateHashCode(hash); + hash.update(this.configs); return hash.finish(); }; @@ -183,13 +183,11 @@ ATNConfigSet.prototype.hashCode = function() { ATNConfigSet.prototype.updateHashCode = function(hash) { if (this.readOnly) { if (this.cachedHashCode === -1) { - var hash = new Hash(); - hash.update(this.configs); - this.cachedHashCode = hash.finish(); + this.cachedHashCode = this.hashCode(); } hash.update(this.cachedHashCode); } else { - hash.update(this.configs); + hash.update(this.hashCode()); } }; From 3e258c5d9d3ed66bd52303ec5f18933f1b76769a Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 24 Aug 2019 01:46:11 +0800 Subject: [PATCH 022/365] avoid hashCode == 0 --- runtime/JavaScript/src/antlr4/PredictionContext.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/runtime/JavaScript/src/antlr4/PredictionContext.js b/runtime/JavaScript/src/antlr4/PredictionContext.js index e69cdea37..8b2e2ece0 100644 --- a/runtime/JavaScript/src/antlr4/PredictionContext.js +++ b/runtime/JavaScript/src/antlr4/PredictionContext.js @@ -111,11 +111,13 @@ Object.defineProperty(PredictionContextCache.prototype, "length", { function SingletonPredictionContext(parent, returnState) { var hashCode = 0; + var hash = new Hash(); if(parent !== null) { - var hash = new Hash(); hash.update(parent, returnState); - hashCode = hash.finish(); + } else { + hash.update(1); } + hashCode = hash.finish(); PredictionContext.call(this, hashCode); this.parentCtx = parent; this.returnState = returnState; From 56eb0a3fab8f40741d25ca426d10e4899318292b Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 24 Aug 2019 01:47:12 +0800 Subject: [PATCH 023/365] fix 2 erroneous calls, thanks @akaJes --- runtime/JavaScript/src/antlr4/Utils.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/runtime/JavaScript/src/antlr4/Utils.js b/runtime/JavaScript/src/antlr4/Utils.js index 2cb939a66..9dad93c18 100644 --- a/runtime/JavaScript/src/antlr4/Utils.js +++ b/runtime/JavaScript/src/antlr4/Utils.js @@ -339,7 +339,7 @@ Hash.prototype.update = function () { if (value == null) continue; if(Array.isArray(value)) - this.update.apply(value); + this.update.apply(this, value); else { var k = 0; switch (typeof(value)) { @@ -354,7 +354,10 @@ Hash.prototype.update = function () { k = value.hashCode(); break; default: - value.updateHashCode(this); + if(value.updateHashCode) + value.updateHashCode(this); + else + console.log("No updateHashCode for " + value.toString()) continue; } k = k * 0xCC9E2D51; @@ -367,7 +370,7 @@ Hash.prototype.update = function () { this.hash = hash; } } -} +}; Hash.prototype.finish = function () { var hash = this.hash ^ (this.count * 4); @@ -377,11 +380,11 @@ Hash.prototype.finish = function () { hash = hash * 0xC2B2AE35; hash = hash ^ (hash >>> 16); return hash; -} +}; function hashStuff() { var hash = new Hash(); - hash.update.apply(arguments); + hash.update.apply(hash, arguments); return hash.finish(); } From 0de9612e5dc78874b5a6aa8183714a48d52fa70b Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 24 Aug 2019 02:10:12 +0800 Subject: [PATCH 024/365] adding original contributor --- contributors.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contributors.txt b/contributors.txt index d40bacd5f..8fd2d2fb8 100644 --- a/contributors.txt +++ b/contributors.txt @@ -221,3 +221,4 @@ YYYY/MM/DD, github id, Full name, email 2019/07/11, olowo726, Olof Wolgast, olof@baah.se 2019/07/16, abhijithneilabraham, Abhijith Neil Abraham, abhijithneilabrahampk@gmail.com 2019/07/26, Braavos96, Eric Hettiaratchi, erichettiaratchi@gmail.com +2019/08/23, akaJes, Oleksandr Mamchyts, akaJes@gmail.com \ No newline at end of file From 5a711ac79ce635b890cec3bb557166ec157ad7d3 Mon Sep 17 00:00:00 2001 From: Markus Franke Date: Tue, 27 Aug 2019 13:26:04 +0200 Subject: [PATCH 025/365] Fix escaping of angle brackets --- runtime/Cpp/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/Cpp/README.md b/runtime/Cpp/README.md index 73fab634f..79bd27884 100644 --- a/runtime/Cpp/README.md +++ b/runtime/Cpp/README.md @@ -55,11 +55,11 @@ Either open the included XCode project and build that or use the cmake compilati Try run cmake -DCMAKE_ANDROID_NDK=/folder/of/android_ndkr17_and_above -DCMAKE_SYSTEM_NAME=Android -DCMAKE_ANDROID_API=14 -DCMAKE_ANDROID_ARCH_ABI=x86 -DCMAKE_ANDROID_STL_TYPE=c++_shared -DCMAKE_ANDROID_NDK_TOOLCHAIN_VERSION=clang -DCMAKE_BUILD_TYPE=Release /folder/antlr4_src_dir -G Ninja. #### Compiling on Linux -- cd /runtime/Cpp (this is where this readme is located) +- cd \/runtime/Cpp (this is where this readme is located) - mkdir build && mkdir run && cd build - cmake .. -DANTLR_JAR_LOCATION=full/path/to/antlr4-4.5.4-SNAPSHOT.jar -DWITH_DEMO=True - make -- DESTDIR=/runtime/Cpp/run make install +- DESTDIR=\/runtime/Cpp/run make install If you don't want to build the demo then simply run cmake without parameters. There is another cmake script available in the subfolder cmake/ for those who prefer the superbuild cmake pattern. From 97e61c07cad52efff8feecc74e2fb2f5733f633a Mon Sep 17 00:00:00 2001 From: Markus Franke Date: Tue, 27 Aug 2019 13:36:07 +0200 Subject: [PATCH 026/365] Update contributors file --- contributors.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contributors.txt b/contributors.txt index d40bacd5f..a6abdb2f1 100644 --- a/contributors.txt +++ b/contributors.txt @@ -221,3 +221,4 @@ YYYY/MM/DD, github id, Full name, email 2019/07/11, olowo726, Olof Wolgast, olof@baah.se 2019/07/16, abhijithneilabraham, Abhijith Neil Abraham, abhijithneilabrahampk@gmail.com 2019/07/26, Braavos96, Eric Hettiaratchi, erichettiaratchi@gmail.com +2019/08/27, wurzelpeter, Markus Franke, markus[hyphen]franke[at]web[dot]de From f0ef1f4433c0a0a74c309687bb3cfcc7da7db440 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Sun, 1 Sep 2019 17:18:53 -0700 Subject: [PATCH 027/365] Remove unused macros from test templates. Remove unused DeclareLocal and PlusMember macros from *.test.stg. DeclareLocal was last used in 5c5228 (June 2015) and PlusMember was last used in 7f16fe6 (Nov 2016). --- .../org/antlr/v4/test/runtime/templates/CSharp.test.stg | 4 ---- .../org/antlr/v4/test/runtime/templates/Chrome.test.stg | 4 ---- .../org/antlr/v4/test/runtime/templates/Cpp.test.stg | 3 --- .../org/antlr/v4/test/runtime/templates/Explorer.test.stg | 4 ---- .../org/antlr/v4/test/runtime/templates/Firefox.test.stg | 4 ---- .../resources/org/antlr/v4/test/runtime/templates/Go.test.stg | 4 ---- .../org/antlr/v4/test/runtime/templates/Java.test.stg | 4 ---- .../org/antlr/v4/test/runtime/templates/Node.test.stg | 4 ---- .../org/antlr/v4/test/runtime/templates/Python2.test.stg | 4 ---- .../org/antlr/v4/test/runtime/templates/Python3.test.stg | 4 ---- .../org/antlr/v4/test/runtime/templates/Safari.test.stg | 4 ---- .../org/antlr/v4/test/runtime/templates/Swift.test.stg | 4 ---- 12 files changed, 47 deletions(-) diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CSharp.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CSharp.test.stg index 597248f2b..6a4d71a73 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CSharp.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CSharp.test.stg @@ -16,8 +16,6 @@ Append(a,b) ::= " + " Concat(a,b) ::= "" -DeclareLocal(s,v) ::= "Object = ;" - AssertIsList(v) ::= "System.Collections.IList __ttt__ = ;" // just use static type system AssignLocal(s,v) ::= " = ;" @@ -32,8 +30,6 @@ SetMember(n,v) ::= <%this. = ;%> AddMember(n,v) ::= <%this. += ;%> -PlusMember(v,n) ::= <% + this.%> - MemberEquals(n,v) ::= <%this. == %> ModMemberEquals(n,m,v) ::= <%this. % == %> diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Chrome.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Chrome.test.stg index e02457a90..3d8db53f2 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Chrome.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Chrome.test.stg @@ -16,8 +16,6 @@ Append(a,b) ::= " + " Concat(a,b) ::= "" -DeclareLocal(s,v) ::= "var = ;" - AssertIsList(v) ::= <> AssignLocal(s,v) ::= " = ;" @@ -32,8 +30,6 @@ SetMember(n,v) ::= <%this. = ;%> AddMember(n,v) ::= <%this. += ;%> -PlusMember(v,n) ::= <% + this.%> - MemberEquals(n,v) ::= <%this. === %> ModMemberEquals(n,m,v) ::= <%this. % === %> diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Cpp.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Cpp.test.stg index c2b1a7f84..3f2c5f10e 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Cpp.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Cpp.test.stg @@ -10,8 +10,6 @@ Cast(t,v) ::= "dynamic_cast\< *>()" // Should actually use a more specific Append(a,b) ::= " + ->toString()" Concat(a,b) ::= "" -DeclareLocal(s,v) ::= " = " - AssertIsList(v) ::= "assert(.size() >= 0);" // Use a method that exists only on a list (vector actually). AssignLocal(s,v) ::= " = ;" @@ -21,7 +19,6 @@ InitBooleanMember(n,v) ::= "bool = ;" GetMember(n) ::= "" SetMember(n,v) ::= " = ;" AddMember(n,v) ::= " += ;" -PlusMember(v,n) ::= " + " MemberEquals(n,v) ::= " == " ModMemberEquals(n,m,v) ::= " % == " ModMemberNotEquals(n,m,v) ::= " % != " diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Explorer.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Explorer.test.stg index 6cfe9ba27..14852c370 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Explorer.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Explorer.test.stg @@ -16,8 +16,6 @@ Append(a,b) ::= " + " Concat(a,b) ::= "" -DeclareLocal(s,v) ::= "var = ;" - AssertIsList(v) ::= <> AssignLocal(s,v) ::= " = ;" @@ -32,8 +30,6 @@ SetMember(n,v) ::= <%this. = ;%> AddMember(n,v) ::= <%this. += ;%> -PlusMember(v,n) ::= <% + this.%> - MemberEquals(n,v) ::= <%this. === %> ModMemberEquals(n,m,v) ::= <%this. % === %> diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Firefox.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Firefox.test.stg index c94be90e8..34d34a6f8 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Firefox.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Firefox.test.stg @@ -16,8 +16,6 @@ Append(a,b) ::= " + " Concat(a,b) ::= "" -DeclareLocal(s,v) ::= "var = ;" - AssertIsList(v) ::= <> AssignLocal(s,v) ::= " = ;" @@ -32,8 +30,6 @@ SetMember(n,v) ::= <%this. = ;%> AddMember(n,v) ::= <%this. += ;%> -PlusMember(v,n) ::= <% + this.%> - MemberEquals(n,v) ::= <%this. === %> ModMemberEquals(n,m,v) ::= <%this. % === %> diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Go.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Go.test.stg index 37fe63441..17061b71d 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Go.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Go.test.stg @@ -16,8 +16,6 @@ Append(a,b) ::= " + fmt.Sprint()" Concat(a,b) ::= "" -DeclareLocal(s, v) ::= "var = " - AssertIsList(v) ::= "" AssignLocal(s, v) ::= " = ;" @@ -32,8 +30,6 @@ SetMember(n, v) ::= <% = ;%> AddMember(n, v) ::= <% += ;%> -PlusMember(v, n) ::= <% + fmt.Sprint()%> - MemberEquals(n, v) ::= <% == %> ModMemberEquals(n, m, v) ::= <% % == %> diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg index 5012aa213..5b64c2936 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg @@ -16,8 +16,6 @@ Append(a,b) ::= " + " Concat(a,b) ::= "" -DeclareLocal(s,v) ::= "Object = ;" - AssertIsList(v) ::= "List\ __ttt__ = ;" // just use static type system AssignLocal(s,v) ::= " = ;" @@ -32,8 +30,6 @@ SetMember(n,v) ::= <%this. = ;%> AddMember(n,v) ::= <%this. += ;%> -PlusMember(v,n) ::= <% + this.%> - MemberEquals(n,v) ::= <%this. == %> ModMemberEquals(n,m,v) ::= <%this. % == %> diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Node.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Node.test.stg index 92cb1dbbb..0b844b3fb 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Node.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Node.test.stg @@ -16,8 +16,6 @@ Append(a,b) ::= " + " Concat(a,b) ::= "" -DeclareLocal(s,v) ::= "var = ;" - AssertIsList(v) ::= <> AssignLocal(s,v) ::= " = ;" @@ -32,8 +30,6 @@ SetMember(n,v) ::= <%this. = ;%> AddMember(n,v) ::= <%this. += ;%> -PlusMember(v,n) ::= <% + this.%> - MemberEquals(n,v) ::= <%this. === %> ModMemberEquals(n,m,v) ::= <%this. % === %> diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python2.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python2.test.stg index cc0d9d9c2..8ef732937 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python2.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python2.test.stg @@ -16,8 +16,6 @@ Append(a,b) ::= " + str()" Concat(a,b) ::= "" -DeclareLocal(s,v) ::= " = " - AssertIsList(v) ::= "assert isinstance(v, (list, tuple))" AssignLocal(s,v) ::= " = " @@ -32,8 +30,6 @@ SetMember(n,v) ::= <%self. = %> AddMember(n,v) ::= <%self. += %> -PlusMember(v,n) ::= <% + str(self.)%> - MemberEquals(n,v) ::= <%self. == %> ModMemberEquals(n,m,v) ::= <%self. % == %> diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python3.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python3.test.stg index c91563017..3876834db 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python3.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python3.test.stg @@ -16,8 +16,6 @@ Append(a,b) ::= " + str()" Concat(a,b) ::= "" -DeclareLocal(s,v) ::= " = " - AssertIsList(v) ::= "assert isinstance(v, (list, tuple))" AssignLocal(s,v) ::= " = " @@ -32,8 +30,6 @@ SetMember(n,v) ::= <%self. = %> AddMember(n,v) ::= <%self. += %> -PlusMember(v,n) ::= <% + str(self.)%> - MemberEquals(n,v) ::= <%self. == %> ModMemberEquals(n,m,v) ::= <%self. % == %> diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Safari.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Safari.test.stg index 71254e0f2..112161500 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Safari.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Safari.test.stg @@ -16,8 +16,6 @@ Append(a,b) ::= " + " Concat(a,b) ::= "" -DeclareLocal(s,v) ::= "var = ;" - AssertIsList(v) ::= <> AssignLocal(s,v) ::= " = ;" @@ -32,8 +30,6 @@ SetMember(n,v) ::= <%this. = ;%> AddMember(n,v) ::= <%this. += ;%> -PlusMember(v,n) ::= <% + this.%> - MemberEquals(n,v) ::= <%this. === %> ModMemberEquals(n,m,v) ::= <%this. % === %> diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg index 100d6f3e7..077968ef4 100755 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg @@ -16,8 +16,6 @@ Append(a,b) ::= " + " Concat(a,b) ::= "" -DeclareLocal(s,v) ::= "var = " - AssertIsList(v) ::= "var __ttt__ = ;" // just use static type system AssignLocal(s,v) ::= " = " @@ -32,8 +30,6 @@ SetMember(n,v) ::= <%self. = %> AddMember(n,v) ::= <%self. += %> -PlusMember(v,n) ::= <% + self.%> - MemberEquals(n,v) ::= <%self. == %> ModMemberEquals(n,m,v) ::= <%self. % == %> From ff1283affb4df5d5192aa53aee395643b874a57a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Henrique?= Date: Tue, 3 Sep 2019 19:21:42 -0300 Subject: [PATCH 028/365] General XPath fixes for the Python3 runtime --- contributors.txt | 1 + runtime/Python3/src/antlr4/xpath/XPath.py | 117 +++++++++++----------- 2 files changed, 62 insertions(+), 56 deletions(-) diff --git a/contributors.txt b/contributors.txt index d40bacd5f..55687a831 100644 --- a/contributors.txt +++ b/contributors.txt @@ -221,3 +221,4 @@ YYYY/MM/DD, github id, Full name, email 2019/07/11, olowo726, Olof Wolgast, olof@baah.se 2019/07/16, abhijithneilabraham, Abhijith Neil Abraham, abhijithneilabrahampk@gmail.com 2019/07/26, Braavos96, Eric Hettiaratchi, erichettiaratchi@gmail.com +2019/09/03, João Henrique, johnnyonflame@hotmail.com diff --git a/runtime/Python3/src/antlr4/xpath/XPath.py b/runtime/Python3/src/antlr4/xpath/XPath.py index c49e3ba94..dcffad518 100644 --- a/runtime/Python3/src/antlr4/xpath/XPath.py +++ b/runtime/Python3/src/antlr4/xpath/XPath.py @@ -47,7 +47,7 @@ #

# Whitespace is not allowed.

# -from antlr4 import CommonTokenStream, DFA, PredictionContextCache, Lexer, LexerATNSimulator +from antlr4 import CommonTokenStream, DFA, PredictionContextCache, Lexer, LexerATNSimulator, ParserRuleContext, TerminalNode from antlr4.InputStream import InputStream from antlr4.Parser import Parser from antlr4.RuleContext import RuleContext @@ -134,7 +134,7 @@ class XPathLexer(Lexer): if _action is not None: _action(localctx, actionIndex) else: - raise Exception("No registered action for:" + str(ruleIndex)) + raise Exception("No registered action for: %d" % ruleIndex) def ID_action(self, localctx:RuleContext , actionIndex:int): if actionIndex == 0: @@ -166,40 +166,40 @@ class XPath(object): try: tokenStream.fill() except LexerNoViableAltException as e: - pos = lexer.getColumn() - msg = "Invalid tokens or characters at index " + str(pos) + " in path '" + path + "'" + pos = lexer.column + msg = "Invalid tokens or characters at index %d in path '%s'" % (pos, path) raise Exception(msg, e) - tokens = tokenStream.getTokens() + tokens = iter(tokenStream.tokens) elements = list() - n = len(tokens) - i=0 - while i < n : - el = tokens[i] - next = None + for el in tokens: + invert = False + anywhere = False + # Check for path separators, if none assume root if el.type in [XPathLexer.ROOT, XPathLexer.ANYWHERE]: - anywhere = el.type == XPathLexer.ANYWHERE - i += 1 - next = tokens[i] - invert = next.type==XPathLexer.BANG - if invert: - i += 1 - next = tokens[i] - pathElement = self.getXPathElement(next, anywhere) - pathElement.invert = invert - elements.append(pathElement) - i += 1 - - elif el.type in [XPathLexer.TOKEN_REF, XPathLexer.RULE_REF, XPathLexer.WILDCARD] : - elements.append( self.getXPathElement(el, False) ) - i += 1 - - elif el.type==Token.EOF : - break - + anywhere = el.type == XPathLexer.ANYWHERE + next_el = next(tokens, None) + if not next_el: + raise Exception('Missing element after %s' % el.getText()) + else: + el = next_el + # Check for bangs + if el.type == XPathLexer.BANG: + invert = True + next_el = next(tokens, None) + if not next_el: + raise Exception('Missing element after %s' % el.getText()) + else: + el = next_el + # Add searched element + if el.type in [XPathLexer.TOKEN_REF, XPathLexer.RULE_REF, XPathLexer.WILDCARD]: + element = self.getXPathElement(el, anywhere) + element.invert = invert + elements.append(element) + elif el.type==Token.EOF: + break else: - raise Exception("Unknown path element " + str(el)) - + raise Exception("Unknown path element %s" % lexer.symbolicNames[el.type]) return elements # @@ -210,24 +210,32 @@ class XPath(object): def getXPathElement(self, wordToken:Token, anywhere:bool): if wordToken.type==Token.EOF: raise Exception("Missing path element at end of path") + word = wordToken.text - ttype = self.parser.getTokenType(word) - ruleIndex = self.parser.getRuleIndex(word) - if wordToken.type==XPathLexer.WILDCARD : - return XPathWildcardAnywhereElement() if anywhere else XPathWildcardElement() elif wordToken.type in [XPathLexer.TOKEN_REF, XPathLexer.STRING]: + tsource = self.parser.getTokenStream().tokenSource - if ttype==Token.INVALID_TYPE: - raise Exception( word + " at index " + str(wordToken.startIndex) + " isn't a valid token name") + ttype = litType = None + if wordToken.type == XPathLexer.TOKEN_REF: + ttype = (tsource.ruleNames.index(word) + 1) if word in tsource.ruleNames else None + else: + litType = tsource.literalNames.index(word) if word in tsource.literalNames else None + + # Decide which one is it + ttype = ttype or litType + + if not ttype: + raise Exception("%s at index %d isn't a valid token name" % (word, wordToken.tokenIndex)) return XPathTokenAnywhereElement(word, ttype) if anywhere else XPathTokenElement(word, ttype) else: + ruleIndex = self.parser.ruleNames.index(word) if word in self.parser.ruleNames else None - if ruleIndex==-1: - raise Exception( word + " at index " + str(wordToken.getStartIndex()) + " isn't a valid rule name") + if not ruleIndex: + raise Exception("%s at index %d isn't a valid rule name" % (word, wordToken.tokenIndex)) return XPathRuleAnywhereElement(word, ruleIndex) if anywhere else XPathRuleElement(word, ruleIndex) @@ -246,18 +254,16 @@ class XPath(object): dummyRoot.children = [t] # don't set t's parent. work = [dummyRoot] - - for i in range(0, len(self.elements)): - next = set() + for element in self.elements: + work_next = list() for node in work: - if len( node.children) > 0 : + if not isinstance(node, TerminalNode) and node.children: # only try to match next element if it has children # e.g., //func/*/stat might have a token node for which # we can't go looking for stat nodes. - matching = self.elements[i].evaluate(node) - next |= matching - i += 1 - work = next + matching = element.evaluate(node) + work_next.extend(matching) + work = work_next return work @@ -283,8 +289,8 @@ class XPathRuleAnywhereElement(XPathElement): self.ruleIndex = ruleIndex def evaluate(self, t:ParseTree): - return Trees.findAllRuleNodes(t, self.ruleIndex) - + # return all ParserRuleContext descendants of t that match ruleIndex (or do not match if inverted) + return filter(lambda c: isinstance(c, ParserRuleContext) and (self.invert ^ (c.getRuleIndex() == self.ruleIndex)), Trees.descendants(t)) class XPathRuleElement(XPathElement): @@ -293,9 +299,8 @@ class XPathRuleElement(XPathElement): self.ruleIndex = ruleIndex def evaluate(self, t:ParseTree): - # return all children of t that match nodeName - return [c for c in Trees.getChildren(t) if isinstance(c, ParserRuleContext) and (c.ruleIndex == self.ruleIndex) == (not self.invert)] - + # return all ParserRuleContext children of t that match ruleIndex (or do not match if inverted) + return filter(lambda c: isinstance(c, ParserRuleContext) and (self.invert ^ (c.getRuleIndex() == self.ruleIndex)), Trees.getChildren(t)) class XPathTokenAnywhereElement(XPathElement): @@ -304,8 +309,8 @@ class XPathTokenAnywhereElement(XPathElement): self.tokenType = tokenType def evaluate(self, t:ParseTree): - return Trees.findAllTokenNodes(t, self.tokenType) - + # return all TerminalNode descendants of t that match tokenType (or do not match if inverted) + return filter(lambda c: isinstance(c, TerminalNode) and (self.invert ^ (c.symbol.type == self.tokenType)), Trees.descendants(t)) class XPathTokenElement(XPathElement): @@ -314,8 +319,8 @@ class XPathTokenElement(XPathElement): self.tokenType = tokenType def evaluate(self, t:ParseTree): - # return all children of t that match nodeName - return [c for c in Trees.getChildren(t) if isinstance(c, TerminalNode) and (c.symbol.type == self.tokenType) == (not self.invert)] + # return all TerminalNode children of t that match tokenType (or do not match if inverted) + return filter(lambda c: isinstance(c, TerminalNode) and (self.invert ^ (c.symbol.type == self.tokenType)), Trees.getChildren(t)) class XPathWildcardAnywhereElement(XPathElement): From 4c2f091e8c9f8e1d40686d7e84453027cc55fff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Henrique?= Date: Wed, 4 Sep 2019 14:34:19 -0300 Subject: [PATCH 029/365] Style touch-ups on Python3 XPath implementation --- runtime/Python3/src/antlr4/xpath/XPath.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/runtime/Python3/src/antlr4/xpath/XPath.py b/runtime/Python3/src/antlr4/xpath/XPath.py index dcffad518..4e8670ff1 100644 --- a/runtime/Python3/src/antlr4/xpath/XPath.py +++ b/runtime/Python3/src/antlr4/xpath/XPath.py @@ -218,23 +218,22 @@ class XPath(object): elif wordToken.type in [XPathLexer.TOKEN_REF, XPathLexer.STRING]: tsource = self.parser.getTokenStream().tokenSource - ttype = litType = None + ttype = -1 if wordToken.type == XPathLexer.TOKEN_REF: - ttype = (tsource.ruleNames.index(word) + 1) if word in tsource.ruleNames else None + if word in tsource.ruleNames: + ttype = tsource.ruleNames.index(word) + 1 else: - litType = tsource.literalNames.index(word) if word in tsource.literalNames else None + if word in tsource.literalNames: + ttype = tsource.literalNames.index(word) - # Decide which one is it - ttype = ttype or litType - - if not ttype: + if ttype == -1: raise Exception("%s at index %d isn't a valid token name" % (word, wordToken.tokenIndex)) return XPathTokenAnywhereElement(word, ttype) if anywhere else XPathTokenElement(word, ttype) else: - ruleIndex = self.parser.ruleNames.index(word) if word in self.parser.ruleNames else None + ruleIndex = self.parser.ruleNames.index(word) if word in self.parser.ruleNames else -1 - if not ruleIndex: + if ruleIndex == -1: raise Exception("%s at index %d isn't a valid rule name" % (word, wordToken.tokenIndex)) return XPathRuleAnywhereElement(word, ruleIndex) if anywhere else XPathRuleElement(word, ruleIndex) From ae2a689a68408b2f76e7c1c7c3f51731fc293ffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Henrique?= Date: Wed, 4 Sep 2019 15:33:24 -0300 Subject: [PATCH 030/365] Fixed missing XPathLexer.STRING case --- runtime/Python3/src/antlr4/xpath/XPath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/Python3/src/antlr4/xpath/XPath.py b/runtime/Python3/src/antlr4/xpath/XPath.py index 4e8670ff1..07fcd0b54 100644 --- a/runtime/Python3/src/antlr4/xpath/XPath.py +++ b/runtime/Python3/src/antlr4/xpath/XPath.py @@ -192,7 +192,7 @@ class XPath(object): else: el = next_el # Add searched element - if el.type in [XPathLexer.TOKEN_REF, XPathLexer.RULE_REF, XPathLexer.WILDCARD]: + if el.type in [XPathLexer.TOKEN_REF, XPathLexer.RULE_REF, XPathLexer.WILDCARD, XPathLexer.STRING]: element = self.getXPathElement(el, anywhere) element.invert = invert elements.append(element) From f15a9f76287ea3083488672b17535ea63470f2ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Henrique?= Date: Wed, 4 Sep 2019 15:53:02 -0300 Subject: [PATCH 031/365] Prevent XPath from returning the same node multiple times in Python3 --- runtime/Python3/src/antlr4/xpath/XPath.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/runtime/Python3/src/antlr4/xpath/XPath.py b/runtime/Python3/src/antlr4/xpath/XPath.py index 07fcd0b54..4d3a4f56f 100644 --- a/runtime/Python3/src/antlr4/xpath/XPath.py +++ b/runtime/Python3/src/antlr4/xpath/XPath.py @@ -261,6 +261,11 @@ class XPath(object): # e.g., //func/*/stat might have a token node for which # we can't go looking for stat nodes. matching = element.evaluate(node) + + # See issue antlr#370 - Prevents XPath from returning the + # same node multiple times + matching = filter(lambda m: m not in work_next, matching) + work_next.extend(matching) work = work_next From 8da2ce3044cf9411b83e04812d0bd76da0b2680c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Henrique?= Date: Wed, 4 Sep 2019 15:59:12 -0300 Subject: [PATCH 032/365] Added XPath test on Python3 target --- runtime/Python3/test/expr/Expr.g4 | 31 ++ runtime/Python3/test/expr/ExprLexer.py | 94 ++++ runtime/Python3/test/expr/ExprParser.py | 658 ++++++++++++++++++++++++ runtime/Python3/test/run.py | 1 + runtime/Python3/test/xpathtest.py | 89 ++++ 5 files changed, 873 insertions(+) create mode 100644 runtime/Python3/test/expr/Expr.g4 create mode 100644 runtime/Python3/test/expr/ExprLexer.py create mode 100644 runtime/Python3/test/expr/ExprParser.py create mode 100644 runtime/Python3/test/xpathtest.py diff --git a/runtime/Python3/test/expr/Expr.g4 b/runtime/Python3/test/expr/Expr.g4 new file mode 100644 index 000000000..662079641 --- /dev/null +++ b/runtime/Python3/test/expr/Expr.g4 @@ -0,0 +1,31 @@ +// Taken from "tool-testsuite/test/org/antlr/v4/test/tool/TestXPath.java" +// Builds ExprLexer.py and ExprParser.py + +grammar Expr; +prog: func+ ; +func: 'def' ID '(' arg (',' arg)* ')' body ; +body: '{' stat+ '}' ; +arg : ID ; +stat: expr ';' # printExpr + | ID '=' expr ';' # assign + | 'return' expr ';' # ret + | ';' # blank + ; +expr: expr ('*'|'/') expr # MulDiv + | expr ('+'|'-') expr # AddSub + | primary # prim + ; +primary + : INT # int + | ID # id + | '(' expr ')' # parens + ; +MUL : '*' ; // assigns token name to '*' used above in grammar +DIV : '/' ; +ADD : '+' ; +SUB : '-' ; +RETURN : 'return' ; +ID : [a-zA-Z]+ ; // match identifiers +INT : [0-9]+ ; // match integers +NEWLINE:'\r'? '\n' -> skip; // return newlines to parser (is end-statement signal) +WS : [ \t]+ -> skip ; // toss out whitespace \ No newline at end of file diff --git a/runtime/Python3/test/expr/ExprLexer.py b/runtime/Python3/test/expr/ExprLexer.py new file mode 100644 index 000000000..e338b0b9e --- /dev/null +++ b/runtime/Python3/test/expr/ExprLexer.py @@ -0,0 +1,94 @@ +# Generated from expr/Expr.g4 by ANTLR 4.7.2 +from antlr4 import * +from io import StringIO +from typing.io import TextIO +import sys + + +def serializedATN(): + with StringIO() as buf: + buf.write("\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\2\23") + buf.write("^\b\1\4\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7") + buf.write("\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t\13\4\f\t\f\4\r\t\r\4\16") + buf.write("\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22\3\2\3\2") + buf.write("\3\2\3\2\3\3\3\3\3\4\3\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3") + buf.write("\b\3\t\3\t\3\n\3\n\3\13\3\13\3\f\3\f\3\r\3\r\3\16\3\16") + buf.write("\3\16\3\16\3\16\3\16\3\16\3\17\6\17H\n\17\r\17\16\17I") + buf.write("\3\20\6\20M\n\20\r\20\16\20N\3\21\5\21R\n\21\3\21\3\21") + buf.write("\3\21\3\21\3\22\6\22Y\n\22\r\22\16\22Z\3\22\3\22\2\2\23") + buf.write("\3\3\5\4\7\5\t\6\13\7\r\b\17\t\21\n\23\13\25\f\27\r\31") + buf.write("\16\33\17\35\20\37\21!\22#\23\3\2\5\4\2C\\c|\3\2\62;\4") + buf.write("\2\13\13\"\"\2a\2\3\3\2\2\2\2\5\3\2\2\2\2\7\3\2\2\2\2") + buf.write("\t\3\2\2\2\2\13\3\2\2\2\2\r\3\2\2\2\2\17\3\2\2\2\2\21") + buf.write("\3\2\2\2\2\23\3\2\2\2\2\25\3\2\2\2\2\27\3\2\2\2\2\31\3") + buf.write("\2\2\2\2\33\3\2\2\2\2\35\3\2\2\2\2\37\3\2\2\2\2!\3\2\2") + buf.write("\2\2#\3\2\2\2\3%\3\2\2\2\5)\3\2\2\2\7+\3\2\2\2\t-\3\2") + buf.write("\2\2\13/\3\2\2\2\r\61\3\2\2\2\17\63\3\2\2\2\21\65\3\2") + buf.write("\2\2\23\67\3\2\2\2\259\3\2\2\2\27;\3\2\2\2\31=\3\2\2\2") + buf.write("\33?\3\2\2\2\35G\3\2\2\2\37L\3\2\2\2!Q\3\2\2\2#X\3\2\2") + buf.write("\2%&\7f\2\2&\'\7g\2\2\'(\7h\2\2(\4\3\2\2\2)*\7*\2\2*\6") + buf.write("\3\2\2\2+,\7.\2\2,\b\3\2\2\2-.\7+\2\2.\n\3\2\2\2/\60\7") + buf.write("}\2\2\60\f\3\2\2\2\61\62\7\177\2\2\62\16\3\2\2\2\63\64") + buf.write("\7=\2\2\64\20\3\2\2\2\65\66\7?\2\2\66\22\3\2\2\2\678\7") + buf.write(",\2\28\24\3\2\2\29:\7\61\2\2:\26\3\2\2\2;<\7-\2\2<\30") + buf.write("\3\2\2\2=>\7/\2\2>\32\3\2\2\2?@\7t\2\2@A\7g\2\2AB\7v\2") + buf.write("\2BC\7w\2\2CD\7t\2\2DE\7p\2\2E\34\3\2\2\2FH\t\2\2\2GF") + buf.write("\3\2\2\2HI\3\2\2\2IG\3\2\2\2IJ\3\2\2\2J\36\3\2\2\2KM\t") + buf.write("\3\2\2LK\3\2\2\2MN\3\2\2\2NL\3\2\2\2NO\3\2\2\2O \3\2\2") + buf.write("\2PR\7\17\2\2QP\3\2\2\2QR\3\2\2\2RS\3\2\2\2ST\7\f\2\2") + buf.write("TU\3\2\2\2UV\b\21\2\2V\"\3\2\2\2WY\t\4\2\2XW\3\2\2\2Y") + buf.write("Z\3\2\2\2ZX\3\2\2\2Z[\3\2\2\2[\\\3\2\2\2\\]\b\22\2\2]") + buf.write("$\3\2\2\2\7\2INQZ\3\b\2\2") + return buf.getvalue() + + +class ExprLexer(Lexer): + + atn = ATNDeserializer().deserialize(serializedATN()) + + decisionsToDFA = [ DFA(ds, i) for i, ds in enumerate(atn.decisionToState) ] + + T__0 = 1 + T__1 = 2 + T__2 = 3 + T__3 = 4 + T__4 = 5 + T__5 = 6 + T__6 = 7 + T__7 = 8 + MUL = 9 + DIV = 10 + ADD = 11 + SUB = 12 + RETURN = 13 + ID = 14 + INT = 15 + NEWLINE = 16 + WS = 17 + + channelNames = [ u"DEFAULT_TOKEN_CHANNEL", u"HIDDEN" ] + + modeNames = [ "DEFAULT_MODE" ] + + literalNames = [ "", + "'def'", "'('", "','", "')'", "'{'", "'}'", "';'", "'='", "'*'", + "'/'", "'+'", "'-'", "'return'" ] + + symbolicNames = [ "", + "MUL", "DIV", "ADD", "SUB", "RETURN", "ID", "INT", "NEWLINE", + "WS" ] + + ruleNames = [ "T__0", "T__1", "T__2", "T__3", "T__4", "T__5", "T__6", + "T__7", "MUL", "DIV", "ADD", "SUB", "RETURN", "ID", "INT", + "NEWLINE", "WS" ] + + grammarFileName = "Expr.g4" + + def __init__(self, input=None, output:TextIO = sys.stdout): + super().__init__(input, output) + self.checkVersion("4.7.2") + self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) + self._actions = None + self._predicates = None + + diff --git a/runtime/Python3/test/expr/ExprParser.py b/runtime/Python3/test/expr/ExprParser.py new file mode 100644 index 000000000..598c778d1 --- /dev/null +++ b/runtime/Python3/test/expr/ExprParser.py @@ -0,0 +1,658 @@ +# Generated from expr/Expr.g4 by ANTLR 4.7.2 +# encoding: utf-8 +from antlr4 import * +from io import StringIO +from typing.io import TextIO +import sys + +def serializedATN(): + with StringIO() as buf: + buf.write("\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\3\23") + buf.write("S\4\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b") + buf.write("\t\b\3\2\6\2\22\n\2\r\2\16\2\23\3\3\3\3\3\3\3\3\3\3\3") + buf.write("\3\7\3\34\n\3\f\3\16\3\37\13\3\3\3\3\3\3\3\3\4\3\4\6\4") + buf.write("&\n\4\r\4\16\4\'\3\4\3\4\3\5\3\5\3\6\3\6\3\6\3\6\3\6\3") + buf.write("\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6\5\6;\n\6\3\7\3\7\3\7\3") + buf.write("\7\3\7\3\7\3\7\3\7\3\7\7\7F\n\7\f\7\16\7I\13\7\3\b\3\b") + buf.write("\3\b\3\b\3\b\3\b\5\bQ\n\b\3\b\2\3\f\t\2\4\6\b\n\f\16\2") + buf.write("\4\3\2\13\f\3\2\r\16\2U\2\21\3\2\2\2\4\25\3\2\2\2\6#\3") + buf.write("\2\2\2\b+\3\2\2\2\n:\3\2\2\2\f<\3\2\2\2\16P\3\2\2\2\20") + buf.write("\22\5\4\3\2\21\20\3\2\2\2\22\23\3\2\2\2\23\21\3\2\2\2") + buf.write("\23\24\3\2\2\2\24\3\3\2\2\2\25\26\7\3\2\2\26\27\7\20\2") + buf.write("\2\27\30\7\4\2\2\30\35\5\b\5\2\31\32\7\5\2\2\32\34\5\b") + buf.write("\5\2\33\31\3\2\2\2\34\37\3\2\2\2\35\33\3\2\2\2\35\36\3") + buf.write("\2\2\2\36 \3\2\2\2\37\35\3\2\2\2 !\7\6\2\2!\"\5\6\4\2") + buf.write("\"\5\3\2\2\2#%\7\7\2\2$&\5\n\6\2%$\3\2\2\2&\'\3\2\2\2") + buf.write("\'%\3\2\2\2\'(\3\2\2\2()\3\2\2\2)*\7\b\2\2*\7\3\2\2\2") + buf.write("+,\7\20\2\2,\t\3\2\2\2-.\5\f\7\2./\7\t\2\2/;\3\2\2\2\60") + buf.write("\61\7\20\2\2\61\62\7\n\2\2\62\63\5\f\7\2\63\64\7\t\2\2") + buf.write("\64;\3\2\2\2\65\66\7\17\2\2\66\67\5\f\7\2\678\7\t\2\2") + buf.write("8;\3\2\2\29;\7\t\2\2:-\3\2\2\2:\60\3\2\2\2:\65\3\2\2\2") + buf.write(":9\3\2\2\2;\13\3\2\2\2<=\b\7\1\2=>\5\16\b\2>G\3\2\2\2") + buf.write("?@\f\5\2\2@A\t\2\2\2AF\5\f\7\6BC\f\4\2\2CD\t\3\2\2DF\5") + buf.write("\f\7\5E?\3\2\2\2EB\3\2\2\2FI\3\2\2\2GE\3\2\2\2GH\3\2\2") + buf.write("\2H\r\3\2\2\2IG\3\2\2\2JQ\7\21\2\2KQ\7\20\2\2LM\7\4\2") + buf.write("\2MN\5\f\7\2NO\7\6\2\2OQ\3\2\2\2PJ\3\2\2\2PK\3\2\2\2P") + buf.write("L\3\2\2\2Q\17\3\2\2\2\t\23\35\':EGP") + return buf.getvalue() + + +class ExprParser ( Parser ): + + grammarFileName = "Expr.g4" + + atn = ATNDeserializer().deserialize(serializedATN()) + + decisionsToDFA = [ DFA(ds, i) for i, ds in enumerate(atn.decisionToState) ] + + sharedContextCache = PredictionContextCache() + + literalNames = [ "", "'def'", "'('", "','", "')'", "'{'", "'}'", + "';'", "'='", "'*'", "'/'", "'+'", "'-'", "'return'" ] + + symbolicNames = [ "", "", "", "", + "", "", "", "", + "", "MUL", "DIV", "ADD", "SUB", "RETURN", + "ID", "INT", "NEWLINE", "WS" ] + + RULE_prog = 0 + RULE_func = 1 + RULE_body = 2 + RULE_arg = 3 + RULE_stat = 4 + RULE_expr = 5 + RULE_primary = 6 + + ruleNames = [ "prog", "func", "body", "arg", "stat", "expr", "primary" ] + + EOF = Token.EOF + T__0=1 + T__1=2 + T__2=3 + T__3=4 + T__4=5 + T__5=6 + T__6=7 + T__7=8 + MUL=9 + DIV=10 + ADD=11 + SUB=12 + RETURN=13 + ID=14 + INT=15 + NEWLINE=16 + WS=17 + + def __init__(self, input:TokenStream, output:TextIO = sys.stdout): + super().__init__(input, output) + self.checkVersion("4.7.2") + self._interp = ParserATNSimulator(self, self.atn, self.decisionsToDFA, self.sharedContextCache) + self._predicates = None + + + + class ProgContext(ParserRuleContext): + + def __init__(self, parser, parent:ParserRuleContext=None, invokingState:int=-1): + super().__init__(parent, invokingState) + self.parser = parser + + def func(self, i:int=None): + if i is None: + return self.getTypedRuleContexts(ExprParser.FuncContext) + else: + return self.getTypedRuleContext(ExprParser.FuncContext,i) + + + def getRuleIndex(self): + return ExprParser.RULE_prog + + + + + def prog(self): + + localctx = ExprParser.ProgContext(self, self._ctx, self.state) + self.enterRule(localctx, 0, self.RULE_prog) + self._la = 0 # Token type + try: + self.enterOuterAlt(localctx, 1) + self.state = 15 + self._errHandler.sync(self) + _la = self._input.LA(1) + while True: + self.state = 14 + self.func() + self.state = 17 + self._errHandler.sync(self) + _la = self._input.LA(1) + if not (_la==ExprParser.T__0): + break + + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class FuncContext(ParserRuleContext): + + def __init__(self, parser, parent:ParserRuleContext=None, invokingState:int=-1): + super().__init__(parent, invokingState) + self.parser = parser + + def ID(self): + return self.getToken(ExprParser.ID, 0) + + def arg(self, i:int=None): + if i is None: + return self.getTypedRuleContexts(ExprParser.ArgContext) + else: + return self.getTypedRuleContext(ExprParser.ArgContext,i) + + + def body(self): + return self.getTypedRuleContext(ExprParser.BodyContext,0) + + + def getRuleIndex(self): + return ExprParser.RULE_func + + + + + def func(self): + + localctx = ExprParser.FuncContext(self, self._ctx, self.state) + self.enterRule(localctx, 2, self.RULE_func) + self._la = 0 # Token type + try: + self.enterOuterAlt(localctx, 1) + self.state = 19 + self.match(ExprParser.T__0) + self.state = 20 + self.match(ExprParser.ID) + self.state = 21 + self.match(ExprParser.T__1) + self.state = 22 + self.arg() + self.state = 27 + self._errHandler.sync(self) + _la = self._input.LA(1) + while _la==ExprParser.T__2: + self.state = 23 + self.match(ExprParser.T__2) + self.state = 24 + self.arg() + self.state = 29 + self._errHandler.sync(self) + _la = self._input.LA(1) + + self.state = 30 + self.match(ExprParser.T__3) + self.state = 31 + self.body() + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class BodyContext(ParserRuleContext): + + def __init__(self, parser, parent:ParserRuleContext=None, invokingState:int=-1): + super().__init__(parent, invokingState) + self.parser = parser + + def stat(self, i:int=None): + if i is None: + return self.getTypedRuleContexts(ExprParser.StatContext) + else: + return self.getTypedRuleContext(ExprParser.StatContext,i) + + + def getRuleIndex(self): + return ExprParser.RULE_body + + + + + def body(self): + + localctx = ExprParser.BodyContext(self, self._ctx, self.state) + self.enterRule(localctx, 4, self.RULE_body) + self._la = 0 # Token type + try: + self.enterOuterAlt(localctx, 1) + self.state = 33 + self.match(ExprParser.T__4) + self.state = 35 + self._errHandler.sync(self) + _la = self._input.LA(1) + while True: + self.state = 34 + self.stat() + self.state = 37 + self._errHandler.sync(self) + _la = self._input.LA(1) + if not ((((_la) & ~0x3f) == 0 and ((1 << _la) & ((1 << ExprParser.T__1) | (1 << ExprParser.T__6) | (1 << ExprParser.RETURN) | (1 << ExprParser.ID) | (1 << ExprParser.INT))) != 0)): + break + + self.state = 39 + self.match(ExprParser.T__5) + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class ArgContext(ParserRuleContext): + + def __init__(self, parser, parent:ParserRuleContext=None, invokingState:int=-1): + super().__init__(parent, invokingState) + self.parser = parser + + def ID(self): + return self.getToken(ExprParser.ID, 0) + + def getRuleIndex(self): + return ExprParser.RULE_arg + + + + + def arg(self): + + localctx = ExprParser.ArgContext(self, self._ctx, self.state) + self.enterRule(localctx, 6, self.RULE_arg) + try: + self.enterOuterAlt(localctx, 1) + self.state = 41 + self.match(ExprParser.ID) + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class StatContext(ParserRuleContext): + + def __init__(self, parser, parent:ParserRuleContext=None, invokingState:int=-1): + super().__init__(parent, invokingState) + self.parser = parser + + + def getRuleIndex(self): + return ExprParser.RULE_stat + + + def copyFrom(self, ctx:ParserRuleContext): + super().copyFrom(ctx) + + + + class RetContext(StatContext): + + def __init__(self, parser, ctx:ParserRuleContext): # actually a ExprParser.StatContext + super().__init__(parser) + self.copyFrom(ctx) + + def RETURN(self): + return self.getToken(ExprParser.RETURN, 0) + def expr(self): + return self.getTypedRuleContext(ExprParser.ExprContext,0) + + + + class BlankContext(StatContext): + + def __init__(self, parser, ctx:ParserRuleContext): # actually a ExprParser.StatContext + super().__init__(parser) + self.copyFrom(ctx) + + + + class PrintExprContext(StatContext): + + def __init__(self, parser, ctx:ParserRuleContext): # actually a ExprParser.StatContext + super().__init__(parser) + self.copyFrom(ctx) + + def expr(self): + return self.getTypedRuleContext(ExprParser.ExprContext,0) + + + + class AssignContext(StatContext): + + def __init__(self, parser, ctx:ParserRuleContext): # actually a ExprParser.StatContext + super().__init__(parser) + self.copyFrom(ctx) + + def ID(self): + return self.getToken(ExprParser.ID, 0) + def expr(self): + return self.getTypedRuleContext(ExprParser.ExprContext,0) + + + + + def stat(self): + + localctx = ExprParser.StatContext(self, self._ctx, self.state) + self.enterRule(localctx, 8, self.RULE_stat) + try: + self.state = 56 + self._errHandler.sync(self) + la_ = self._interp.adaptivePredict(self._input,3,self._ctx) + if la_ == 1: + localctx = ExprParser.PrintExprContext(self, localctx) + self.enterOuterAlt(localctx, 1) + self.state = 43 + self.expr(0) + self.state = 44 + self.match(ExprParser.T__6) + pass + + elif la_ == 2: + localctx = ExprParser.AssignContext(self, localctx) + self.enterOuterAlt(localctx, 2) + self.state = 46 + self.match(ExprParser.ID) + self.state = 47 + self.match(ExprParser.T__7) + self.state = 48 + self.expr(0) + self.state = 49 + self.match(ExprParser.T__6) + pass + + elif la_ == 3: + localctx = ExprParser.RetContext(self, localctx) + self.enterOuterAlt(localctx, 3) + self.state = 51 + self.match(ExprParser.RETURN) + self.state = 52 + self.expr(0) + self.state = 53 + self.match(ExprParser.T__6) + pass + + elif la_ == 4: + localctx = ExprParser.BlankContext(self, localctx) + self.enterOuterAlt(localctx, 4) + self.state = 55 + self.match(ExprParser.T__6) + pass + + + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class ExprContext(ParserRuleContext): + + def __init__(self, parser, parent:ParserRuleContext=None, invokingState:int=-1): + super().__init__(parent, invokingState) + self.parser = parser + + + def getRuleIndex(self): + return ExprParser.RULE_expr + + + def copyFrom(self, ctx:ParserRuleContext): + super().copyFrom(ctx) + + + class PrimContext(ExprContext): + + def __init__(self, parser, ctx:ParserRuleContext): # actually a ExprParser.ExprContext + super().__init__(parser) + self.copyFrom(ctx) + + def primary(self): + return self.getTypedRuleContext(ExprParser.PrimaryContext,0) + + + + class MulDivContext(ExprContext): + + def __init__(self, parser, ctx:ParserRuleContext): # actually a ExprParser.ExprContext + super().__init__(parser) + self.copyFrom(ctx) + + def expr(self, i:int=None): + if i is None: + return self.getTypedRuleContexts(ExprParser.ExprContext) + else: + return self.getTypedRuleContext(ExprParser.ExprContext,i) + + def MUL(self): + return self.getToken(ExprParser.MUL, 0) + def DIV(self): + return self.getToken(ExprParser.DIV, 0) + + + class AddSubContext(ExprContext): + + def __init__(self, parser, ctx:ParserRuleContext): # actually a ExprParser.ExprContext + super().__init__(parser) + self.copyFrom(ctx) + + def expr(self, i:int=None): + if i is None: + return self.getTypedRuleContexts(ExprParser.ExprContext) + else: + return self.getTypedRuleContext(ExprParser.ExprContext,i) + + def ADD(self): + return self.getToken(ExprParser.ADD, 0) + def SUB(self): + return self.getToken(ExprParser.SUB, 0) + + + + def expr(self, _p:int=0): + _parentctx = self._ctx + _parentState = self.state + localctx = ExprParser.ExprContext(self, self._ctx, _parentState) + _prevctx = localctx + _startState = 10 + self.enterRecursionRule(localctx, 10, self.RULE_expr, _p) + self._la = 0 # Token type + try: + self.enterOuterAlt(localctx, 1) + localctx = ExprParser.PrimContext(self, localctx) + self._ctx = localctx + _prevctx = localctx + + self.state = 59 + self.primary() + self._ctx.stop = self._input.LT(-1) + self.state = 69 + self._errHandler.sync(self) + _alt = self._interp.adaptivePredict(self._input,5,self._ctx) + while _alt!=2 and _alt!=ATN.INVALID_ALT_NUMBER: + if _alt==1: + if self._parseListeners is not None: + self.triggerExitRuleEvent() + _prevctx = localctx + self.state = 67 + self._errHandler.sync(self) + la_ = self._interp.adaptivePredict(self._input,4,self._ctx) + if la_ == 1: + localctx = ExprParser.MulDivContext(self, ExprParser.ExprContext(self, _parentctx, _parentState)) + self.pushNewRecursionContext(localctx, _startState, self.RULE_expr) + self.state = 61 + if not self.precpred(self._ctx, 3): + from antlr4.error.Errors import FailedPredicateException + raise FailedPredicateException(self, "self.precpred(self._ctx, 3)") + self.state = 62 + _la = self._input.LA(1) + if not(_la==ExprParser.MUL or _la==ExprParser.DIV): + self._errHandler.recoverInline(self) + else: + self._errHandler.reportMatch(self) + self.consume() + self.state = 63 + self.expr(4) + pass + + elif la_ == 2: + localctx = ExprParser.AddSubContext(self, ExprParser.ExprContext(self, _parentctx, _parentState)) + self.pushNewRecursionContext(localctx, _startState, self.RULE_expr) + self.state = 64 + if not self.precpred(self._ctx, 2): + from antlr4.error.Errors import FailedPredicateException + raise FailedPredicateException(self, "self.precpred(self._ctx, 2)") + self.state = 65 + _la = self._input.LA(1) + if not(_la==ExprParser.ADD or _la==ExprParser.SUB): + self._errHandler.recoverInline(self) + else: + self._errHandler.reportMatch(self) + self.consume() + self.state = 66 + self.expr(3) + pass + + + self.state = 71 + self._errHandler.sync(self) + _alt = self._interp.adaptivePredict(self._input,5,self._ctx) + + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.unrollRecursionContexts(_parentctx) + return localctx + + class PrimaryContext(ParserRuleContext): + + def __init__(self, parser, parent:ParserRuleContext=None, invokingState:int=-1): + super().__init__(parent, invokingState) + self.parser = parser + + + def getRuleIndex(self): + return ExprParser.RULE_primary + + + def copyFrom(self, ctx:ParserRuleContext): + super().copyFrom(ctx) + + + + class ParensContext(PrimaryContext): + + def __init__(self, parser, ctx:ParserRuleContext): # actually a ExprParser.PrimaryContext + super().__init__(parser) + self.copyFrom(ctx) + + def expr(self): + return self.getTypedRuleContext(ExprParser.ExprContext,0) + + + + class IdContext(PrimaryContext): + + def __init__(self, parser, ctx:ParserRuleContext): # actually a ExprParser.PrimaryContext + super().__init__(parser) + self.copyFrom(ctx) + + def ID(self): + return self.getToken(ExprParser.ID, 0) + + + class IntContext(PrimaryContext): + + def __init__(self, parser, ctx:ParserRuleContext): # actually a ExprParser.PrimaryContext + super().__init__(parser) + self.copyFrom(ctx) + + def INT(self): + return self.getToken(ExprParser.INT, 0) + + + + def primary(self): + + localctx = ExprParser.PrimaryContext(self, self._ctx, self.state) + self.enterRule(localctx, 12, self.RULE_primary) + try: + self.state = 78 + self._errHandler.sync(self) + token = self._input.LA(1) + if token in [ExprParser.INT]: + localctx = ExprParser.IntContext(self, localctx) + self.enterOuterAlt(localctx, 1) + self.state = 72 + self.match(ExprParser.INT) + pass + elif token in [ExprParser.ID]: + localctx = ExprParser.IdContext(self, localctx) + self.enterOuterAlt(localctx, 2) + self.state = 73 + self.match(ExprParser.ID) + pass + elif token in [ExprParser.T__1]: + localctx = ExprParser.ParensContext(self, localctx) + self.enterOuterAlt(localctx, 3) + self.state = 74 + self.match(ExprParser.T__1) + self.state = 75 + self.expr(0) + self.state = 76 + self.match(ExprParser.T__3) + pass + else: + raise NoViableAltException(self) + + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + + + def sempred(self, localctx:RuleContext, ruleIndex:int, predIndex:int): + if self._predicates == None: + self._predicates = dict() + self._predicates[5] = self.expr_sempred + pred = self._predicates.get(ruleIndex, None) + if pred is None: + raise Exception("No predicate with index:" + str(ruleIndex)) + else: + return pred(localctx, predIndex) + + def expr_sempred(self, localctx:ExprContext, predIndex:int): + if predIndex == 0: + return self.precpred(self._ctx, 3) + + + if predIndex == 1: + return self.precpred(self._ctx, 2) + + + + + diff --git a/runtime/Python3/test/run.py b/runtime/Python3/test/run.py index c9ae18877..5aad7896c 100644 --- a/runtime/Python3/test/run.py +++ b/runtime/Python3/test/run.py @@ -3,5 +3,6 @@ import os src_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'src') sys.path.insert(0,src_path) from TestTokenStreamRewriter import TestTokenStreamRewriter +from xpathtest import XPathTest import unittest unittest.main() \ No newline at end of file diff --git a/runtime/Python3/test/xpathtest.py b/runtime/Python3/test/xpathtest.py new file mode 100644 index 000000000..278bffe5b --- /dev/null +++ b/runtime/Python3/test/xpathtest.py @@ -0,0 +1,89 @@ +import antlr4 +from antlr4 import InputStream, CommonTokenStream, TerminalNode +from antlr4.xpath.XPath import XPath +import unittest +from expr.ExprParser import ExprParser +from expr.ExprLexer import ExprLexer + +def tokenToString(token, ruleNames): + if isinstance(token, TerminalNode): + return str(token) + else: + return ruleNames[token.getRuleIndex()] + +class XPathTest(unittest.TestCase): + def setUp(self): + self.input_stream = InputStream( + "def f(x,y) { x = 3+4; y; ; }\n" + "def g(x) { return 1+2*x; }\n" + ) + + # Create the Token Stream + self.lexer = ExprLexer(self.input_stream) + self.stream = CommonTokenStream(self.lexer) + self.stream.fill() + + # Create the parser and expression parse tree + self.parser = ExprParser(self.stream) + self.tree = self.parser.prog() + + def testValidPaths(self): + valid_paths = [ + "/prog/func", # all funcs under prog at root + "/prog/*", # all children of prog at root + "/*/func", # all func kids of any root node + "prog", # prog must be root node + "/prog", # prog must be root node + "/*", # any root + "*", # any root + "//ID", # any ID in tree + "//expr/primary/ID", # any ID child of a primary under any expr + "//body//ID", # any ID under a body + "//'return'", # any 'return' literal in tree, matched by literal name + "//RETURN", # any 'return' literal in tree, matched by symbolic name + "//primary/*", # all kids of any primary + "//func/*/stat", # all stat nodes grandkids of any func node + "/prog/func/'def'", # all def literal kids of func kid of prog + "//stat/';'", # all ';' under any stat node + "//expr/primary/!ID",# anything but ID under primary under any expr node + "//expr/!primary", # anything but primary under any expr node + "//!*", # nothing anywhere + "/!*", # nothing at root + "//expr//ID" # any ID under any expression (tests antlr/antlr4#370) + ] + + expected_results = [ + "[func, func]", + "[func, func]", + "[func, func]", + "[prog]", + "[prog]", + "[prog]", + "[prog]", + "[f, x, y, x, y, g, x, x]", + "[y, x]", + "[x, y, x]", + "[return]", + "[return]", + "[3, 4, y, 1, 2, x]", + "[stat, stat, stat, stat]", + "[def, def]", + "[;, ;, ;, ;]", + "[3, 4, 1, 2]", + "[expr, expr, expr, expr, expr, expr]", + "[]", + "[]", + "[y, x]", + ] + + for path, expected in zip(valid_paths, expected_results): + # Build test string + res = XPath.findAll(self.tree, path, self.parser) + res_str = ", ".join([tokenToString(token, self.parser.ruleNames) for token in res]) + res_str = "[%s]" % res_str + + # Test against expected output + self.assertEqual(res_str, expected, "Failed test %s" % path) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From b0eb1825fb943ffee8d92af6ef09d1b858ad1b00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Henrique?= Date: Wed, 4 Sep 2019 16:43:12 -0300 Subject: [PATCH 033/365] Spaces/Tabs mishap --- runtime/Python3/test/xpathtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/Python3/test/xpathtest.py b/runtime/Python3/test/xpathtest.py index 278bffe5b..03b6101a9 100644 --- a/runtime/Python3/test/xpathtest.py +++ b/runtime/Python3/test/xpathtest.py @@ -15,7 +15,7 @@ class XPathTest(unittest.TestCase): def setUp(self): self.input_stream = InputStream( "def f(x,y) { x = 3+4; y; ; }\n" - "def g(x) { return 1+2*x; }\n" + "def g(x) { return 1+2*x; }\n" ) # Create the Token Stream From ae282133430c326e4523f075f19a61c549e8d9a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Henrique?= Date: Thu, 5 Sep 2019 14:29:25 -0300 Subject: [PATCH 034/365] Python3 XPath: Use Token.INVALID_TYPE instead of -1 on getXPathElement --- runtime/Python3/src/antlr4/xpath/XPath.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/Python3/src/antlr4/xpath/XPath.py b/runtime/Python3/src/antlr4/xpath/XPath.py index 4d3a4f56f..58b05c466 100644 --- a/runtime/Python3/src/antlr4/xpath/XPath.py +++ b/runtime/Python3/src/antlr4/xpath/XPath.py @@ -218,7 +218,7 @@ class XPath(object): elif wordToken.type in [XPathLexer.TOKEN_REF, XPathLexer.STRING]: tsource = self.parser.getTokenStream().tokenSource - ttype = -1 + ttype = Token.INVALID_TYPE if wordToken.type == XPathLexer.TOKEN_REF: if word in tsource.ruleNames: ttype = tsource.ruleNames.index(word) + 1 @@ -226,7 +226,7 @@ class XPath(object): if word in tsource.literalNames: ttype = tsource.literalNames.index(word) - if ttype == -1: + if ttype == Token.INVALID_TYPE: raise Exception("%s at index %d isn't a valid token name" % (word, wordToken.tokenIndex)) return XPathTokenAnywhereElement(word, ttype) if anywhere else XPathTokenElement(word, ttype) From 3c1231738e7dd00330fd79f8ef1159e1e116f8d0 Mon Sep 17 00:00:00 2001 From: Iman Hosseini Date: Tue, 10 Sep 2019 02:29:13 -0400 Subject: [PATCH 035/365] Update contributors.txt --- contributors.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contributors.txt b/contributors.txt index d40bacd5f..c3a388f5b 100644 --- a/contributors.txt +++ b/contributors.txt @@ -221,3 +221,4 @@ YYYY/MM/DD, github id, Full name, email 2019/07/11, olowo726, Olof Wolgast, olof@baah.se 2019/07/16, abhijithneilabraham, Abhijith Neil Abraham, abhijithneilabrahampk@gmail.com 2019/07/26, Braavos96, Eric Hettiaratchi, erichettiaratchi@gmail.com +2019/09/10, ImanHosseini, Iman Hosseini, hosseini.iman@yahoo.com From e2b1ae7c79b4c04f1f9edd2a36523faf34e56515 Mon Sep 17 00:00:00 2001 From: neko1235 Date: Tue, 10 Sep 2019 16:01:16 -0700 Subject: [PATCH 036/365] Fix data race in LexerATNSimulator There is a potential memory consistency problem. --- .../Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index ec74ce5bd..9b246d3b7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -78,7 +78,7 @@ public class LexerATNSimulator extends ATNSimulator { protected final SimState prevAccept = new SimState(); - public static int match_calls = 0; + public static volatile int match_calls = 0; public LexerATNSimulator(ATN atn, DFA[] decisionToDFA, PredictionContextCache sharedContextCache) From f11ce46ddf244bc61be90775ade6fa3ca2e7833e Mon Sep 17 00:00:00 2001 From: neko1235 Date: Tue, 10 Sep 2019 16:08:49 -0700 Subject: [PATCH 037/365] Update contributors.txt --- contributors.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contributors.txt b/contributors.txt index f385aeeef..e9cefd6f0 100644 --- a/contributors.txt +++ b/contributors.txt @@ -222,4 +222,5 @@ YYYY/MM/DD, github id, Full name, email 2019/07/16, abhijithneilabraham, Abhijith Neil Abraham, abhijithneilabrahampk@gmail.com 2019/07/26, Braavos96, Eric Hettiaratchi, erichettiaratchi@gmail.com 2019/09/10, ImanHosseini, Iman Hosseini, hosseini.iman@yahoo.com -2019/09/03, João Henrique, johnnyonflame@hotmail.com \ No newline at end of file +2019/09/03, João Henrique, johnnyonflame@hotmail.com +2019/09/10, neko1235, Ihar Mokharau, igor.mohorev@gmail.com From db0a57c6ee4629b02c32abfb12fc2d325eaa5d57 Mon Sep 17 00:00:00 2001 From: neko1235 Date: Tue, 10 Sep 2019 21:16:49 -0700 Subject: [PATCH 038/365] Increment match_calls atomically This ensures the correctness of the counter value, i.e. the value is the number of calls to the match() method. --- .../Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index 9b246d3b7..79c6c73e5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -103,7 +103,9 @@ public class LexerATNSimulator extends ATNSimulator { } public int match(CharStream input, int mode) { - match_calls++; + synchronized (LexerATNSimulator.class) { + match_calls++; + } this.mode = mode; int mark = input.mark(); try { From f88f763983e4b48c48646bd233f80d17f56d949d Mon Sep 17 00:00:00 2001 From: neko1235 Date: Tue, 10 Sep 2019 22:44:42 -0700 Subject: [PATCH 039/365] Remove the match_calls counter This fixes the potential data race caused by unsynchronized concurrent access. --- .../Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index 79c6c73e5..79d903053 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -78,8 +78,6 @@ public class LexerATNSimulator extends ATNSimulator { protected final SimState prevAccept = new SimState(); - public static volatile int match_calls = 0; - public LexerATNSimulator(ATN atn, DFA[] decisionToDFA, PredictionContextCache sharedContextCache) { @@ -103,9 +101,6 @@ public class LexerATNSimulator extends ATNSimulator { } public int match(CharStream input, int mode) { - synchronized (LexerATNSimulator.class) { - match_calls++; - } this.mode = mode; int mark = input.mark(); try { From 789d746636dd9fd84862f4950922feaeb3f9f4b8 Mon Sep 17 00:00:00 2001 From: Marcos Passos Date: Sat, 14 Sep 2019 09:36:37 -0300 Subject: [PATCH 040/365] PHP Target --- .gitmodules | 3 + .travis.yml | 7 + .travis/before-install-linux-php.sh | 10 + .travis/run-tests-php.sh | 9 + README.md | 1 + appveyor.yml | 5 +- contributors.txt | 5 +- doc/php-target.md | 111 ++ doc/targets.md | 7 +- runtime-testsuite/pom.xml | 17 +- .../v4/test/runtime/templates/CSharp.test.stg | 10 + .../v4/test/runtime/templates/Chrome.test.stg | 10 + .../v4/test/runtime/templates/Cpp.test.stg | 6 + .../test/runtime/templates/Explorer.test.stg | 10 + .../test/runtime/templates/Firefox.test.stg | 10 + .../v4/test/runtime/templates/Go.test.stg | 10 + .../v4/test/runtime/templates/Java.test.stg | 10 + .../v4/test/runtime/templates/Node.test.stg | 10 + .../v4/test/runtime/templates/PHP.test.stg | 272 ++++ .../test/runtime/templates/Python2.test.stg | 10 + .../test/runtime/templates/Python3.test.stg | 10 + .../v4/test/runtime/templates/Safari.test.stg | 10 + .../v4/test/runtime/templates/Swift.test.stg | 10 + .../v4/test/runtime/BaseRuntimeTest.java | 2 + .../CompositeParsersDescriptors.java | 4 +- .../FullContextParsingDescriptors.java | 2 +- .../descriptors/LeftRecursionDescriptors.java | 8 +- .../descriptors/LexerExecDescriptors.java | 4 + .../SemPredEvalParserDescriptors.java | 6 +- .../v4/test/runtime/php/BasePHPTest.java | 599 ++++++++ .../test/runtime/php/TestCompositeLexers.java | 25 + .../runtime/php/TestCompositeParsers.java | 25 + .../runtime/php/TestFullContextParsing.java | 25 + .../test/runtime/php/TestLeftRecursion.java | 25 + .../v4/test/runtime/php/TestLexerErrors.java | 25 + .../v4/test/runtime/php/TestLexerExec.java | 25 + .../v4/test/runtime/php/TestListeners.java | 25 + .../v4/test/runtime/php/TestParseTrees.java | 25 + .../v4/test/runtime/php/TestParserErrors.java | 25 + .../v4/test/runtime/php/TestParserExec.java | 25 + .../v4/test/runtime/php/TestPerformance.java | 25 + .../runtime/php/TestSemPredEvalLexer.java | 25 + .../runtime/php/TestSemPredEvalParser.java | 25 + .../antlr/v4/test/runtime/php/TestSets.java | 25 + runtime/PHP | 1 + .../v4/tool/templates/codegen/PHP/PHP.stg | 1231 +++++++++++++++++ .../model/decl/AltLabelStructDecl.java | 2 + .../antlr/v4/codegen/target/PHPTarget.java | 105 ++ 48 files changed, 2854 insertions(+), 23 deletions(-) create mode 100644 .gitmodules create mode 100755 .travis/before-install-linux-php.sh create mode 100755 .travis/run-tests-php.sh create mode 100644 doc/php-target.md create mode 100644 runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/PHP.test.stg create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestCompositeLexers.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestCompositeParsers.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestFullContextParsing.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestLeftRecursion.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestLexerErrors.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestLexerExec.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestListeners.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestParseTrees.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestParserErrors.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestParserExec.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestPerformance.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestSemPredEvalLexer.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestSemPredEvalParser.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestSets.java create mode 160000 runtime/PHP create mode 100644 tool/resources/org/antlr/v4/tool/templates/codegen/PHP/PHP.stg create mode 100644 tool/src/org/antlr/v4/codegen/target/PHPTarget.java diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..7944e8e32 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "runtime/PHP"] + path = runtime/PHP + url = https://github.com/antlr/antlr-php-runtime.git diff --git a/.travis.yml b/.travis.yml index 880cf8907..0220f4ca2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -152,6 +152,13 @@ matrix: jdk: openjdk8 env: TARGET=csharp stage: main-test + - os: linux + language: php + php: + - 7.2 + jdk: openjdk8 + env: TARGET=php + stage: main-test - os: linux jdk: openjdk8 dist: trusty diff --git a/.travis/before-install-linux-php.sh b/.travis/before-install-linux-php.sh new file mode 100755 index 000000000..b95e3b31d --- /dev/null +++ b/.travis/before-install-linux-php.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -euo pipefail + +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +sudo apt-get update -qq + +php -v + +mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V \ No newline at end of file diff --git a/.travis/run-tests-php.sh b/.travis/run-tests-php.sh new file mode 100755 index 000000000..853efbd86 --- /dev/null +++ b/.travis/run-tests-php.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail + +php_path=$(which php) + +composer install -d ../runtime/PHP + +mvn -q -DPHP_PATH="${php_path}" -Dparallel=methods -DthreadCount=4 -Dtest=php.* test diff --git a/README.md b/README.md index d63bc4f66..ec5c21c62 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ ANTLR project lead and supreme dictator for life * [Janyou](https://github.com/janyou) (Swift target) * [Ewan Mellor](https://github.com/ewanmellor), [Hanzhou Shi](https://github.com/hanjoes) (Swift target merging) * [Ben Hamilton](https://github.com/bhamiltoncx) (Full Unicode support in serialized ATN and all languages' runtimes for code points > U+FFFF) +* [Marcos Passos](https://github.com/marcospassos) (PHP target) ## Useful information diff --git a/appveyor.yml b/appveyor.yml index 266eeebad..d58657c98 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,6 +4,9 @@ cache: - '%USERPROFILE%\.nuget\packages -> **\project.json' image: Visual Studio 2017 build: off +install: + - git submodule update --init --recursive + - cinst -y php composer build_script: - mvn -DskipTests install --batch-mode - msbuild /target:restore /target:rebuild /property:Configuration=Release /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /verbosity:detailed runtime/CSharp/runtime/CSharp/Antlr4.dotnet.sln @@ -11,7 +14,7 @@ build_script: after_build: - msbuild /target:pack /property:Configuration=Release /verbosity:detailed runtime/CSharp/runtime/CSharp/Antlr4.dotnet.sln test_script: - - mvn install -Dantlr-python2-python="C:\Python27\python.exe" -Dantlr-python3-python="C:\Python35\python.exe" -Dantlr-javascript-nodejs="C:\Program Files (x86)\nodejs\node.exe" --batch-mode + - mvn install -Dantlr-php-php="C:\tools\php73\php.exe" -Dantlr-python2-python="C:\Python27\python.exe" -Dantlr-python3-python="C:\Python35\python.exe" -Dantlr-javascript-nodejs="C:\Program Files (x86)\nodejs\node.exe" --batch-mode artifacts: - path: 'runtime\**\*.nupkg' name: NuGet \ No newline at end of file diff --git a/contributors.txt b/contributors.txt index f385aeeef..6ff7f398d 100644 --- a/contributors.txt +++ b/contributors.txt @@ -222,4 +222,7 @@ YYYY/MM/DD, github id, Full name, email 2019/07/16, abhijithneilabraham, Abhijith Neil Abraham, abhijithneilabrahampk@gmail.com 2019/07/26, Braavos96, Eric Hettiaratchi, erichettiaratchi@gmail.com 2019/09/10, ImanHosseini, Iman Hosseini, hosseini.iman@yahoo.com -2019/09/03, João Henrique, johnnyonflame@hotmail.com \ No newline at end of file +2019/09/03, João Henrique, johnnyonflame@hotmail.com +2019/09/10, yar3333, Yaroslav Sivakov, yar3333@gmail.com +2019/09/10, marcospassos, Marcos Passos, marcospassos.com@gmail.com +2019/09/10, amorimjuliana, Juliana Amorim, juu.amorim@gmail.com \ No newline at end of file diff --git a/doc/php-target.md b/doc/php-target.md new file mode 100644 index 000000000..75eae465c --- /dev/null +++ b/doc/php-target.md @@ -0,0 +1,111 @@ +# ANTLR4 Runtime for PHP + +### First steps + +#### 1. Install ANTLR4 + +[The getting started guide](https://github.com/antlr/antlr4/blob/master/doc/getting-started.md) +should get you started. + +#### 2. Install the PHP ANTLR runtime + +Each target language for ANTLR has a runtime package for running parser +generated by ANTLR4. The runtime provides a common set of tools for using your parser. + +Install the runtime with Composer: + +```bash +composer install antlr/antlr4 +``` + +#### 3. Generate your parser + +You use the ANTLR4 "tool" to generate a parser. These will reference the ANTLR +runtime, installed above. + +Suppose you're using a UNIX system and have set up an alias for the ANTLR4 tool +as described in [the getting started guide](https://github.com/antlr/antlr4/blob/master/doc/getting-started.md). +To generate your PHP parser, run the following command: + +```bash +antlr4 -Dlanguage=PHP MyGrammar.g4 +``` + +For a full list of antlr4 tool options, please visit the +[tool documentation page](https://github.com/antlr/antlr4/blob/master/doc/tool-options.md). + +### Complete example + +Suppose you're using the JSON grammar from https://github.com/antlr/grammars-v4/tree/master/json. + +Then, invoke `antlr4 -Dlanguage=PHP JSON.g4`. The result of this is a +collection of `.php` files in the `parser` directory including: +``` +JsonParser.php +JsonBaseListener.php +JsonLexer.php +JsonListener.php +``` + +Another common option to the ANTLR tool is `-visitor`, which generates a parse +tree visitor, but we won't be doing that here. For a full list of antlr4 tool +options, please visit the [tool documentation page](tool-options.md). + +We'll write a small main func to call the generated parser/lexer +(assuming they are separate). This one writes out the encountered +`ParseTreeContext`'s: + +```php +getText(); + } +} + +$input = InputStream::fromPath($argv[1]); +$lexer = new JSONLexer($input); +$tokens = new CommonTokenStream($lexer); +$parser = new JSONParser($tokens); +$parser->addErrorListener(new DiagnosticErrorListener()); +$parser->setBuildParseTree(true); +$tree = $parser->json(); + +ParseTreeWalker::default()->walk(new TreeShapeListener(), $tree); +``` + +Create a `example.json` file: +```json +{"a":1} +``` + +Parse the input file: + +``` +php json.php example.json +``` + +The expected output is: + +``` +{"a":1} +{"a":1} +"a":1 +1 +``` \ No newline at end of file diff --git a/doc/targets.md b/doc/targets.md index 418630bb1..c2341ec48 100644 --- a/doc/targets.md +++ b/doc/targets.md @@ -9,12 +9,13 @@ This page lists the available and upcoming ANTLR runtimes. Please note that you * [Go](go-target.md) * [C++](cpp-target.md) * [Swift](swift-target.md) +* [PHP](php-target.md) ## Target feature parity New features generally appear in the Java target and then migrate to the other targets, but these other targets don't always get updated in the same overall tool release. This section tries to identify features added to Java that have not been added to the other targets. -|Feature|Java|C♯|Python2|Python3|JavaScript|Go|C++|Swift| -|---|---|---|---|---|---|---|---|---| -|Ambiguous tree construction|4.5.1|-|-|-|-|-|-|-| +|Feature|Java|C♯|Python2|Python3|JavaScript|Go|C++|Swift|PHP +|---|---|---|---|---|---|---|---|---|---| +|Ambiguous tree construction|4.5.1|-|-|-|-|-|-|-|-| diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index ebfdbd723..b9e6bd936 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -109,14 +109,15 @@ -Dfile.encoding=UTF-8 - **/csharp/Test*.java - **/java/Test*.java - **/go/Test*.java - **/javascript/node/Test*.java - **/python2/Test*.java - **/python3/Test*.java - ${antlr.tests.swift} - + **/csharp/Test*.java + **/java/Test*.java + **/go/Test*.java + **/javascript/node/Test*.java + **/python2/Test*.java + **/python3/Test*.java + **/php/Test*.java + ${antlr.tests.swift} + diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CSharp.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CSharp.test.stg index 597248f2b..fb95440ce 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CSharp.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CSharp.test.stg @@ -14,6 +14,8 @@ Cast(t,v) ::= "(())" Append(a,b) ::= " + " +AppendStr(a,b) ::= <%%> + Concat(a,b) ::= "" DeclareLocal(s,v) ::= "Object = ;" @@ -26,6 +28,12 @@ InitIntMember(n,v) ::= <%int = ;%> InitBooleanMember(n,v) ::= <%bool = ;%> +InitIntVar(n,v) ::= <%%> + +IntArg(n) ::= "int " + +VarRef(n) ::= "" + GetMember(n) ::= <%this.%> SetMember(n,v) ::= <%this. = ;%> @@ -94,6 +102,8 @@ bool Property() { ParserPropertyCall(p, call) ::= "

." +PositionAdjustingLexerDef() ::= "" + PositionAdjustingLexer() ::= << PositionAdjustingLexer.prototype.resetAcceptPosition = function(index, line, column) { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Firefox.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Firefox.test.stg index c94be90e8..c8a6c1d57 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Firefox.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Firefox.test.stg @@ -14,6 +14,8 @@ Cast(t,v) ::= "" Append(a,b) ::= " + " +AppendStr(a,b) ::= <%%> + Concat(a,b) ::= "" DeclareLocal(s,v) ::= "var = ;" @@ -26,6 +28,12 @@ InitIntMember(n,v) ::= <%this. = ;%> InitBooleanMember(n,v) ::= <%this. = ;%> +InitIntVar(n,v) ::= <%%> + +IntArg(n) ::= "" + +VarRef(n) ::= "" + GetMember(n) ::= <%this.%> SetMember(n,v) ::= <%this. = ;%> @@ -100,6 +108,8 @@ this.Property = function() { ParserPropertyCall(p, call) ::= "

." +PositionAdjustingLexerDef() ::= "" + PositionAdjustingLexer() ::= << PositionAdjustingLexer.prototype.resetAcceptPosition = function(index, line, column) { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Go.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Go.test.stg index 37fe63441..b2909f4e1 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Go.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Go.test.stg @@ -14,6 +14,8 @@ Cast(t,v) ::= "()" Append(a,b) ::= " + fmt.Sprint()" +AppendStr(a,b) ::= " + " + Concat(a,b) ::= "" DeclareLocal(s, v) ::= "var = " @@ -26,6 +28,12 @@ InitIntMember(n, v) ::= <%var int = ; var _ int = ; %> InitBooleanMember(n, v) ::= <%var bool = ; var _ bool = ; %> +InitIntVar(n,v) ::= <%%> + +IntArg(n) ::= "int " + +VarRef(n) ::= "" + GetMember(n) ::= <%%> SetMember(n, v) ::= <% = ;%> @@ -94,6 +102,8 @@ func (p *TParser) Property() bool { ParserPropertyCall(p, call) ::= "

." +PositionAdjustingLexerDef() ::= "" + PositionAdjustingLexer() ::= << func (p *PositionAdjustingLexer) NextToken() antlr.Token { if _, ok := p.Interpreter.(*PositionAdjustingLexerATNSimulator); !ok { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg index 5012aa213..11f159743 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg @@ -14,6 +14,8 @@ Cast(t,v) ::= "(())" Append(a,b) ::= " + " +AppendStr(a,b) ::= <%%> + Concat(a,b) ::= "" DeclareLocal(s,v) ::= "Object = ;" @@ -26,6 +28,12 @@ InitIntMember(n,v) ::= <%int = ;%> InitBooleanMember(n,v) ::= <%boolean = ;%> +InitIntVar(n,v) ::= <%%> + +IntArg(n) ::= "int " + +VarRef(n) ::= "" + GetMember(n) ::= <%this.%> SetMember(n,v) ::= <%this. = ;%> @@ -94,6 +102,8 @@ boolean Property() { ParserPropertyCall(p, call) ::= "

." +PositionAdjustingLexerDef() ::= "" + PositionAdjustingLexer() ::= << @Override diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Node.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Node.test.stg index 92cb1dbbb..81573d6c0 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Node.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Node.test.stg @@ -14,6 +14,8 @@ Cast(t,v) ::= "" Append(a,b) ::= " + " +AppendStr(a,b) ::= <%%> + Concat(a,b) ::= "" DeclareLocal(s,v) ::= "var = ;" @@ -26,6 +28,12 @@ InitIntMember(n,v) ::= <%this. = ;%> InitBooleanMember(n,v) ::= <%this. = ;%> +InitIntVar(n,v) ::= <%%> + +IntArg(n) ::= "" + +VarRef(n) ::= "" + GetMember(n) ::= <%this.%> SetMember(n,v) ::= <%this. = ;%> @@ -98,6 +106,8 @@ this.Property = function() { ParserPropertyCall(p, call) ::= "

." +PositionAdjustingLexerDef() ::= "" + PositionAdjustingLexer() ::= << PositionAdjustingLexer.prototype.resetAcceptPosition = function(index, line, column) { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/PHP.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/PHP.test.stg new file mode 100644 index 000000000..a2ea87466 --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/PHP.test.stg @@ -0,0 +1,272 @@ +writeln(s) ::= < . \PHP_EOL;>> +write(s) ::= <;>> +writeList(s) ::= <;>> + +False() ::= "false" +True() ::= "true" +Not(v) ::= "!" +Assert(s) ::= <);>> +Cast(t,v) ::= "" +Append(a,b) ::= " . " +AppendStr(a,b) ::= <%%> +Concat(a,b) ::= "" + +DeclareLocal(s,v) ::= " = ;" + +AssertIsList(v) ::= "assert(\is_array());" // just use static type system +AssignLocal(s,v) ::= " = ;" + +InitIntMember(n,v) ::= <%public \$ = ;%> +InitBooleanMember(n,v) ::= <%public \$ = ;%> +InitIntVar(n,v) ::= <%\$ = ;%> +IntArg(n) ::= "int " +VarRef(n) ::= "$" + +GetMember(n) ::= <%\$this->%> +SetMember(n,v) ::= <%\$this-> = ;%> +AddMember(n,v) ::= <%\$this-> += ;%> +PlusMember(v,n) ::= <% + \$this->%> +MemberEquals(n,v) ::= <%\$this-> == %> +ModMemberEquals(n,m,v) ::= <%\$this-> % == %> +ModMemberNotEquals(n,m,v) ::= <%\$this-> % !== %> + +DumpDFA() ::= "\$this->dumpDFA();" +Pass() ::= "" + +StringList() ::= "" +BuildParseTrees() ::= "\$this->setBuildParseTree(true);" +BailErrorStrategy() ::= <%\$this->setErrorHandler(new Antlr\\Antlr4\\Runtime\\Error\\BailErrorStrategy());%> + +ToStringTree(s) ::= <%->toStringTree(\$this->getRuleNames())%> +Column() ::= "\$this->getCharPositionInLine()" +Text() ::= "\$this->getText()" +ValEquals(a,b) ::= <%===%> +TextEquals(a) ::= <%\$this->getText() === ""%> +PlusText(a) ::= <%"" . \$this->getText()%> +InputText() ::= "\$this->input->getText()" +LTEquals(i, v) ::= <%\$this->input->LT()->getText() === %> +LANotEquals(i, v) ::= <%\$this->input->LA() !== %> +TokenStartColumnEquals(i) ::= <%\$this->tokenStartCharPositionInLine === %> + +ImportListener(X) ::= "" + +GetExpectedTokenNames() ::= "\$this->getExpectedTokens()->toStringVocabulary(\$this->getVocabulary())" + +RuleInvocationStack() ::= "'[' . \implode(', ', \$this->getRuleInvocationStack()) . ']'" + +LL_EXACT_AMBIG_DETECTION() ::= <<\$this->interp->setPredictionMode(Antlr\\Antlr4\\Runtime\\Atn\\PredictionMode::LL_EXACT_AMBIG_DETECTION);>> + + +ParserToken(parser, token) ::= <%::%> + +Production(p) ::= <%

%> + +Result(r) ::= <%%> + +ParserPropertyMember() ::= << +@members { +public function Property() : bool +{ + return true; +} +} +>> + +ParserPropertyCall(p, call) ::= "

->" + +PositionAdjustingLexerDef() ::= << +class PositionAdjustingLexerATNSimulator extends LexerATNSimulator { + public function resetAcceptPosition(CharStream \$input, int \$index, int \$line, int \$charPositionInLine) : void { + \$input->seek(\$index); + \$this->line = \$line; + \$this->charPositionInLine = \$charPositionInLine; + \$this->consume(\$input); + } +} +>> + +PositionAdjustingLexer() ::= << +public function nextToken() : Antlr\\Antlr4\\Runtime\\Token +{ + if (!\$this->interp instanceof PositionAdjustingLexerATNSimulator) { + \$this->interp = new PositionAdjustingLexerATNSimulator(\$this, self::\$atn, self::\$decisionToDFA, self::\$sharedContextCache); + } + + return parent::nextToken(); +} + +public function emit() : Antlr\\Antlr4\\Runtime\\Token +{ + switch (\$this->type) { + case self::TOKENS: + \$this->handleAcceptPositionForKeyword('tokens'); + break; + + case self::LABEL: + \$this->handleAcceptPositionForIdentifier(); + break; + } + + return parent::emit(); +} + +private function handleAcceptPositionForIdentifier() : bool +{ + \$tokenText = \$this->getText(); + \$identifierLength = 0; + while (\$identifierLength \< \strlen(\$tokenText) && self::isIdentifierChar(\$tokenText[\$identifierLength])) { + \$identifierLength++; + } + + if (\$this->getInputStream()->getIndex() > \$this->tokenStartCharIndex + \$identifierLength) { + \$offset = \$identifierLength - 1; + \$this->getInterpreter()->resetAcceptPosition(\$this->getInputStream(), \$this->tokenStartCharIndex + \$offset, \$this->tokenStartLine, \$this->tokenStartCharPositionInLine + \$offset); + + return true; + } + + return false; +} + +private function handleAcceptPositionForKeyword(string \$keyword) : bool +{ + if (\$this->getInputStream()->getIndex() > \$this->tokenStartCharIndex + \strlen(\$keyword)) { + \$offset = \strlen(\$keyword) - 1; + \$this->getInterpreter()->resetAcceptPosition(\$this->getInputStream(), \$this->tokenStartCharIndex + \$offset, \$this->tokenStartLine, \$this->tokenStartCharPositionInLine + \$offset); + + return true; + } + + return false; +} + +private static function isIdentifierChar(string \$c) : bool +{ + return \ctype_alnum(\$c) || \$c === '_'; +} +>> + +BasicListener(X) ::= << +@parser::definitions { +class LeafListener extends TBaseListener +{ + public function visitTerminal(Antlr\\Antlr4\\Runtime\\Tree\\TerminalNode \$node) : void + { + echo \$node->getSymbol()->getText() . \PHP_EOL; + } +} +} +>> + +WalkListener(s) ::= << +\$walker = new Antlr\\Antlr4\\Runtime\\Tree\\ParseTreeWalker(); +\$walker->walk(new LeafListener(), ); +>> + +TreeNodeWithAltNumField(X) ::= << +@parser::contexts { +class MyRuleNode extends ParserRuleContext +{ + public \$altNum; + + public function getAltNumber() : int + { + return \$this->altNum; + } + + public function setAltNumber(int \$altNum) : void + { + \$this->altNum = \$altNum; + } +} +} +>> + +TokenGetterListener(X) ::= << +@parser::definitions { +class LeafListener extends TBaseListener { + public function exitA(Context\\AContext \$ctx) : void { + if (\$ctx->getChildCount() === 2) { + echo \sprintf('%s %s [%s]',\$ctx->INT(0)->getSymbol()->getText(), \$ctx->INT(1)->getSymbol()->getText(), \implode(', ', \$ctx->INT())) . \PHP_EOL; + } else { + echo \$ctx->ID()->getSymbol() . \PHP_EOL; + } + } +} +} +>> + +RuleGetterListener(X) ::= << +@parser::definitions { +class LeafListener extends TBaseListener { + public function exitA(Context\\AContext \$ctx) : void + { + if (\$ctx->getChildCount() === 2) { + echo \sprintf('%s %s %s', \$ctx->b(0)->start->getText(), \$ctx->b(1)->start->getText(),\$ctx->b()[0]->start->getText()) . \PHP_EOL; + } else { + echo \$ctx->b(0)->start->getText() . \PHP_EOL; + } + } +} +} +>> + + +LRListener(X) ::= << +@parser::definitions { +class LeafListener extends TBaseListener { + public function exitE(Context\\EContext \$ctx) : void + { + if (\$ctx->getChildCount() === 3) { + echo \sprintf('%s %s %s', \$ctx->e(0)->start->getText(), \$ctx->e(1)->start->getText(), \$ctx->e()[0]->start->getText()) . \PHP_EOL; + } else { + echo \$ctx->INT()->getSymbol()->getText() . \PHP_EOL; + } + } +} +} +>> + +LRWithLabelsListener(X) ::= << +@parser::definitions { +class LeafListener extends TBaseListener +{ + public function exitCall(Context\\CallContext \$ctx) : void { + echo \sprintf('%s %s',\$ctx->e()->start->getText(),\$ctx->eList()) . \PHP_EOL; + } + + public function exitInt(Context\\IntContext \$ctx) : void { + echo \$ctx->INT()->getSymbol()->getText() . \PHP_EOL; + } +} +} +>> + +DeclareContextListGettersFunction() ::= << +public function foo() : void { + \$s = null; + \$a = \$s->a(); + \$b = \$s->b(); +} +>> + +Declare_foo() ::= << +public function foo() : void { + echo 'foo' . \PHP_EOL; +} +>> + +Invoke_foo() ::= "\$this->foo();" + +Declare_pred() ::= <pred()" + +ParserTokenType(t) ::= "Parser::" +ContextRuleFunction(ctx, rule) ::= "->" +StringType() ::= "" +ContextMember(ctx, subctx, member) ::= "->->" diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python2.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python2.test.stg index cc0d9d9c2..3102f4120 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python2.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python2.test.stg @@ -14,6 +14,8 @@ Cast(t,v) ::= "" Append(a,b) ::= " + str()" +AppendStr(a,b) ::= " + " + Concat(a,b) ::= "" DeclareLocal(s,v) ::= " = " @@ -26,6 +28,12 @@ InitIntMember(n,v) ::= <% = %> InitBooleanMember(n,v) ::= <% = %> +InitIntVar(n,v) ::= <%%> + +IntArg(n) ::= "" + +VarRef(n) ::= "" + GetMember(n) ::= <%self.%> SetMember(n,v) ::= <%self. = %> @@ -94,6 +102,8 @@ def Property(self): ParserPropertyCall(p, call) ::= "

." +PositionAdjustingLexerDef() ::= "" + PositionAdjustingLexer() ::= << def resetAcceptPosition(self, index, line, column): diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python3.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python3.test.stg index c91563017..858b76113 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python3.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python3.test.stg @@ -14,6 +14,8 @@ Cast(t,v) ::= "" Append(a,b) ::= " + str()" +AppendStr(a,b) ::= " + " + Concat(a,b) ::= "" DeclareLocal(s,v) ::= " = " @@ -26,6 +28,12 @@ InitIntMember(n,v) ::= <% = %> InitBooleanMember(n,v) ::= <% = %> +InitIntVar(n,v) ::= <%%> + +IntArg(n) ::= "" + +VarRef(n) ::= "" + GetMember(n) ::= <%self.%> SetMember(n,v) ::= <%self. = %> @@ -99,6 +107,8 @@ def Property(self): ParserPropertyCall(p, call) ::= "

." +PositionAdjustingLexerDef() ::= "" + PositionAdjustingLexer() ::= << def resetAcceptPosition(self, index, line, column): diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Safari.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Safari.test.stg index 71254e0f2..2a09970c2 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Safari.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Safari.test.stg @@ -14,6 +14,8 @@ Cast(t,v) ::= "" Append(a,b) ::= " + " +AppendStr(a,b) ::= <%%> + Concat(a,b) ::= "" DeclareLocal(s,v) ::= "var = ;" @@ -26,6 +28,12 @@ InitIntMember(n,v) ::= <%this. = ;%> InitBooleanMember(n,v) ::= <%this. = ;%> +InitIntVar(n,v) ::= <%%> + +IntArg(n) ::= "" + +VarRef(n) ::= "" + GetMember(n) ::= <%this.%> SetMember(n,v) ::= <%this. = ;%> @@ -98,6 +106,8 @@ this.Property = function() { ParserPropertyCall(p, call) ::= "

." +PositionAdjustingLexerDef() ::= "" + PositionAdjustingLexer() ::= << PositionAdjustingLexer.prototype.resetAcceptPosition = function(index, line, column) { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg index 100d6f3e7..d45168517 100755 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg @@ -14,6 +14,8 @@ Cast(t,v) ::= "(( as! ))" Append(a,b) ::= " + " +AppendStr(a,b) ::= <%%> + Concat(a,b) ::= "" DeclareLocal(s,v) ::= "var = " @@ -26,6 +28,12 @@ InitIntMember(n,v) ::= <%var = %> InitBooleanMember(n,v) ::= <%var = %> +InitIntVar(n,v) ::= <%%> + +IntArg(n) ::= "int " + +VarRef(n) ::= "" + GetMember(n) ::= <%self.%> SetMember(n,v) ::= <%self. = %> @@ -96,6 +104,8 @@ ParserPropertyCall(parser, property) ::= << . >> +PositionAdjustingLexerDef() ::= "" + PositionAdjustingLexer() ::= << override diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java index 71cd1e34d..f7874d671 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java @@ -50,6 +50,7 @@ public abstract class BaseRuntimeTest { "Go", "CSharp", "Python2", "Python3", + "PHP", "Node", "Safari", "Firefox", "Explorer", "Chrome" }; public final static String[] JavaScriptTargets = { @@ -116,6 +117,7 @@ public abstract class BaseRuntimeTest { System.out.printf("Ignore "+descriptor); return; } + if ( descriptor.getTestType().contains("Parser") ) { testParser(descriptor); } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/CompositeParsersDescriptors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/CompositeParsersDescriptors.java index 435dcc977..6134b265b 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/CompositeParsersDescriptors.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/CompositeParsersDescriptors.java @@ -418,7 +418,7 @@ public class CompositeParsersDescriptors { parser grammar S; type_ : 'int' ; decl : type_ ID ';' - | type_ ID init_ ';' {}; + | type_ ID init_ ';' {}; init_ : '=' INT; */ @CommentHasStringValue @@ -532,7 +532,7 @@ public class CompositeParsersDescriptors { /** parser grammar S; - a @after {} : B; + a @after {} : B; */ @CommentHasStringValue public String slaveGrammarS; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/FullContextParsingDescriptors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/FullContextParsingDescriptors.java index 942d07be5..fb71b3863 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/FullContextParsingDescriptors.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/FullContextParsingDescriptors.java @@ -372,7 +372,7 @@ public class FullContextParsingDescriptors { : expr_or_assign*; expr_or_assign : expr '++' {} - | expr {} + | expr {} ; expr: expr_primary ('\<-' ID)?; expr_primary diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LeftRecursionDescriptors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LeftRecursionDescriptors.java index 66dd5760d..01e868d6e 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LeftRecursionDescriptors.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LeftRecursionDescriptors.java @@ -529,8 +529,8 @@ public class LeftRecursionDescriptors { | e '+' e {$v = (0)}, {})> + (1)}, {})>;} # binary | INT {$v = $INT.int;} # anInt | '(' e ')' {$v = $e.v;} # parens - | left=e INC {$v = $left.v + 1;} # unary - | left=e DEC {$v = $left.v - 1;} # unary + | left=e INC {$v = $left.v + 1;} # unary + | left=e DEC {$v = $left.v - 1;} # unary | ID {} # anID ; ID : 'a'..'z'+ ; @@ -636,9 +636,9 @@ public class LeftRecursionDescriptors { grammar T; s : e {} ; e returns [ result] - : ID '=' e1=e {$result = "(" + $ID.text + "=" + $e1.result + ")";} + : ID '=' e1=e {$result = ;} | ID {$result = $ID.text;} - | e1=e '+' e2=e {$result = "(" + $e1.result + "+" + $e2.result + ")";} + | e1=e '+' e2=e {$result = ;} ; ID : 'a'..'z'+ ; INT : '0'..'9'+ ; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LexerExecDescriptors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LexerExecDescriptors.java index 60cb86cb4..7c74e74cb 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LexerExecDescriptors.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LexerExecDescriptors.java @@ -766,6 +766,10 @@ public class LexerExecDescriptors { /** lexer grammar PositionAdjustingLexer; + @definitions { + + } + @members { } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/SemPredEvalParserDescriptors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/SemPredEvalParserDescriptors.java index ed4f9a384..b09f07488 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/SemPredEvalParserDescriptors.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/SemPredEvalParserDescriptors.java @@ -144,8 +144,8 @@ public class SemPredEvalParserDescriptors { /** grammar T; s : b[2] ';' | b[2] '.' ; // decision in s drills down to ctx-dependent pred in a; - b[int i] : a[i] ; - a[int i] + b[] : a[] ; + a[] : {}? ID {} | {}? ID {} ; @@ -310,7 +310,7 @@ public class SemPredEvalParserDescriptors { grammar T; @parser::members {} primary - : ID {} + : ID {} | {}? 'enum' {} ; ID : [a-z]+ ; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java new file mode 100644 index 000000000..8f2cc5bcc --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java @@ -0,0 +1,599 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.antlr.v4.Tool; +import org.antlr.v4.automata.LexerATNFactory; +import org.antlr.v4.automata.ParserATNFactory; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNDeserializer; +import org.antlr.v4.runtime.atn.ATNSerializer; +import org.antlr.v4.semantics.SemanticPipeline; +import org.antlr.v4.test.runtime.ErrorQueue; +import org.antlr.v4.test.runtime.RuntimeTestSupport; +import org.antlr.v4.test.runtime.StreamVacuum; +import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.LexerGrammar; +import org.stringtemplate.v4.ST; + +import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString; +import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class BasePHPTest implements RuntimeTestSupport { + public static final String newline = System.getProperty("line.separator"); + + public String tmpdir = null; + + /** + * If error during parser execution, store stderr here; can't return + * stdout and stderr. This doesn't trap errors from running antlr. + */ + protected String stderrDuringParse; + + /** + * Errors found while running antlr + */ + protected StringBuilder antlrToolErrors; + + private String getPropertyPrefix() { + return "antlr-php"; + } + + @Override + public void testSetUp() throws Exception { + // new output dir for each test + String propName = getPropertyPrefix() + "-test-dir"; + String prop = System.getProperty(propName); + + if (prop != null && prop.length() > 0) { + tmpdir = prop; + } else { + String classSimpleName = getClass().getSimpleName(); + String threadName = Thread.currentThread().getName(); + String childPath = String.format("%s-%s-%s", classSimpleName, threadName, System.currentTimeMillis()); + tmpdir = new File(System.getProperty("java.io.tmpdir"), childPath).getAbsolutePath(); + } + + antlrToolErrors = new StringBuilder(); + } + + @Override + public void testTearDown() throws Exception { + } + + @Override + public String getTmpDir() { + return tmpdir; + } + + @Override + public String getStdout() { + return null; + } + + @Override + public String getParseErrors() { + return stderrDuringParse; + } + + @Override + public String getANTLRToolErrors() { + if (antlrToolErrors.length() == 0) { + return null; + } + + return antlrToolErrors.toString(); + } + + protected ATN createATN(Grammar g, boolean useSerializer) { + if (g.atn == null) { + semanticProcess(g); + + assertEquals(0, g.tool.getNumErrors()); + + ParserATNFactory f; + + if (g.isLexer()) { + f = new LexerATNFactory((LexerGrammar) g); + } else { + f = new ParserATNFactory(g); + } + + g.atn = f.createATN(); + assertEquals(0, g.tool.getNumErrors()); + } + + ATN atn = g.atn; + + if (useSerializer) { + char[] serialized = ATNSerializer.getSerializedAsChars(atn); + + return new ATNDeserializer().deserialize(serialized); + } + + return atn; + } + + protected void semanticProcess(Grammar g) { + if (g.ast != null && !g.ast.hasErrors) { + Tool antlr = new Tool(); + SemanticPipeline sem = new SemanticPipeline(g); + sem.process(); + + if (g.getImportedGrammars() != null) { + for (Grammar imp: g.getImportedGrammars()) { + antlr.processNonCombinedGrammar(imp, false); + } + } + } + } + + protected String execLexer( + String grammarFileName, + String grammarStr, + String lexerName, + String input + ) { + return execLexer(grammarFileName, grammarStr, lexerName, input, false); + } + + @Override + public String execLexer( + String grammarFileName, + String grammarStr, + String lexerName, + String input, + boolean showDFA + ) { + boolean success = rawGenerateAndBuildRecognizer( + grammarFileName, + grammarStr, + null, + lexerName, + "-no-listener" + ); + assertTrue(success); + writeFile(tmpdir, "input", input); + writeLexerTestFile(lexerName, showDFA); + String output = execModule("Test.php"); + + return output; + } + + public String execParser( + String grammarFileName, + String grammarStr, + String parserName, + String lexerName, + String listenerName, + String visitorName, + String startRuleName, + String input, + boolean showDiagnosticErrors + ) { + return execParser_( + grammarFileName, + grammarStr, + parserName, + lexerName, + listenerName, + visitorName, + startRuleName, + input, + showDiagnosticErrors, + false + ); + } + + public String execParser_( + String grammarFileName, + String grammarStr, + String parserName, + String lexerName, + String listenerName, + String visitorName, + String startRuleName, + String input, + boolean debug, + boolean trace + ) { + boolean success = rawGenerateAndBuildRecognizer( + grammarFileName, + grammarStr, + parserName, + lexerName, + "-visitor" + ); + + assertTrue(success); + + writeFile(tmpdir, "input", input); + + rawBuildRecognizerTestFile( + parserName, + lexerName, + listenerName, + visitorName, + startRuleName, + debug, + trace + ); + + return execRecognizer(); + } + + /** + * Return true if all is well + */ + protected boolean rawGenerateAndBuildRecognizer( + String grammarFileName, + String grammarStr, + String parserName, + String lexerName, + String... extraOptions + ) { + return rawGenerateAndBuildRecognizer( + grammarFileName, + grammarStr, + parserName, + lexerName, + false, + extraOptions + ); + } + + /** + * Return true if all is well + */ + protected boolean rawGenerateAndBuildRecognizer( + String grammarFileName, + String grammarStr, + String parserName, + String lexerName, + boolean defaultListener, + String... extraOptions + ) { + ErrorQueue equeue = antlrOnString(getTmpDir(), "PHP", grammarFileName, grammarStr, defaultListener, extraOptions); + + if (!equeue.errors.isEmpty()) { + return false; + } + + List files = new ArrayList(); + + if (lexerName != null) { + files.add(lexerName + ".php"); + } + + if (parserName != null) { + files.add(parserName + ".php"); + Set optionsSet = new HashSet(Arrays.asList(extraOptions)); + + if (!optionsSet.contains("-no-listener")) { + files.add(grammarFileName.substring(0, grammarFileName.lastIndexOf('.')) + "Listener.php"); + } + + if (optionsSet.contains("-visitor")) { + files.add(grammarFileName.substring(0, grammarFileName.lastIndexOf('.')) + "Visitor.php"); + } + } + + return true; + } + + protected void rawBuildRecognizerTestFile( + String parserName, + String lexerName, + String listenerName, + String visitorName, + String parserStartRuleName, + boolean debug, + boolean trace + ) { + this.stderrDuringParse = null; + if (parserName == null) { + writeLexerTestFile(lexerName, false); + } else { + writeParserTestFile( + parserName, + lexerName, + listenerName, + visitorName, + parserStartRuleName, + debug, + trace + ); + } + } + + public String execRecognizer() { + return execModule("Test.php"); + } + + public String execModule(String fileName) { + String phpPath = locatePhp(); + String runtimePath = locateRuntime(); + + File tmpdirFile = new File(tmpdir); + String modulePath = new File(tmpdirFile, fileName).getAbsolutePath(); + String inputPath = new File(tmpdirFile, "input").getAbsolutePath(); + Path outputPath = tmpdirFile.toPath().resolve("output").toAbsolutePath(); + + try { + ProcessBuilder builder = new ProcessBuilder(phpPath, modulePath, inputPath, outputPath.toString()); + builder.environment().put("RUNTIME", runtimePath); + builder.directory(tmpdirFile); + Process process = builder.start(); + StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); + StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); + stdoutVacuum.start(); + stderrVacuum.start(); + process.waitFor(); + stdoutVacuum.join(); + stderrVacuum.join(); + String output = stdoutVacuum.toString(); + + if (output.length() == 0) { + output = null; + } + + if (stderrVacuum.toString().length() > 0) { + this.stderrDuringParse = stderrVacuum.toString(); + } + + return output; + } catch (Exception e) { + System.err.println("can't exec recognizer"); + e.printStackTrace(System.err); + } + return null; + } + + private String locateTool(String tool) { + final String phpPath = System.getProperty("PHP_PATH"); + + if (phpPath != null && new File(phpPath).exists()) { + return phpPath; + } + + String[] roots = {"/usr/local/bin/", "/opt/local/bin", "/usr/bin/"}; + + for (String root: roots) { + if (new File(root + tool).exists()) { + return root + tool; + } + } + + throw new RuntimeException("Could not locate " + tool); + } + + protected String locatePhp() { + String propName = getPropertyPrefix() + "-php"; + String prop = System.getProperty(propName); + + if (prop == null || prop.length() == 0) { + prop = locateTool("php"); + } + + File file = new File(prop); + + if (!file.exists()) { + throw new RuntimeException("Missing system property:" + propName); + } + + return file.getAbsolutePath(); + } + + protected String locateRuntime() { + String propName = "antlr-php-runtime"; + String prop = System.getProperty(propName); + + if (prop == null || prop.length() == 0) { + prop = "../runtime/PHP"; + } + + File file = new File(prop); + + if (!file.exists()) { + throw new RuntimeException("Missing system property:" + propName); + } + + try { + return file.getCanonicalPath(); + } catch (IOException e) { + return file.getAbsolutePath(); + } + } + + protected void mkdir(String dir) { + File f = new File(dir); + f.mkdirs(); + } + + protected void writeLexerTestFile(String lexerName, boolean showDFA) { + ST outputFileST = new ST( + "\\($input);\n" + + "$lexer->addErrorListener(new ConsoleErrorListener());" + + "$tokens = new CommonTokenStream($lexer);\n" + + "$tokens->fill();\n" + + "\n" + + "foreach ($tokens->getAllTokens() as $token) {\n" + + " echo $token . \\PHP_EOL;\n" + + "}" + + (showDFA + ? "echo $lexer->getInterpreter()->getDFA(Lexer::DEFAULT_MODE)->toLexerString();\n" + : "") + ); + + outputFileST.add("lexerName", lexerName); + + writeFile(tmpdir, "Test.php", outputFileST.render()); + } + + protected void writeParserTestFile( + String parserName, String lexerName, + String listenerName, String visitorName, + String parserStartRuleName, boolean debug, boolean trace + ) { + if (!parserStartRuleName.endsWith(")")) { + parserStartRuleName += "()"; + } + ST outputFileST = new ST( + "\\getChildCount(); $i \\< $count; $i++) {\n" + + " $parent = $ctx->getChild($i)->getParent();\n" + + "\n" + + " if (!($parent instanceof RuleNode) || $parent->getRuleContext() !== $ctx) {\n" + + " throw new RuntimeException('Invalid parse tree shape detected.');\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + + "\n" + + "$input = InputStream::fromPath($argv[1]);\n" + + "$lexer = new ($input);\n" + + "$lexer->addErrorListener(new ConsoleErrorListener());" + + "$tokens = new CommonTokenStream($lexer);\n" + + "" + + "$parser->addErrorListener(new ConsoleErrorListener());" + + "$parser->setBuildParseTree(true);\n" + + "$tree = $parser->;\n\n" + + "ParseTreeWalker::default()->walk(new TreeShapeListener(), $tree);\n" + ); + + String stSource = "$parser = new ($tokens);\n"; + + if (debug) { + stSource += "$parser->addErrorListener(new DiagnosticErrorListener());\n"; + } + + if (trace) { + stSource += "$parser->setTrace(true);\n"; + } + + ST createParserST = new ST(stSource); + outputFileST.add("createParser", createParserST); + outputFileST.add("parserName", parserName); + outputFileST.add("lexerName", lexerName); + outputFileST.add("listenerName", listenerName); + outputFileST.add("visitorName", visitorName); + outputFileST.add("parserStartRuleName", parserStartRuleName); + + writeFile(tmpdir, "Test.php", outputFileST.render()); + } + + protected void eraseFiles(File dir) { + String[] files = dir.list(); + for (int i = 0; files != null && i < files.length; i++) { + new File(dir, files[i]).delete(); + } + } + + @Override + public void eraseTempDir() { + boolean doErase = true; + String propName = getPropertyPrefix() + "-erase-test-dir"; + String prop = System.getProperty(propName); + if (prop != null && prop.length() > 0) { + doErase = Boolean.getBoolean(prop); + } + if (doErase) { + File tmpdirF = new File(tmpdir); + if (tmpdirF.exists()) { + eraseFiles(tmpdirF); + tmpdirF.delete(); + } + } + } + + /** + * Sort a list + */ + public > List sort(List data) { + List dup = new ArrayList(); + dup.addAll(data); + Collections.sort(dup); + return dup; + } + + /** + * Return map sorted by key + */ + public , V> LinkedHashMap sort(Map data) { + LinkedHashMap dup = new LinkedHashMap(); + List keys = new ArrayList(); + keys.addAll(data.keySet()); + Collections.sort(keys); + for (K k: keys) { + dup.put(k, data.get(k)); + } + return dup; + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestCompositeLexers.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestCompositeLexers.java new file mode 100644 index 000000000..d1d353875 --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestCompositeLexers.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.CompositeLexersDescriptors; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestCompositeLexers extends BaseRuntimeTest { + public TestCompositeLexers(RuntimeTestDescriptor descriptor) { + super(descriptor,new BasePHPTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(CompositeLexersDescriptors.class, "PHP"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestCompositeParsers.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestCompositeParsers.java new file mode 100644 index 000000000..dd5b0015a --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestCompositeParsers.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.CompositeParsersDescriptors; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestCompositeParsers extends BaseRuntimeTest { + public TestCompositeParsers(RuntimeTestDescriptor descriptor) { + super(descriptor,new BasePHPTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(CompositeParsersDescriptors.class, "PHP"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestFullContextParsing.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestFullContextParsing.java new file mode 100644 index 000000000..60efc0718 --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestFullContextParsing.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.FullContextParsingDescriptors; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestFullContextParsing extends BaseRuntimeTest { + public TestFullContextParsing(RuntimeTestDescriptor descriptor) { + super(descriptor,new BasePHPTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(FullContextParsingDescriptors.class, "PHP"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestLeftRecursion.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestLeftRecursion.java new file mode 100644 index 000000000..cb200ef38 --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestLeftRecursion.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.LeftRecursionDescriptors; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestLeftRecursion extends BaseRuntimeTest { + public TestLeftRecursion(RuntimeTestDescriptor descriptor) { + super(descriptor,new BasePHPTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(LeftRecursionDescriptors.class, "PHP"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestLexerErrors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestLexerErrors.java new file mode 100644 index 000000000..cd7a5c596 --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestLexerErrors.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.LexerErrorsDescriptors; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestLexerErrors extends BaseRuntimeTest { + public TestLexerErrors(RuntimeTestDescriptor descriptor) { + super(descriptor,new BasePHPTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(LexerErrorsDescriptors.class, "PHP"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestLexerExec.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestLexerExec.java new file mode 100644 index 000000000..03595f564 --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestLexerExec.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.LexerExecDescriptors; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestLexerExec extends BaseRuntimeTest { + public TestLexerExec(RuntimeTestDescriptor descriptor) { + super(descriptor,new BasePHPTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(LexerExecDescriptors.class, "PHP"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestListeners.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestListeners.java new file mode 100644 index 000000000..52260158d --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestListeners.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.ListenersDescriptors; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestListeners extends BaseRuntimeTest { + public TestListeners(RuntimeTestDescriptor descriptor) { + super(descriptor,new BasePHPTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(ListenersDescriptors.class, "PHP"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestParseTrees.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestParseTrees.java new file mode 100644 index 000000000..656e14d71 --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestParseTrees.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.ParseTreesDescriptors; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestParseTrees extends BaseRuntimeTest { + public TestParseTrees(RuntimeTestDescriptor descriptor) { + super(descriptor,new BasePHPTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(ParseTreesDescriptors.class, "PHP"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestParserErrors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestParserErrors.java new file mode 100644 index 000000000..ac3ab88ef --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestParserErrors.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.ParserErrorsDescriptors; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestParserErrors extends BaseRuntimeTest { + public TestParserErrors(RuntimeTestDescriptor descriptor) { + super(descriptor,new BasePHPTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(ParserErrorsDescriptors.class, "PHP"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestParserExec.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestParserExec.java new file mode 100644 index 000000000..01e3f321c --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestParserExec.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.ParserExecDescriptors; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestParserExec extends BaseRuntimeTest { + public TestParserExec(RuntimeTestDescriptor descriptor) { + super(descriptor,new BasePHPTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(ParserExecDescriptors.class, "PHP"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestPerformance.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestPerformance.java new file mode 100644 index 000000000..7459d77d8 --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestPerformance.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.PerformanceDescriptors; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestPerformance extends BaseRuntimeTest { + public TestPerformance(RuntimeTestDescriptor descriptor) { + super(descriptor,new BasePHPTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(PerformanceDescriptors.class, "PHP"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestSemPredEvalLexer.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestSemPredEvalLexer.java new file mode 100644 index 000000000..ec7f14efc --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestSemPredEvalLexer.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.SemPredEvalLexerDescriptors; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestSemPredEvalLexer extends BaseRuntimeTest { + public TestSemPredEvalLexer(RuntimeTestDescriptor descriptor) { + super(descriptor,new BasePHPTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(SemPredEvalLexerDescriptors.class, "PHP"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestSemPredEvalParser.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestSemPredEvalParser.java new file mode 100644 index 000000000..1441444c5 --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestSemPredEvalParser.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.SemPredEvalParserDescriptors; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestSemPredEvalParser extends BaseRuntimeTest { + public TestSemPredEvalParser(RuntimeTestDescriptor descriptor) { + super(descriptor,new BasePHPTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(SemPredEvalParserDescriptors.class, "PHP"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestSets.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestSets.java new file mode 100644 index 000000000..996045361 --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/TestSets.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.php; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.SetsDescriptors; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestSets extends BaseRuntimeTest { + public TestSets(RuntimeTestDescriptor descriptor) { + super(descriptor,new BasePHPTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(SetsDescriptors.class, "PHP"); + } +} diff --git a/runtime/PHP b/runtime/PHP new file mode 160000 index 000000000..9e1b759d0 --- /dev/null +++ b/runtime/PHP @@ -0,0 +1 @@ +Subproject commit 9e1b759d02220eedf477771169bdfb6a186a2984 diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/PHP/PHP.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/PHP/PHP.stg new file mode 100644 index 000000000..5376d3566 --- /dev/null +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/PHP/PHP.stg @@ -0,0 +1,1231 @@ +/* + * [The "BSD license"] + * Copyright (c) 2012-2016 Terence Parr + * Copyright (c) 2012-2016 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +phpTypeInitMap ::= [ + "int":"0", + "long":"0", + "float":"0.0", + "double":"0.0", + "boolean":"false", + default:"null" +] + +// args must be , + +ParserFile(file, parser, namedActions, contextSuperClass) ::= << + + +>> + +ListenerFile(file, header, namedActions) ::= << + + +namespace ; + +

+use Antlr\\Antlr4\\Runtime\\Tree\\ParseTreeListener; + +/** + * This interface defines a complete listener for a parse tree produced by + * {@see }. + */ +interface Listener extends ParseTreeListener { + + * Enter a parse tree produced by the `` + * labeled alternative in {@see ::()\}. + + * Enter a parse tree produced by {@see ::()\}. + + * @param $context The parse tree. + */ +public function enter(Context\\Context $context) : void; +/** + + * Exit a parse tree produced by the `` labeled alternative + * in {@see ::()\}. + + * Exit a parse tree produced by {@see ::()\}. + + * @param $context The parse tree. + */ +public function exit(Context\\Context $context) : void;}; separator="\n"> +} +>> + +BaseListenerFile(file, header, namedActions) ::= << + + +namespace ; + +
+ +use Antlr\\Antlr4\\Runtime\\ParserRuleContext; +use Antlr\\Antlr4\\Runtime\\Tree\\ErrorNode; +use Antlr\\Antlr4\\Runtime\\Tree\\TerminalNode; + +/** + * This class provides an empty implementation of {@see Listener}, + * which can be extended to create a listener which only needs to handle a subset + * of the available methods. + */ +class BaseListener implements Listener +{ + (Context\\Context $context) : void {\} + +/** + * {@inheritdoc\} + * + * The default implementation does nothing. + */ +public function exit(Context\\Context $context) : void {\}}; separator="\n"> + + /** + * {@inheritdoc\} + * + * The default implementation does nothing. + */ + public function enterEveryRule(ParserRuleContext $context) : void {} + + /** + * {@inheritdoc\} + * + * The default implementation does nothing. + */ + public function exitEveryRule(ParserRuleContext $context) : void {} + + /** + * {@inheritdoc\} + * + * The default implementation does nothing. + */ + public function visitTerminal(TerminalNode $node) : void {} + + /** + * {@inheritdoc\} + * + * The default implementation does nothing. + */ + public function visitErrorNode(ErrorNode $node) : void {} +} +>> + +VisitorFile(file, header, namedActions) ::= << + + +namespace ; + +
+use Antlr\\Antlr4\\Runtime\\Tree\\ParseTreeVisitor; + +/** + * This interface defines a complete generic visitor for a parse tree produced + by {@see }. + */ +interface Visitor extends ParseTreeVisitor +{ + + * Visit a parse tree produced by the `` labeled alternative + * in {@see ::()\}. + + * Visit a parse tree produced by {@see ::()\}. + + * @param Context $context The parse tree. + * + * @return mixed The visitor result. + */ +public function visit(Context\\Context $context);}; separator="\n\n"> +} +>> + +BaseVisitorFile(file, header, namedActions) ::= << + + +namespace ; + +
+use Antlr\\Antlr4\\Runtime\\Tree\\AbstractParseTreeVisitor; + +/** + * This class provides an empty implementation of {@see Visitor}, + * which can be extended to create a visitor which only needs to handle a subset + * of the available methods. + */ +class BaseVisitor extends AbstractParseTreeVisitor implements Visitor +{ + (Context\\Context $context) +{ + return $this->visitChildren($context); +\}}; separator="\n\n"> +} +>> + +fileHeader(grammarFileName, ANTLRVersion) ::= << +\ by ANTLR +>> +Parser(parser, funcs, atn, sempredFuncs, superClass) ::= << + +>> + +Parser_(parser, funcs, atn, sempredFuncs, ctor, superClass) ::= << +namespace { + + use Antlr\\Antlr4\\Runtime\\Atn\\ATN; + use Antlr\\Antlr4\\Runtime\\Atn\\ATNDeserializer; + use Antlr\\Antlr4\\Runtime\\Atn\\ParserATNSimulator; + use Antlr\\Antlr4\\Runtime\\Dfa\\DFA; + use Antlr\\Antlr4\\Runtime\\Error\\Exceptions\\FailedPredicateException; + use Antlr\\Antlr4\\Runtime\\Error\\Exceptions\\NoViableAltException; + use Antlr\\Antlr4\\Runtime\\PredictionContexts\\PredictionContextCache; + use Antlr\\Antlr4\\Runtime\\Error\\Exceptions\\RecognitionException; + use Antlr\\Antlr4\\Runtime\\RuleContext; + use Antlr\\Antlr4\\Runtime\\Token; + use Antlr\\Antlr4\\Runtime\\TokenStream; + use Antlr\\Antlr4\\Runtime\\Vocabulary; + use Antlr\\Antlr4\\Runtime\\VocabularyImpl; + use Antlr\\Antlr4\\Runtime\\RuntimeMetaData; + use Antlr\\Antlr4\\Runtime\\Parser; + + + final class extends + { + + public const = }; separator=", ", wrap, anchor>; + + + public const = }; separator=", ", wrap, anchor>; + + /** + * @var array\ + */ + public const RULE_NAMES = [ + '}; separator=", ", wrap, anchor> + ]; + + + + + protected static $atn; + protected static $decisionToDFA; + protected static $sharedContextCache; + + + + + + + + + private static function initialize() : void + { + if (self::$atn !== null) { + return; + } + + RuntimeMetaData::checkVersion('', RuntimeMetaData::VERSION); + + $atn = (new ATNDeserializer())->deserialize(self::SERIALIZED_ATN); + + $decisionToDFA = []; + for ($i = 0, $count = $atn->getNumberOfDecisions(); $i \< $count; $i++) { + $decisionToDFA[] = new DFA($atn->getDecisionState($i), $i); + } + + self::$atn = $atn; + self::$decisionToDFA = $decisionToDFA; + self::$sharedContextCache = new PredictionContextCache(); + } + + public function getGrammarFileName() : string + { + return ""; + } + + public function getRuleNames() : array + { + return self::RULE_NAMES; + } + + public function getSerializedATN() : string + { + return self::SERIALIZED_ATN; + } + + public function getATN() : ATN + { + return self::$atn; + } + + public function getVocabulary() : Vocabulary + { + static $vocabulary; + + return $vocabulary = $vocabulary ?? new VocabularyImpl(self::LITERAL_NAMES, self::SYMBOLIC_NAMES); + } + + + + + + + public function sempred(?RuleContext $localContext, int $ruleIndex, int $predicateIndex) : bool + { + switch ($ruleIndex) { + : + return $this->sempred($localContext, $predicateIndex);}; separator="\n\n"> + + default: + return true; + } + } + + + + } +} + +namespace \\Context { + use Antlr\\Antlr4\\Runtime\\ParserRuleContext; + use Antlr\\Antlr4\\Runtime\\Token; + use Antlr\\Antlr4\\Runtime\\Tree\\ParseTreeVisitor; + use Antlr\\Antlr4\\Runtime\\Tree\\TerminalNode; + use Antlr\\Antlr4\\Runtime\\Tree\\ParseTreeListener; + use \\; + use \\Visitor; + use \\Listener; + + + }; separator="\n\n"> + }; separator="\n\n">}> +} +>> + +vocabulary(literalNames, symbolicNames) ::= << +/** + * @var array\ + */ +private const LITERAL_NAMES = [ + }; null="null", separator=", ", wrap, anchor> +]; + +/** + * @var array\ + */ +private const SYMBOLIC_NAMES = [ + }; null="null", separator=", ", wrap, anchor> +]; +>> + +dumpActions(recog, argFuncs, actionFuncs, sempredFuncs) ::= << + + +public function action(?RuleContext $localContext, int $ruleIndex, int $actionIndex) : void +{ + switch ($ruleIndex) { + : + $this->action($localContext, $actionIndex); + + break;}; separator="\n\n"> + } +} + + + + + +public function sempred(?RuleContext $localContext, int $ruleIndex, int $predicateIndex) : bool +{ + switch ($ruleIndex) { + : + return $this->sempred($localContext, $predicateIndex);}; separator="\n\n"> + } + + return true; +} + + +>> + +parser_ctor(p) ::= << +public function __construct(TokenStream $input) +{ + parent::__construct($input); + + self::initialize(); + + $this->interp = new ParserATNSimulator($this, self::$atn, self::$decisionToDFA, self::$sharedContextCache); +} +>> + +/** + * This generates a private method since the actionIndex is generated, making + * an overriding implementation impossible to maintain. + */ +RuleActionFunction(r, actions) ::= << +private function action(? $localContext, int $actionIndex) : void +{ + switch ($actionIndex) { + : + + + break;}; separator="\n\n"> + } +} +>> + +/** + * This generates a private method since the predicateIndex is generated, making + * an overriding implementation impossible to maintain. + */ +RuleSempredFunction(r, actions) ::= << +private function sempred(?Context\\ $localContext, int $predicateIndex) : bool +{ + switch ($predicateIndex) { + : + return ;}; separator="\n\n"> + } + + return true; +} +>> + +RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,finallyAction,exceptions,postamble) ::= << +/** + * @throws RecognitionException + */ + }>public function () : Context\\ +{ + $localContext = new Context\\($this->ctx, $this->getState()}>); + + $this->enterRule($localContext, , ::RULE_); + + + + try { + + + + } catch (RecognitionException $exception) { + $localContext->exception = $exception; + $this->errorHandler->reportError($this, $exception); + $this->errorHandler->recover($this, $exception); + } finally { + + $this->exitRule(); + } + + return $localContext; +} +>> + +LeftRecursiveRuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,finallyAction,postamble) ::= << +/** + * @throws RecognitionException + */ + }>public function () : Context\\ +{ + return $this->recursive(0}>); +} + +/** + * @throws RecognitionException + */ +private function recursive(int $precedence}>) : Context\\ +{ + $parentContext = $this->ctx; + $parentState = $this->getState(); + $localContext = new Context\\($this->ctx, $parentState}>); + $previousContext = $localContext; + $startState = ; + $this->enterRecursionRule($localContext, , ::RULE_, $precedence); + + + + try { + + + + } catch (RecognitionException $exception) { + $localContext->exception = $exception; + $this->errorHandler->reportError($this, $exception); + $this->errorHandler->recover($this, $exception); + } finally { + + $this->unrollRecursionContexts($parentContext); + } + + return $localContext; +} +>> + +CodeBlockForOuterMostAlt(currentOuterMostAltCodeBlock, locals, preamble, ops) ::= << +$localContext = new Context\\Context($localContext); +$this->enterOuterAlt($localContext, ); + +>> + +CodeBlockForAlt(currentAltCodeBlock, locals, preamble, ops) ::= << + + + +>> + +LL1AltBlock(choice, preamble, alts, error) ::= << +$this->setState(); +$this->errorHandler->sync($this); + = $this->input->LT(1); + + +switch ($this->input->LA(1)) { + + + break;}; separator="\n\n"> + +default: + +} +>> + +LL1OptionalBlock(choice, alts, error) ::= << +$this->setState(); +$this->errorHandler->sync($this); + +switch ($this->input->LA(1)) { + + + break;}; separator="\n\n"> + +default: + break; +} +>> + +LL1OptionalBlockSingleAlt(choice, expr, alts, preamble, error, followExpr) ::= << +$this->setState(); +$this->errorHandler->sync($this); + + +if () { + +} +>> + +LL1StarBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= << +$this->setState(); +$this->errorHandler->sync($this); + + +while () { + + $this->setState(); + $this->errorHandler->sync($this); + +} +>> + +LL1PlusBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= << +$this->setState(); +$this->errorHandler->sync($this); + + +do { + + $this->setState(); + $this->errorHandler->sync($this); + +} while (); +>> + +// LL(*) stuff + +AltBlock(choice, preamble, alts, error) ::= << +$this->setState(); +$this->errorHandler->sync($this); + = $this->input->LT(1); + + +switch ($this->getInterpreter()->adaptivePredict($this->input, , $this->ctx)) { + : + +break;}; separator="\n\n"> +} +>> + +OptionalBlock(choice, alts, error) ::= << +$this->setState(); +$this->errorHandler->sync($this); + +switch ($this->getInterpreter()->adaptivePredict($this->input, , $this->ctx)) { ++1: + + break;}; separator="\n\n"> +} +>> + +StarBlock(choice, alts, sync, iteration) ::= << +$this->setState(); +$this->errorHandler->sync($this); + +$alt = $this->getInterpreter()->adaptivePredict($this->input, , $this->ctx); + +while ($alt !== && $alt !== ATN::INVALID_ALT_NUMBER) { + if ($alt === 1+1) { + + + } + + $this->setState(); + $this->errorHandler->sync($this); + + $alt = $this->getInterpreter()->adaptivePredict($this->input, , $this->ctx); +} +>> + +PlusBlock(choice, alts, error) ::= << +$this->setState(); +$this->errorHandler->sync($this); + +$alt = 1+1; + +do { + switch ($alt) { + +1: + + + break;}; separator="\n\n"> + default: + + } + + $this->setState(); + $this->errorHandler->sync($this); + + $alt = $this->getInterpreter()->adaptivePredict($this->input, , $this->ctx); +} while ($alt !== && $alt !== ATN::INVALID_ALT_NUMBER); +>> + +Sync(s) ::= "sync();" + +ThrowNoViableAlt(t) ::= "throw new NoViableAltException($this);" + +TestSetInline(s) ::= << +}; separator=" || "> +>> + +// Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test +testShiftInRange(shiftAmount) ::= << +(() & ~0x3f) === 0 +>> + +// produces smaller bytecode only when bits.ttypes contains more than two items +bitsetBitfieldComparison(s, bits) ::= <% +(})> && ((1 \<\< ) & ()}; separator=" | ">)) !== 0) +%> + +isZero ::= [ +"0":true, +default:false +] + +offsetShiftVar(shiftAmount, offset) ::= <% +($ - )$ +%> +offsetShiftConst(shiftAmount, offset) ::= <% +(self:: - )self:: +%> + +// produces more efficient bytecode when bits.ttypes contains at most two items +bitsetInlineComparison(s, bits) ::= <% + === self::}; separator=" || "> +%> + +cases(ttypes) ::= << +:::}; separator="\n"> +>> + +InvokeRule(r, argExprsChunks) ::= << +$this->setState(); + = }>$this->recursive(,); +>> + +MatchToken(m) ::= << +$this->setState(); + = }>$this->match(::); +>> + +MatchSet(m, expr, capture) ::= "" + +MatchNotSet(m, expr, capture) ::= "" + +CommonSetStuff(m, expr, capture, invert) ::= << +$this->setState(); + + = }>$this->input->LT(1); + + +if ($ \<= 0 || !()) { + = }>$this->errorHandler->recoverInline($this); +} else { + if ($this->input->LA(1) === Token::EOF) { + $this->matchedEOF = true; + } + + $this->errorHandler->reportMatch($this); + $this->consume(); +} +>> + +Wildcard(w) ::= << +$this->setState(); + = }>$this->matchWildcard(); +>> + +// ACTION STUFF + +Action(a, foo, chunks) ::= "" + +ArgAction(a, chunks) ::= "" + +SemPred(p, chunks, failChunks) ::= << +$this->setState(); + +if (!()) { + throw new FailedPredicateException($this, , , ); +} +>> + +ExceptionClause(e, catchArg, catchAction) ::= << +catch () { + +} +>> + +// lexer actions are not associated with model objects + +LexerSkipCommand() ::= "$this->skip();" +LexerMoreCommand() ::= "$this->more();" +LexerPopModeCommand() ::= "$this->popMode();" + +LexerTypeCommand(arg, grammar) ::= "$this->type = ;" +LexerChannelCommand(arg, grammar) ::= "$this->channel = ;" +LexerModeCommand(arg, grammar) ::= "$this->mode = ;" +LexerPushModeCommand(arg, grammar) ::= "$this->pushMode();" + +ActionText(t) ::= "" +ActionTemplate(t) ::= "" +ArgRef(a) ::= "$localContext->" +LocalRef(a) ::= "$localContext->" +RetValueRef(a) ::= "$localContext->" +QRetValueRef(a) ::= "->->" +/** How to translate $tokenLabel */ +TokenRef(t) ::= "->" +LabelRef(t) ::= "->" +ListLabelRef(t) ::= "->" +SetAttr(s,rhsChunks) ::= "-> = ;" + +TokenLabelType() ::= "" +InputSymbolType() ::= "" + +TokenPropertyRef_text(t) ::= "(-> !== null ? ->->getText() : null)" +TokenPropertyRef_type(t) ::= "(-> !== null ? ->->getType() : 0)" +TokenPropertyRef_line(t) ::= "(-> !== null ? ->->getLine() : 0)" +TokenPropertyRef_pos(t) ::= "(-> !== null ? ->->getCharPositionInLine() : 0)" +TokenPropertyRef_channel(t) ::= "(-> !== null ? ->->getChannel() : 0)" +TokenPropertyRef_index(t) ::= "(-> !== null ? ->->getTokenIndex() : 0)" +TokenPropertyRef_int(t) ::= "(-> !== null ? (int) ->->getText() : 0)" + +RulePropertyRef_start(r) ::= "(-> !== null ? (->->start) : null)" +RulePropertyRef_stop(r) ::= "(-> !== null ? (->->stop) : null)" +RulePropertyRef_text(r) ::= "(-> !== null ? $this->input->getTextByTokens(->->start, ->->stop) : null)" +RulePropertyRef_ctx(r) ::= "->" +RulePropertyRef_parser(r)::= "\$this" + +ThisRulePropertyRef_start(r) ::= "$localContext->start" +ThisRulePropertyRef_stop(r) ::= "$localContext->stop" +ThisRulePropertyRef_text(r) ::= "$this->input->getTextByTokens($localContext->start, $this->input->LT(-1))" +ThisRulePropertyRef_ctx(r) ::= "$localContext" +ThisRulePropertyRef_parser(r)::= "$this" + +NonLocalAttrRef(s) ::= "\$this->getInvokingContext()->" +SetNonLocalAttr(s, rhsChunks) ::= "\$this->getInvokingContext()-> = ;" + +AddToLabelList(a) ::= "->[] = ;" + +TokenDecl(t) ::= " $" +TokenTypeDecl(t) ::= "" +TokenListDecl(t) ::= "array $ = []" +RuleContextDecl(r) ::= " $" +RuleContextListDecl(rdecl) ::= "array $ = []" +AttributeDecl(d) ::= " $ = " + +PropertiesDecl(struct) ::= << + + |null $ + */ +public $;}; separator="\n\n"> + + + + + + |null $ + */ +public $;}; separator="\n\n"> + + + + + + |null $ + */ +public $;}; separator="\n\n"> + + + + + + \>|null $ + */ +public $;}; separator="\n\n"> + + + + + + |null $ + */ +public $ = ;}; separator="\n\n"> + + +>> + +ContextTokenGetterDecl(t) ::= << +public function () : ?TerminalNode +{ + return $this->getToken(::, 0); +} +>> + +ContextTokenListGetterDecl(t) ::= << +>> + +ContextTokenListIndexedGetterDecl(t) ::= << +/** + * @return array\|TerminalNode|null + */ +public function (?int $index = null) +{ + if ($index === null) { + return $this->getTokens(::); + } + + return $this->getToken(::, $index); +} +>> + +ContextRuleGetterDecl(r) ::= << +public function () : ? +{ + return $this->getTypedRuleContext(::class, 0); +} +>> + +ContextRuleListGetterDecl(r) ::= << +>> + +ContextRuleListIndexedGetterDecl(r) ::= << +/** + * @return array\<\>||null + */ +public function (?int $index = null) +{ + if ($index === null) { + return $this->getTypedRuleContexts(::class); + } + + return $this->getTypedRuleContext(::class, $index); +} +>> + +LexerRuleContext() ::= "RuleContext" + +/** + * The rule context name is the rule followed by a suffix; e.g., r becomes rContext. + */ +RuleContextNameSuffix() ::= "Context" + +ImplicitTokenLabel(tokenName) ::= "" +ImplicitRuleLabel(ruleName) ::= "" +ImplicitSetLabel(id) ::= "_tset" +ListLabelName(label) ::= "
use Antlr\\Antlr4\\Runtime\\Tree\\ParseTreeVisitor; /** - * This interface defines a complete generic visitor for a parse tree produced - by {@see }. + * This interface defines a complete generic visitor for a parse tree produced by {@see }. */ interface Visitor extends ParseTreeVisitor { @@ -166,7 +166,8 @@ interface Visitor extends ParseTreeVisitor * Visit a parse tree produced by {@see ::()\}. - * @param Context $context The parse tree. + * + * @param Context\\Context $context The parse tree. * * @return mixed The visitor result. */ @@ -206,7 +207,10 @@ public function visit(Context\\Context fileHeader(grammarFileName, ANTLRVersion) ::= << \ by ANTLR +/* + * Generated from by ANTLR + */ + >> Parser(parser, funcs, atn, sempredFuncs, superClass) ::= << @@ -214,7 +218,7 @@ Parser(parser, funcs, atn, sempredFuncs, superClass) ::= << Parser_(parser, funcs, atn, sempredFuncs, ctor, superClass) ::= << namespace { - + use Antlr\\Antlr4\\Runtime\\Atn\\ATN; use Antlr\\Antlr4\\Runtime\\Atn\\ATNDeserializer; use Antlr\\Antlr4\\Runtime\\Atn\\ParserATNSimulator; @@ -230,7 +234,7 @@ namespace { use Antlr\\Antlr4\\Runtime\\VocabularyImpl; use Antlr\\Antlr4\\Runtime\\RuntimeMetaData; use Antlr\\Antlr4\\Runtime\\Parser; - + final class extends { @@ -1061,7 +1065,7 @@ LexerFile(lexerFile, lexer, namedActions) ::= << Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass) ::= << namespace { - + use Antlr\\Antlr4\\Runtime\\Atn\\ATNDeserializer; use Antlr\\Antlr4\\Runtime\\Atn\\LexerATNSimulator; use Antlr\\Antlr4\\Runtime\\Lexer; @@ -1073,7 +1077,7 @@ namespace { use Antlr\\Antlr4\\Runtime\\Vocabulary; use Antlr\\Antlr4\\Runtime\\RuntimeMetaData; use Antlr\\Antlr4\\Runtime\\VocabularyImpl; - + final class extends { From 9723e4b9c92731598d32a87f64070d77d0e25b23 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Fri, 11 Oct 2019 14:44:53 +0200 Subject: [PATCH 057/365] C++ code cleanup Converted back all TO_DO items to TODO (they were renamed ANTLR4 TODO items, but I had some for C++ which I all wanted to fix). Also removed some default copy assignment operators which clang warned about. --- runtime/Cpp/runtime/src/ANTLRErrorStrategy.h | 4 ++-- runtime/Cpp/runtime/src/Lexer.cpp | 4 ++-- runtime/Cpp/runtime/src/Parser.cpp | 4 ++-- runtime/Cpp/runtime/src/Vocabulary.h | 1 - runtime/Cpp/runtime/src/atn/ATNConfig.h | 5 ++--- runtime/Cpp/runtime/src/atn/ATNConfigSet.h | 6 +++--- runtime/Cpp/runtime/src/atn/AtomTransition.h | 4 ++-- runtime/Cpp/runtime/src/atn/LexerATNSimulator.cpp | 2 +- runtime/Cpp/runtime/src/atn/ParserATNSimulator.cpp | 2 +- runtime/Cpp/runtime/src/atn/ParserATNSimulator.h | 2 +- runtime/Cpp/runtime/src/atn/PredicateTransition.h | 4 ++-- runtime/Cpp/runtime/src/atn/PredictionContext.cpp | 4 ++-- runtime/Cpp/runtime/src/atn/ProfilingATNSimulator.cpp | 2 +- runtime/Cpp/runtime/src/tree/pattern/ParseTreeMatch.h | 3 +-- runtime/Cpp/runtime/src/tree/pattern/ParseTreePattern.h | 3 +-- 15 files changed, 23 insertions(+), 27 deletions(-) diff --git a/runtime/Cpp/runtime/src/ANTLRErrorStrategy.h b/runtime/Cpp/runtime/src/ANTLRErrorStrategy.h index 67576c607..a3eecd14c 100755 --- a/runtime/Cpp/runtime/src/ANTLRErrorStrategy.h +++ b/runtime/Cpp/runtime/src/ANTLRErrorStrategy.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. +/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -24,7 +24,7 @@ namespace antlr4 { /// Implementations of this interface report syntax errors by calling /// . ///

- /// TO_DO: what to do about lexers + /// TODO: what to do about lexers /// class ANTLR4CPP_PUBLIC ANTLRErrorStrategy { public: diff --git a/runtime/Cpp/runtime/src/Lexer.cpp b/runtime/Cpp/runtime/src/Lexer.cpp index 3abd4b862..7eb64c8ef 100755 --- a/runtime/Cpp/runtime/src/Lexer.cpp +++ b/runtime/Cpp/runtime/src/Lexer.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. +/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -273,7 +273,7 @@ std::string Lexer::getErrorDisplay(const std::string &s) { } void Lexer::recover(RecognitionException * /*re*/) { - // TO_DO: Do we lose character or line position information? + // TODO: Do we lose character or line position information? _input->consume(); } diff --git a/runtime/Cpp/runtime/src/Parser.cpp b/runtime/Cpp/runtime/src/Parser.cpp index f65da1433..9ddb262d9 100755 --- a/runtime/Cpp/runtime/src/Parser.cpp +++ b/runtime/Cpp/runtime/src/Parser.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. +/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -464,7 +464,7 @@ bool Parser::precpred(RuleContext * /*localctx*/, int precedence) { } bool Parser::inContext(const std::string &/*context*/) { - // TO_DO: useful in parser? + // TODO: useful in parser? return false; } diff --git a/runtime/Cpp/runtime/src/Vocabulary.h b/runtime/Cpp/runtime/src/Vocabulary.h index df78b4364..7dbf85cd3 100755 --- a/runtime/Cpp/runtime/src/Vocabulary.h +++ b/runtime/Cpp/runtime/src/Vocabulary.h @@ -16,7 +16,6 @@ namespace dfa { public: Vocabulary(Vocabulary const&) = default; virtual ~Vocabulary(); - Vocabulary& operator=(Vocabulary const&) = default; /// Gets an empty instance. /// diff --git a/runtime/Cpp/runtime/src/atn/ATNConfig.h b/runtime/Cpp/runtime/src/atn/ATNConfig.h index 700a6e120..a78b5c0c5 100755 --- a/runtime/Cpp/runtime/src/atn/ATNConfig.h +++ b/runtime/Cpp/runtime/src/atn/ATNConfig.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. +/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -57,7 +57,7 @@ namespace atn { *

* closure() tracks the depth of how far we dip into the outer context: * depth > 0. Note that it may not be totally accurate depth since I - * don't ever decrement. TO_DO: make it a boolean then

+ * don't ever decrement. TODO: make it a boolean then

* *

* For memory efficiency, the {@link #isPrecedenceFilterSuppressed} method @@ -87,7 +87,6 @@ namespace atn { ATNConfig(ATNConfig const&) = default; virtual ~ATNConfig(); - ATNConfig& operator=(ATNConfig const&) = default; virtual size_t hashCode() const; diff --git a/runtime/Cpp/runtime/src/atn/ATNConfigSet.h b/runtime/Cpp/runtime/src/atn/ATNConfigSet.h index 843b055eb..850a07c12 100755 --- a/runtime/Cpp/runtime/src/atn/ATNConfigSet.h +++ b/runtime/Cpp/runtime/src/atn/ATNConfigSet.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. +/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -18,8 +18,8 @@ namespace atn { /// Track the elements as they are added to the set; supports get(i) std::vector> configs; - // TO_DO: these fields make me pretty uncomfortable but nice to pack up info together, saves recomputation - // TO_DO: can we track conflicts as they are added to save scanning configs later? + // TODO: these fields make me pretty uncomfortable but nice to pack up info together, saves recomputation + // TODO: can we track conflicts as they are added to save scanning configs later? size_t uniqueAlt; /** Currently this is only used when we detect SLL conflict; this does diff --git a/runtime/Cpp/runtime/src/atn/AtomTransition.h b/runtime/Cpp/runtime/src/atn/AtomTransition.h index b3fa18864..cc22e5ad9 100755 --- a/runtime/Cpp/runtime/src/atn/AtomTransition.h +++ b/runtime/Cpp/runtime/src/atn/AtomTransition.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. +/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -10,7 +10,7 @@ namespace antlr4 { namespace atn { - /// TO_DO: make all transitions sets? no, should remove set edges. + /// TODO: make all transitions sets? no, should remove set edges. class ANTLR4CPP_PUBLIC AtomTransition final : public Transition { public: /// The token type or character value; or, signifies special label. diff --git a/runtime/Cpp/runtime/src/atn/LexerATNSimulator.cpp b/runtime/Cpp/runtime/src/atn/LexerATNSimulator.cpp index c21ef0b2c..827c3d59f 100755 --- a/runtime/Cpp/runtime/src/atn/LexerATNSimulator.cpp +++ b/runtime/Cpp/runtime/src/atn/LexerATNSimulator.cpp @@ -422,7 +422,7 @@ Ref LexerATNSimulator::getEpsilonTarget(CharStream *input, const if (config->context == nullptr|| config->context->hasEmptyPath()) { // execute actions anywhere in the start rule for a token. // - // TO_DO: if the entry rule is invoked recursively, some + // TODO: if the entry rule is invoked recursively, some // actions may be executed during the recursive call. The // problem can appear when hasEmptyPath() is true but // isEmpty() is false. In this case, the config needs to be diff --git a/runtime/Cpp/runtime/src/atn/ParserATNSimulator.cpp b/runtime/Cpp/runtime/src/atn/ParserATNSimulator.cpp index 7d6cd00f6..30918ff0d 100755 --- a/runtime/Cpp/runtime/src/atn/ParserATNSimulator.cpp +++ b/runtime/Cpp/runtime/src/atn/ParserATNSimulator.cpp @@ -922,7 +922,7 @@ void ParserATNSimulator::closure_(Ref const& config, ATNConfigSet *co } } - configs->dipsIntoOuterContext = true; // TO_DO: can remove? only care when we add to set per middle of this method + configs->dipsIntoOuterContext = true; // TODO: can remove? only care when we add to set per middle of this method assert(newDepth > INT_MIN); newDepth--; diff --git a/runtime/Cpp/runtime/src/atn/ParserATNSimulator.h b/runtime/Cpp/runtime/src/atn/ParserATNSimulator.h index 7e92c906b..6520a44bd 100755 --- a/runtime/Cpp/runtime/src/atn/ParserATNSimulator.h +++ b/runtime/Cpp/runtime/src/atn/ParserATNSimulator.h @@ -760,7 +760,7 @@ namespace atn { virtual bool evalSemanticContext(Ref const& pred, ParserRuleContext *parserCallStack, size_t alt, bool fullCtx); - /* TO_DO: If we are doing predicates, there is no point in pursuing + /* TODO: If we are doing predicates, there is no point in pursuing closure operations if we reach a DFA state that uniquely predicts alternative. We will not be caching that DFA state and it is a waste to pursue the closure. Might have to advance when we do diff --git a/runtime/Cpp/runtime/src/atn/PredicateTransition.h b/runtime/Cpp/runtime/src/atn/PredicateTransition.h index fed28bdf3..4d9b4205d 100755 --- a/runtime/Cpp/runtime/src/atn/PredicateTransition.h +++ b/runtime/Cpp/runtime/src/atn/PredicateTransition.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. +/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -11,7 +11,7 @@ namespace antlr4 { namespace atn { - /// TO_DO: this is old comment: + /// TODO: this is old comment: /// A tree of semantic predicates from the grammar AST if label==SEMPRED. /// In the ATN, labels will always be exactly one predicate, but the DFA /// may have to combine a bunch of them as it collects predicates from diff --git a/runtime/Cpp/runtime/src/atn/PredictionContext.cpp b/runtime/Cpp/runtime/src/atn/PredictionContext.cpp index 597e083a6..860a18056 100755 --- a/runtime/Cpp/runtime/src/atn/PredictionContext.cpp +++ b/runtime/Cpp/runtime/src/atn/PredictionContext.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. +/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -335,7 +335,7 @@ Ref PredictionContext::mergeArrays(const Ref M = std::make_shared(mergedParents, mergedReturnStates); // if we created same array as a or b, return that instead - // TO_DO: track whether this is possible above during merge sort for speed + // TODO: track whether this is possible above during merge sort for speed if (*M == *a) { if (mergeCache != nullptr) { mergeCache->put(a, b, a); diff --git a/runtime/Cpp/runtime/src/atn/ProfilingATNSimulator.cpp b/runtime/Cpp/runtime/src/atn/ProfilingATNSimulator.cpp index 6b2762022..62fc12f0a 100755 --- a/runtime/Cpp/runtime/src/atn/ProfilingATNSimulator.cpp +++ b/runtime/Cpp/runtime/src/atn/ProfilingATNSimulator.cpp @@ -99,7 +99,7 @@ std::unique_ptr ProfilingATNSimulator::computeReachSet(ATNConfigSe _decisions[_currentDecision].LL_ATNTransitions++; // count computation even if error if (reachConfigs != nullptr) { } else { // no reach on current lookahead symbol. ERROR. - // TO_DO: does not handle delayed errors per getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule() + // TODO: does not handle delayed errors per getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule() _decisions[_currentDecision].errors.push_back(ErrorInfo(_currentDecision, closure, _input, _startIndex, _llStopIndex, true)); } } else { diff --git a/runtime/Cpp/runtime/src/tree/pattern/ParseTreeMatch.h b/runtime/Cpp/runtime/src/tree/pattern/ParseTreeMatch.h index 35cb90ae4..eefde46c8 100755 --- a/runtime/Cpp/runtime/src/tree/pattern/ParseTreeMatch.h +++ b/runtime/Cpp/runtime/src/tree/pattern/ParseTreeMatch.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. +/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -45,7 +45,6 @@ namespace pattern { const std::map> &labels, ParseTree *mismatchedNode); ParseTreeMatch(ParseTreeMatch const&) = default; virtual ~ParseTreeMatch(); - ParseTreeMatch& operator=(ParseTreeMatch const&) = default; ///

/// Get the last node associated with a specific {@code label}. diff --git a/runtime/Cpp/runtime/src/tree/pattern/ParseTreePattern.h b/runtime/Cpp/runtime/src/tree/pattern/ParseTreePattern.h index 3df2f633e..d5b86ff47 100755 --- a/runtime/Cpp/runtime/src/tree/pattern/ParseTreePattern.h +++ b/runtime/Cpp/runtime/src/tree/pattern/ParseTreePattern.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. +/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -30,7 +30,6 @@ namespace pattern { ParseTree *patternTree); ParseTreePattern(ParseTreePattern const&) = default; virtual ~ParseTreePattern(); - ParseTreePattern& operator=(ParseTreePattern const&) = default; /// /// Match a specific parse tree against this tree pattern. From 4d649fbf71b9bbf064a33678e2c3a04f90f2be8b Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Fri, 11 Oct 2019 14:47:39 +0200 Subject: [PATCH 058/365] Another attempt at solving this issue There was already a patch to fix this issue, but it had some problems and was closed by the author after a while. --- runtime/Cpp/runtime/src/tree/xpath/XPath.cpp | 36 ++++++++++++-------- runtime/Cpp/runtime/src/tree/xpath/XPath.h | 10 +++--- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/runtime/Cpp/runtime/src/tree/xpath/XPath.cpp b/runtime/Cpp/runtime/src/tree/xpath/XPath.cpp index 6dd13832d..09dc7cc34 100755 --- a/runtime/Cpp/runtime/src/tree/xpath/XPath.cpp +++ b/runtime/Cpp/runtime/src/tree/xpath/XPath.cpp @@ -28,8 +28,8 @@ XPath::XPath(Parser *parser, const std::string &path) { _elements = split(path); } -std::vector XPath::split(const std::string &path) { - ANTLRFileStream in(path); +std::vector> XPath::split(const std::string &path) { + ANTLRInputStream in(path); XPathLexer lexer(&in); lexer.removeErrorListeners(); XPathLexerErrorListener listener; @@ -44,7 +44,7 @@ std::vector XPath::split(const std::string &path) { } std::vector tokens = tokenStream.getTokens(); - std::vector elements; + std::vector> elements; size_t n = tokens.size(); size_t i = 0; bool done = false; @@ -62,9 +62,9 @@ std::vector XPath::split(const std::string &path) { i++; next = tokens[i]; } - XPathElement pathElement = getXPathElement(next, anywhere); - pathElement.setInvert(invert); - elements.push_back(pathElement); + std::unique_ptr pathElement = getXPathElement(next, anywhere); + pathElement->setInvert(invert); + elements.push_back(std::move(pathElement)); i++; break; @@ -81,25 +81,26 @@ std::vector XPath::split(const std::string &path) { break; default : - throw IllegalArgumentException("Unknow path element " + el->toString()); + throw IllegalArgumentException("Unknown path element " + el->toString()); } } return elements; } -XPathElement XPath::getXPathElement(Token *wordToken, bool anywhere) { +std::unique_ptr XPath::getXPathElement(Token *wordToken, bool anywhere) { if (wordToken->getType() == Token::EOF) { throw IllegalArgumentException("Missing path element at end of path"); } + std::string word = wordToken->getText(); size_t ttype = _parser->getTokenType(word); ssize_t ruleIndex = _parser->getRuleIndex(word); switch (wordToken->getType()) { case XPathLexer::WILDCARD : if (anywhere) - return XPathWildcardAnywhereElement(); - return XPathWildcardElement(); + return std::unique_ptr(new XPathWildcardAnywhereElement()); + return std::unique_ptr(new XPathWildcardElement()); case XPathLexer::TOKEN_REF: case XPathLexer::STRING : @@ -107,21 +108,26 @@ XPathElement XPath::getXPathElement(Token *wordToken, bool anywhere) { throw IllegalArgumentException(word + " at index " + std::to_string(wordToken->getStartIndex()) + " isn't a valid token name"); } if (anywhere) - return XPathTokenAnywhereElement(word, (int)ttype); - return XPathTokenElement(word, (int)ttype); + return std::unique_ptr(new XPathTokenAnywhereElement(word, (int)ttype)); + return std::unique_ptr(new XPathTokenElement(word, (int)ttype)); default : if (ruleIndex == -1) { throw IllegalArgumentException(word + " at index " + std::to_string(wordToken->getStartIndex()) + " isn't a valid rule name"); } if (anywhere) - return XPathRuleAnywhereElement(word, (int)ruleIndex); - return XPathRuleElement(word, (int)ruleIndex); + return std::unique_ptr(new XPathRuleAnywhereElement(word, (int)ruleIndex)); + return std::unique_ptr(new XPathRuleElement(word, (int)ruleIndex)); } } static ParserRuleContext dummyRoot; +std::vector XPath::findAll(ParseTree *tree, std::string const& xpath, Parser *parser) { + XPath p(parser, xpath); + return p.evaluate(tree); +} + std::vector XPath::evaluate(ParseTree *t) { dummyRoot.children = { t }; // don't set t's parent. @@ -135,7 +141,7 @@ std::vector XPath::evaluate(ParseTree *t) { // only try to match next element if it has children // e.g., //func/*/stat might have a token node for which // we can't go looking for stat nodes. - auto matching = _elements[i].evaluate(node); + auto matching = _elements[i]->evaluate(node); next.insert(next.end(), matching.begin(), matching.end()); } } diff --git a/runtime/Cpp/runtime/src/tree/xpath/XPath.h b/runtime/Cpp/runtime/src/tree/xpath/XPath.h index 07a35921c..0360c3861 100755 --- a/runtime/Cpp/runtime/src/tree/xpath/XPath.h +++ b/runtime/Cpp/runtime/src/tree/xpath/XPath.h @@ -61,8 +61,10 @@ namespace xpath { XPath(Parser *parser, const std::string &path); virtual ~XPath() {} - // TO_DO: check for invalid token/rule names, bad syntax - virtual std::vector split(const std::string &path); + // TODO: check for invalid token/rule names, bad syntax + virtual std::vector> split(const std::string &path); + + static std::vector findAll(ParseTree *tree, std::string const& xpath, Parser *parser); /// Return a list of all nodes starting at {@code t} as root that satisfy the /// path. The root {@code /} is relative to the node passed to @@ -71,13 +73,13 @@ namespace xpath { protected: std::string _path; - std::vector _elements; + std::vector> _elements; Parser *_parser; /// Convert word like {@code *} or {@code ID} or {@code expr} to a path /// element. {@code anywhere} is {@code true} if {@code //} precedes the /// word. - virtual XPathElement getXPathElement(Token *wordToken, bool anywhere); + virtual std::unique_ptr getXPathElement(Token *wordToken, bool anywhere); }; } // namespace xpath From c37ef7e0ea1277d1f900a8bda9784bf45a1f2207 Mon Sep 17 00:00:00 2001 From: zqlu Date: Tue, 22 Oct 2019 20:31:36 +0800 Subject: [PATCH 059/365] Fix main entry point for JavaScript runtime npm package --- runtime/JavaScript/src/antlr4/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/JavaScript/src/antlr4/package.json b/runtime/JavaScript/src/antlr4/package.json index afb4618da..d8b6fe3d4 100644 --- a/runtime/JavaScript/src/antlr4/package.json +++ b/runtime/JavaScript/src/antlr4/package.json @@ -2,7 +2,7 @@ "name": "antlr4", "version": "4.7.2", "description": "JavaScript runtime for ANTLR4", - "main": "src/antlr4/index.js", + "main": "index.js", "repository": "antlr/antlr4.git", "keywords": [ "lexer", From 1304e1b8d4dd7094fee50d05cbde29fb955af583 Mon Sep 17 00:00:00 2001 From: Tabari Alexander Date: Tue, 29 Oct 2019 11:36:47 -0400 Subject: [PATCH 060/365] Fix placement of members in JS target. The location of was moved be in the lexer constructor, like it is for the parser constructor. --- .../antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg index 38c9ad840..2b310fb70 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg @@ -815,6 +815,7 @@ var decisionsToDFA = atn.decisionToState.map( function(ds, index) { return new a function (input) { antlr4.Lexer.call(this, input); this._interp = new antlr4.atn.LexerATNSimulator(this, atn, decisionsToDFA, new antlr4.PredictionContextCache()); + return this; } @@ -850,8 +851,6 @@ Object.defineProperty(.prototype, "atn", { .prototype.grammarFileName = ""; - - exports. = ; From 87a99e558aca4cc884c08fda8f44d05d7da10980 Mon Sep 17 00:00:00 2001 From: Tabari Alexander Date: Tue, 29 Oct 2019 12:11:49 -0400 Subject: [PATCH 061/365] Signed contributers.txt --- contributors.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contributors.txt b/contributors.txt index d810479b6..7ba0f2025 100644 --- a/contributors.txt +++ b/contributors.txt @@ -229,4 +229,5 @@ YYYY/MM/DD, github id, Full name, email 2019/09/10, marcospassos, Marcos Passos, marcospassos.com@gmail.com 2019/09/10, amorimjuliana, Juliana Amorim, juu.amorim@gmail.com 2019/09/17, kaz, Kazuki Sawada, kazuki@6715.jp -2019/09/28, lmy269, Mingyang Liu, lmy040758@gmail.com \ No newline at end of file +2019/09/28, lmy269, Mingyang Liu, lmy040758@gmail.com +2019/10/29, tehbone, Tabari Alexander, tehbone@gmail.com From 3ae5d90296aa099373544a71995551fb73826db0 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Wed, 30 Oct 2019 13:35:22 +0100 Subject: [PATCH 062/365] Fixed an error regarding wrong unique_ptr usage Also added VS 2019 project files. --- runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj | 651 ++++++++++++ .../runtime/antlr4cpp-vs2019.vcxproj.filters | 987 ++++++++++++++++++ .../xcshareddata/xcschemes/antlr4.xcscheme | 4 - runtime/Cpp/runtime/src/tree/xpath/XPath.cpp | 7 +- runtime/Cpp/runtime/src/tree/xpath/XPath.h | 1 - 5 files changed, 1642 insertions(+), 8 deletions(-) create mode 100644 runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj create mode 100644 runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj.filters diff --git a/runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj b/runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj new file mode 100644 index 000000000..b93d8f641 --- /dev/null +++ b/runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj @@ -0,0 +1,651 @@ + + + + + Debug Static + Win32 + + + Debug Static + x64 + + + Debug DLL + Win32 + + + Debug DLL + x64 + + + Release Static + Win32 + + + Release Static + x64 + + + Release DLL + Win32 + + + Release DLL + x64 + + + + {83BE66CD-9C4F-4F84-B72A-DD1855C8FC8A} + Win32Proj + antlr4cpp + 10.0 + + + + DynamicLibrary + true + Unicode + v142 + + + StaticLibrary + true + Unicode + v142 + + + DynamicLibrary + true + Unicode + v142 + + + StaticLibrary + true + Unicode + v142 + + + DynamicLibrary + false + true + Unicode + v142 + + + StaticLibrary + false + true + Unicode + v142 + + + DynamicLibrary + false + true + Unicode + v142 + + + StaticLibrary + false + true + Unicode + v142 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + $(SolutionDir)bin\vs-2019\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ + antlr4-runtime + + + true + $(SolutionDir)bin\vs-2019\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ + antlr4-runtime + + + true + $(SolutionDir)bin\vs-2019\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ + antlr4-runtime + + + true + $(SolutionDir)bin\vs-2019\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ + antlr4-runtime + + + false + $(SolutionDir)bin\vs-2019\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ + antlr4-runtime + + + false + $(SolutionDir)bin\vs-2019\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ + antlr4-runtime + + + false + $(SolutionDir)bin\vs-2019\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ + antlr4-runtime + + + false + $(SolutionDir)bin\vs-2019\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ + antlr4-runtime + + + + Level4 + Disabled + ANTLR4CPP_EXPORTS;%(PreprocessorDefinitions) + src;%(AdditionalIncludeDirectories) + + + + + 4251 + true + false + + + Windows + true + + + + + Level4 + Disabled + ANTLR4CPP_EXPORTS;%(PreprocessorDefinitions) + src;%(AdditionalIncludeDirectories) + + + + + 4251 + true + false + + + Windows + true + + + + + Level4 + Disabled + ANTLR4CPP_EXPORTS;%(PreprocessorDefinitions) + src;%(AdditionalIncludeDirectories) + + + + + 4251 + true + false + + + Windows + true + + + + + Level4 + Disabled + ANTLR4CPP_STATIC;%(PreprocessorDefinitions) + src;%(AdditionalIncludeDirectories) + + + + + 4251 + true + false + + + Windows + true + + + + + Level4 + MaxSpeed + true + true + ANTLR4CPP_DLL;ANTLR4CPP_EXPORTS;%(PreprocessorDefinitions) + src;%(AdditionalIncludeDirectories) + + + + + 4251 + true + + + Windows + true + true + true + + + + + Level4 + MaxSpeed + true + true + ANTLR4CPP_EXPORTS;%(PreprocessorDefinitions) + src;%(AdditionalIncludeDirectories) + + + + + 4251 + true + + + Windows + true + true + true + + + + + Level4 + MaxSpeed + true + true + ANTLR4CPP_DLL;ANTLR4CPP_EXPORTS;%(PreprocessorDefinitions) + src;%(AdditionalIncludeDirectories) + + + + + 4251 + true + + + Windows + true + true + true + + + + + Level4 + MaxSpeed + true + true + ANTLR4CPP_STATIC;%(PreprocessorDefinitions) + src;%(AdditionalIncludeDirectories) + + + + + 4251 + true + + + Windows + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj.filters b/runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj.filters new file mode 100644 index 000000000..eb3b687be --- /dev/null +++ b/runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj.filters @@ -0,0 +1,987 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + {587a2726-4856-4d21-937a-fbaebaa90232} + + + {2662156f-1508-4dad-b991-a8298a6db9bf} + + + {5b1e59b1-7fa5-46a5-8d92-965bd709cca0} + + + {9de9fe74-5d67-441d-a972-3cebe6dfbfcc} + + + {89fd3896-0ab1-476d-8d64-a57f10a5e73b} + + + {23939d7b-8e11-421e-80eb-b2cfdfdd64e9} + + + {05f2bacb-b5b2-4ca3-abe1-ca9a7239ecaa} + + + {d3b2ae2d-836b-4c73-8180-aca4ebb7d658} + + + {6674a0f0-c65d-4a00-a9e5-1f243b89d0a2} + + + {1893fffe-7a2b-4708-8ce5-003aa9b749f7} + + + {053a0632-27bc-4043-b5e8-760951b3b5b9} + + + {048c180d-44cf-49ca-a7aa-d0053fea07f5} + + + {3181cae5-cc15-4050-8c45-22af44a823de} + + + {290632d2-c56e-4005-a417-eb83b9531e1a} + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\dfa + + + Header Files\dfa + + + Header Files\dfa + + + Header Files\dfa + + + Header Files\misc + + + Header Files\misc + + + Header Files\misc + + + Header Files\support + + + Header Files\support + + + Header Files\support + + + Header Files\support + + + Header Files\support + + + Header Files\tree + + + Header Files\tree + + + Header Files\tree + + + Header Files\tree + + + Header Files\tree + + + Header Files\tree + + + Header Files\tree + + + Header Files\tree + + + Header Files\tree + + + Header Files\tree + + + Header Files\tree + + + Header Files\tree + + + Header Files\tree + + + Header Files\tree\pattern + + + Header Files\tree\pattern + + + Header Files\tree\pattern + + + Header Files\tree\pattern + + + Header Files\tree\pattern + + + Header Files\tree\pattern + + + Header Files\tree\pattern + + + Header Files\tree\pattern + + + Header Files\tree\xpath + + + Header Files + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\atn + + + Header Files\misc + + + Header Files + + + Header Files + + + Header Files\support + + + Header Files\tree\xpath + + + Header Files\tree\xpath + + + Header Files\tree\xpath + + + Header Files\tree\xpath + + + Header Files\tree\xpath + + + Header Files\tree\xpath + + + Header Files\tree\xpath + + + Header Files\tree\xpath + + + Header Files\tree\xpath + + + Header Files + + + Header Files + + + Header Files\tree + + + Header Files + + + Header Files\support + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\dfa + + + Source Files\dfa + + + Source Files\dfa + + + Source Files\dfa + + + Source Files\misc + + + Source Files\misc + + + Source Files\misc + + + Source Files\support + + + Source Files\support + + + Source Files\support + + + Source Files\tree + + + Source Files\tree + + + Source Files\tree + + + Source Files\tree + + + Source Files\tree\pattern + + + Source Files\tree\pattern + + + Source Files\tree\pattern + + + Source Files\tree\pattern + + + Source Files\tree\pattern + + + Source Files\tree\pattern + + + Source Files\tree\pattern + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files\atn + + + Source Files + + + Source Files + + + Source Files\support + + + Source Files\tree\xpath + + + Source Files\tree\xpath + + + Source Files\tree\xpath + + + Source Files\tree\xpath + + + Source Files\tree\xpath + + + Source Files\tree\xpath + + + Source Files\tree\xpath + + + Source Files\tree\xpath + + + Source Files\tree\xpath + + + Source Files\tree\xpath + + + Source Files + + + Source Files\tree + + + Source Files\tree + + + Source Files + + + Source Files + + + Source Files + + + Source Files\atn + + + Source Files\atn + + + Source Files\misc + + + Source Files + + + Source Files + + + Source Files + + + Source Files\support + + + Source Files\tree + + + Source Files\tree + + + Source Files\tree + + + Source Files\tree + + + Source Files\tree\pattern + + + \ No newline at end of file diff --git a/runtime/Cpp/runtime/antlrcpp.xcodeproj/xcshareddata/xcschemes/antlr4.xcscheme b/runtime/Cpp/runtime/antlrcpp.xcodeproj/xcshareddata/xcschemes/antlr4.xcscheme index f2e132436..dc8e3432a 100644 --- a/runtime/Cpp/runtime/antlrcpp.xcodeproj/xcshareddata/xcschemes/antlr4.xcscheme +++ b/runtime/Cpp/runtime/antlrcpp.xcodeproj/xcshareddata/xcschemes/antlr4.xcscheme @@ -29,8 +29,6 @@ shouldUseLaunchSchemeArgsEnv = "YES"> - - - - > XPath::split(const std::string &path) { @@ -134,14 +133,16 @@ std::vector XPath::evaluate(ParseTree *t) { std::vector work = { &dummyRoot }; size_t i = 0; - while (i < _elements.size()) { + std::vector> elements = split(_path); + + while (i < elements.size()) { std::vector next; for (auto node : work) { if (!node->children.empty()) { // only try to match next element if it has children // e.g., //func/*/stat might have a token node for which // we can't go looking for stat nodes. - auto matching = _elements[i]->evaluate(node); + auto matching = elements[i]->evaluate(node); next.insert(next.end(), matching.begin(), matching.end()); } } diff --git a/runtime/Cpp/runtime/src/tree/xpath/XPath.h b/runtime/Cpp/runtime/src/tree/xpath/XPath.h index 0360c3861..e38d482d5 100755 --- a/runtime/Cpp/runtime/src/tree/xpath/XPath.h +++ b/runtime/Cpp/runtime/src/tree/xpath/XPath.h @@ -73,7 +73,6 @@ namespace xpath { protected: std::string _path; - std::vector> _elements; Parser *_parser; /// Convert word like {@code *} or {@code ID} or {@code expr} to a path From 2758f0ba020e9b88c1ccf599e52c955fec51bc22 Mon Sep 17 00:00:00 2001 From: Ben List Date: Wed, 6 Nov 2019 09:00:01 -0500 Subject: [PATCH 063/365] Missing lexer/parermodifier clarification in example --- doc/lexicon.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/lexicon.md b/doc/lexicon.md index 078dc3e7a..5ae203515 100644 --- a/doc/lexicon.md +++ b/doc/lexicon.md @@ -26,8 +26,8 @@ The Javadoc comments are hidden from the parser and are ignored at the moment. Token names always start with a capital letter and so do lexer rules as defined by Java’s `Character.isUpperCase` method. Parser rule names always start with a lowercase letter (those that fail `Character.isUpperCase`). The initial character can be followed by uppercase and lowercase letters, digits, and underscores. Here are some sample names: ``` -ID, LPAREN, RIGHT_CURLY // token names/rules -expr, simpleDeclarator, d2, header_file // rule names +ID, LPAREN, RIGHT_CURLY // token names/lexer rules +expr, simpleDeclarator, d2, header_file // parser rule names ``` Like Java, ANTLR accepts Unicode characters in ANTLR names: From 9d29a1aa1e42d36b1b4747c49da2793624c2ff7a Mon Sep 17 00:00:00 2001 From: Ben List Date: Wed, 6 Nov 2019 09:02:51 -0500 Subject: [PATCH 064/365] Signed contributors.txt --- contributors.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contributors.txt b/contributors.txt index 9751a6e90..e0d902148 100644 --- a/contributors.txt +++ b/contributors.txt @@ -230,4 +230,5 @@ YYYY/MM/DD, github id, Full name, email 2019/09/10, amorimjuliana, Juliana Amorim, juu.amorim@gmail.com 2019/09/17, kaz, Kazuki Sawada, kazuki@6715.jp 2019/09/28, lmy269, Mingyang Liu, lmy040758@gmail.com -2019/10/31, a-square, Alexei Averchenko, lex.aver@gmail.com \ No newline at end of file +2019/10/31, a-square, Alexei Averchenko, lex.aver@gmail.com +2019/11/05, listba, Ben List, ben.list89@gmail.com From fa4b498fc27f467b4310a612a63eee757d78dfcc Mon Sep 17 00:00:00 2001 From: jerry_liu Date: Thu, 7 Mar 2019 11:31:08 +0800 Subject: [PATCH 065/365] Add pretty print AST --- contributors.txt | 3 ++- runtime/Cpp/runtime/src/RuleContext.cpp | 12 +++++----- runtime/Cpp/runtime/src/RuleContext.h | 6 ++--- runtime/Cpp/runtime/src/tree/ParseTree.h | 4 ++-- .../Cpp/runtime/src/tree/TerminalNodeImpl.cpp | 4 ++-- .../Cpp/runtime/src/tree/TerminalNodeImpl.h | 4 ++-- runtime/Cpp/runtime/src/tree/Trees.cpp | 23 ++++++++++++++----- runtime/Cpp/runtime/src/tree/Trees.h | 6 ++--- 8 files changed, 37 insertions(+), 25 deletions(-) diff --git a/contributors.txt b/contributors.txt index 9751a6e90..6346f1566 100644 --- a/contributors.txt +++ b/contributors.txt @@ -230,4 +230,5 @@ YYYY/MM/DD, github id, Full name, email 2019/09/10, amorimjuliana, Juliana Amorim, juu.amorim@gmail.com 2019/09/17, kaz, Kazuki Sawada, kazuki@6715.jp 2019/09/28, lmy269, Mingyang Liu, lmy040758@gmail.com -2019/10/31, a-square, Alexei Averchenko, lex.aver@gmail.com \ No newline at end of file +2019/10/31, a-square, Alexei Averchenko, lex.aver@gmail.com +2019/03/07, foxeverl, Liu Xinfeng, liuxf1986[at]gmail[dot]com diff --git a/runtime/Cpp/runtime/src/RuleContext.cpp b/runtime/Cpp/runtime/src/RuleContext.cpp index 73cfe24e1..467e5ec42 100755 --- a/runtime/Cpp/runtime/src/RuleContext.cpp +++ b/runtime/Cpp/runtime/src/RuleContext.cpp @@ -75,16 +75,16 @@ antlrcpp::Any RuleContext::accept(tree::ParseTreeVisitor *visitor) { return visitor->visitChildren(this); } -std::string RuleContext::toStringTree(Parser *recog) { - return tree::Trees::toStringTree(this, recog); +std::string RuleContext::toStringTree(Parser *recog, bool pretty) { + return tree::Trees::toStringTree(this, recog, pretty); } -std::string RuleContext::toStringTree(std::vector &ruleNames) { - return tree::Trees::toStringTree(this, ruleNames); +std::string RuleContext::toStringTree(std::vector &ruleNames, bool pretty) { + return tree::Trees::toStringTree(this, ruleNames, pretty); } -std::string RuleContext::toStringTree() { - return toStringTree(nullptr); +std::string RuleContext::toStringTree(bool pretty) { + return toStringTree(nullptr, pretty); } diff --git a/runtime/Cpp/runtime/src/RuleContext.h b/runtime/Cpp/runtime/src/RuleContext.h index bf0ff6631..9ee0d2def 100755 --- a/runtime/Cpp/runtime/src/RuleContext.h +++ b/runtime/Cpp/runtime/src/RuleContext.h @@ -110,15 +110,15 @@ namespace antlr4 { /// (root child1 .. childN). Print just a node if this is a leaf. /// We have to know the recognizer so we can get rule names. /// - virtual std::string toStringTree(Parser *recog) override; + virtual std::string toStringTree(Parser *recog, bool pretty = false) override; /// /// Print out a whole tree, not just a node, in LISP format /// (root child1 .. childN). Print just a node if this is a leaf. /// - virtual std::string toStringTree(std::vector &ruleNames); + virtual std::string toStringTree(std::vector &ruleNames, bool pretty = false); - virtual std::string toStringTree() override; + virtual std::string toStringTree(bool pretty = false) override; virtual std::string toString() override; std::string toString(Recognizer *recog); std::string toString(const std::vector &ruleNames); diff --git a/runtime/Cpp/runtime/src/tree/ParseTree.h b/runtime/Cpp/runtime/src/tree/ParseTree.h index ee50b8039..088aac3ef 100755 --- a/runtime/Cpp/runtime/src/tree/ParseTree.h +++ b/runtime/Cpp/runtime/src/tree/ParseTree.h @@ -39,12 +39,12 @@ namespace tree { /// Print out a whole tree, not just a node, in LISP format /// {@code (root child1 .. childN)}. Print just a node if this is a leaf. - virtual std::string toStringTree() = 0; + virtual std::string toStringTree(bool pretty = false) = 0; virtual std::string toString() = 0; /// Specialize toStringTree so that it can print out more information /// based upon the parser. - virtual std::string toStringTree(Parser *parser) = 0; + virtual std::string toStringTree(Parser *parser, bool pretty = false) = 0; virtual bool operator == (const ParseTree &other) const; diff --git a/runtime/Cpp/runtime/src/tree/TerminalNodeImpl.cpp b/runtime/Cpp/runtime/src/tree/TerminalNodeImpl.cpp index 0f806d4e7..7ab121b73 100755 --- a/runtime/Cpp/runtime/src/tree/TerminalNodeImpl.cpp +++ b/runtime/Cpp/runtime/src/tree/TerminalNodeImpl.cpp @@ -41,7 +41,7 @@ std::string TerminalNodeImpl::getText() { return symbol->getText(); } -std::string TerminalNodeImpl::toStringTree(Parser * /*parser*/) { +std::string TerminalNodeImpl::toStringTree(Parser * /*parser*/, bool /*pretty*/) { return toString(); } @@ -52,6 +52,6 @@ std::string TerminalNodeImpl::toString() { return symbol->getText(); } -std::string TerminalNodeImpl::toStringTree() { +std::string TerminalNodeImpl::toStringTree(bool /*pretty*/) { return toString(); } diff --git a/runtime/Cpp/runtime/src/tree/TerminalNodeImpl.h b/runtime/Cpp/runtime/src/tree/TerminalNodeImpl.h index d865c58e8..6f65d8204 100755 --- a/runtime/Cpp/runtime/src/tree/TerminalNodeImpl.h +++ b/runtime/Cpp/runtime/src/tree/TerminalNodeImpl.h @@ -23,9 +23,9 @@ namespace tree { virtual antlrcpp::Any accept(ParseTreeVisitor *visitor) override; virtual std::string getText() override; - virtual std::string toStringTree(Parser *parser) override; + virtual std::string toStringTree(Parser *parser, bool pretty = false) override; virtual std::string toString() override; - virtual std::string toStringTree() override; + virtual std::string toStringTree(bool pretty = false) override; }; diff --git a/runtime/Cpp/runtime/src/tree/Trees.cpp b/runtime/Cpp/runtime/src/tree/Trees.cpp index fabad0141..72b9b8b48 100755 --- a/runtime/Cpp/runtime/src/tree/Trees.cpp +++ b/runtime/Cpp/runtime/src/tree/Trees.cpp @@ -25,17 +25,17 @@ using namespace antlrcpp; Trees::Trees() { } -std::string Trees::toStringTree(ParseTree *t) { - return toStringTree(t, nullptr); +std::string Trees::toStringTree(ParseTree *t, bool pretty) { + return toStringTree(t, nullptr, pretty); } -std::string Trees::toStringTree(ParseTree *t, Parser *recog) { +std::string Trees::toStringTree(ParseTree *t, Parser *recog, bool pretty) { if (recog == nullptr) - return toStringTree(t, std::vector()); - return toStringTree(t, recog->getRuleNames()); + return toStringTree(t, std::vector(), pretty); + return toStringTree(t, recog->getRuleNames(), pretty); } -std::string Trees::toStringTree(ParseTree *t, const std::vector &ruleNames) { +std::string Trees::toStringTree(ParseTree *t, const std::vector &ruleNames, bool pretty) { std::string temp = antlrcpp::escapeWhitespace(Trees::getNodeText(t, ruleNames), false); if (t->children.empty()) { return temp; @@ -48,6 +48,7 @@ std::string Trees::toStringTree(ParseTree *t, const std::vector &ru std::stack stack; size_t childIndex = 0; ParseTree *run = t; + size_t indentationLevel = 1; while (childIndex < run->children.size()) { if (childIndex > 0) { ss << ' '; @@ -59,6 +60,13 @@ std::string Trees::toStringTree(ParseTree *t, const std::vector &ru stack.push(childIndex); run = child; childIndex = 0; + if (pretty) { + ++indentationLevel; + ss << std::endl; + for (size_t i = 0; i < indentationLevel; ++i) { + ss << " "; + } + } ss << "(" << temp << " "; } else { ss << temp; @@ -68,6 +76,9 @@ std::string Trees::toStringTree(ParseTree *t, const std::vector &ru childIndex = stack.top(); stack.pop(); run = run->parent; + if (pretty) { + --indentationLevel; + } ss << ")"; } else { break; diff --git a/runtime/Cpp/runtime/src/tree/Trees.h b/runtime/Cpp/runtime/src/tree/Trees.h index e6a1bb88e..d9d04624f 100755 --- a/runtime/Cpp/runtime/src/tree/Trees.h +++ b/runtime/Cpp/runtime/src/tree/Trees.h @@ -18,17 +18,17 @@ namespace tree { /// Print out a whole tree in LISP form. getNodeText is used on the /// node payloads to get the text for the nodes. Detect /// parse trees and extract data appropriately. - static std::string toStringTree(ParseTree *t); + static std::string toStringTree(ParseTree *t, bool pretty = false); /// Print out a whole tree in LISP form. getNodeText is used on the /// node payloads to get the text for the nodes. Detect /// parse trees and extract data appropriately. - static std::string toStringTree(ParseTree *t, Parser *recog); + static std::string toStringTree(ParseTree *t, Parser *recog, bool pretty = false); /// Print out a whole tree in LISP form. getNodeText is used on the /// node payloads to get the text for the nodes. Detect /// parse trees and extract data appropriately. - static std::string toStringTree(ParseTree *t, const std::vector &ruleNames); + static std::string toStringTree(ParseTree *t, const std::vector &ruleNames, bool pretty = false); static std::string getNodeText(ParseTree *t, Parser *recog); static std::string getNodeText(ParseTree *t, const std::vector &ruleNames); From cf0658a6ab377307ee2a304c6c0352c83fb4c64c Mon Sep 17 00:00:00 2001 From: liuxinfeng Date: Mon, 11 Nov 2019 21:39:04 +0800 Subject: [PATCH 066/365] update date --- contributors.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributors.txt b/contributors.txt index 6346f1566..b9a7b6a2e 100644 --- a/contributors.txt +++ b/contributors.txt @@ -231,4 +231,4 @@ YYYY/MM/DD, github id, Full name, email 2019/09/17, kaz, Kazuki Sawada, kazuki@6715.jp 2019/09/28, lmy269, Mingyang Liu, lmy040758@gmail.com 2019/10/31, a-square, Alexei Averchenko, lex.aver@gmail.com -2019/03/07, foxeverl, Liu Xinfeng, liuxf1986[at]gmail[dot]com +2019/11/11, foxeverl, Liu Xinfeng, liuxf1986[at]gmail[dot]com From 9029af3ef4fe61e74cc6d280050293453045bc16 Mon Sep 17 00:00:00 2001 From: Felix Nieuwenhuizen Date: Sun, 17 Nov 2019 14:09:35 +0100 Subject: [PATCH 067/365] Fix for CMake 3.14 --- runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake | 10 ++++++++-- runtime/Cpp/cmake/README.md | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake b/runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake index 675cbf579..3523cf2d3 100644 --- a/runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake +++ b/runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake @@ -109,6 +109,12 @@ else() endif() # Seperate build step as rarely people want both +set(ANTLR4_BUILD_DIR ${ANTLR4_ROOT}) +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.14.0") + # CMake 3.14 builds in above's SOURCE_SUBDIR when BUILD_IN_SOURCE is true + set(ANTLR4_BUILD_DIR ${ANTLR4_ROOT}/runtime/Cpp) +endif() + ExternalProject_Add_Step( antlr4_runtime build_static @@ -118,7 +124,7 @@ ExternalProject_Add_Step( DEPENDS antlr4_runtime BYPRODUCTS ${ANTLR4_STATIC_LIBRARIES} EXCLUDE_FROM_MAIN 1 - WORKING_DIRECTORY ${ANTLR4_ROOT}) + WORKING_DIRECTORY ${ANTLR4_BUILD_DIR}) ExternalProject_Add_StepTargets(antlr4_runtime build_static) add_library(antlr4_static STATIC IMPORTED) @@ -135,7 +141,7 @@ ExternalProject_Add_Step( DEPENDS antlr4_runtime BYPRODUCTS ${ANTLR4_SHARED_LIBRARIES} ${ANTLR4_RUNTIME_LIBRARIES} EXCLUDE_FROM_MAIN 1 - WORKING_DIRECTORY ${ANTLR4_ROOT}) + WORKING_DIRECTORY ${ANTLR4_BUILD_DIR}) ExternalProject_Add_StepTargets(antlr4_runtime build_shared) add_library(antlr4_shared SHARED IMPORTED) diff --git a/runtime/Cpp/cmake/README.md b/runtime/Cpp/cmake/README.md index e27c77e50..1db315321 100644 --- a/runtime/Cpp/cmake/README.md +++ b/runtime/Cpp/cmake/README.md @@ -40,7 +40,8 @@ antlr_target(SampleGrammarLexer TLexer.g4 LEXER PACKAGE antlrcpptest) antlr_target(SampleGrammarParser TParser.g4 PARSER PACKAGE antlrcpptest - DEPENDS_ANTLR SampleGrammarLexer) + DEPENDS_ANTLR SampleGrammarLexer + COMPILE_FLAGS -lib ${ANTLR_SampleGrammarLexer_OUTPUT_DIR}) # include generated files in project environment include_directories(${ANTLR_SampleGrammarLexer_OUTPUT_DIR}) From 275184175343ffc49d65d8be4ace49c2803102f3 Mon Sep 17 00:00:00 2001 From: Felix Nieuwenhuizen Date: Sun, 17 Nov 2019 14:12:12 +0100 Subject: [PATCH 068/365] Sign contributors.txty Signed-off-by: Felix Nieuwenhuizen --- contributors.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contributors.txt b/contributors.txt index 2c82eaffb..5bc01729a 100644 --- a/contributors.txt +++ b/contributors.txt @@ -233,3 +233,4 @@ YYYY/MM/DD, github id, Full name, email 2019/09/28, lmy269, Mingyang Liu, lmy040758@gmail.com 2019/10/31, a-square, Alexei Averchenko, lex.aver@gmail.com 2019/11/11, foxeverl, Liu Xinfeng, liuxf1986[at]gmail[dot]com +2019/11/17, felixn, Felix Nieuwenhuizhen, felix@tdlrali.com From 97c5057f6128734348040dfab5f50d17ad62f703 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Sun, 5 May 2019 14:15:27 -0700 Subject: [PATCH 069/365] [Swift] Check process exit status during tests. Check the exit status of the processes run during the runtime-testsuite for Swift. This means that we'll catch low-level issues earlier. This changeset also changes the exception handling so that exceptions throw as far up as possible so that we don't proceed with a test when something has already broken. --- .../v4/test/runtime/swift/BaseSwiftTest.java | 77 ++++++++++++------- 1 file changed, 49 insertions(+), 28 deletions(-) diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java index 34bb56ff7..2e71deb9b 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java @@ -57,12 +57,23 @@ public class BaseSwiftTest implements RuntimeTestSupport { throw new RuntimeException("Swift runtime file not found at:" + swiftRuntime.getPath()); } ANTLR_RUNTIME_PATH = swiftRuntime.getPath(); - fastFailRunProcess(ANTLR_RUNTIME_PATH, SWIFT_CMD, "build"); + try { + fastFailRunProcess(ANTLR_RUNTIME_PATH, SWIFT_CMD, "build"); + } + catch (IOException | InterruptedException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } // shutdown logic Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { - fastFailRunProcess(ANTLR_RUNTIME_PATH, SWIFT_CMD, "package", "clean"); + try { + fastFailRunProcess(ANTLR_RUNTIME_PATH, SWIFT_CMD, "package", "clean"); + } + catch (IOException | InterruptedException e) { + e.printStackTrace(); + } } }); } @@ -145,8 +156,14 @@ public class BaseSwiftTest implements RuntimeTestSupport { String projectName = "testcase-" + System.currentTimeMillis(); String projectDir = getTmpDir() + "/" + projectName; - buildProject(projectDir, projectName); - return execTest(projectDir, projectName); + try { + buildProject(projectDir, projectName); + return execTest(projectDir, projectName); + } + catch (IOException | InterruptedException e) { + e.printStackTrace(); + return null; + } } @Override @@ -183,7 +200,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { Collections.addAll(this.sourceFiles, files); } - private void buildProject(String projectDir, String projectName) { + private void buildProject(String projectDir, String projectName) throws IOException, InterruptedException { mkdir(projectDir); fastFailRunProcess(projectDir, SWIFT_CMD, "package", "init", "--type", "executable"); for (String sourceFile: sourceFiles) { @@ -191,20 +208,16 @@ public class BaseSwiftTest implements RuntimeTestSupport { fastFailRunProcess(getTmpDir(), "mv", "-f", absPath, projectDir + "/Sources/" + projectName); } fastFailRunProcess(getTmpDir(), "mv", "-f", "input", projectDir); - - try { - String dylibPath = ANTLR_RUNTIME_PATH + "/.build/debug/"; - Pair buildResult = runProcess(projectDir, SWIFT_CMD, "build", - "-Xswiftc", "-I"+dylibPath, - "-Xlinker", "-L"+dylibPath, - "-Xlinker", "-lAntlr4", - "-Xlinker", "-rpath", - "-Xlinker", dylibPath); - if (buildResult.b.length() > 0) { - throw new RuntimeException("unit test build failed: " + buildResult.a + "\n" + buildResult.b); - } - } catch (IOException | InterruptedException e) { - e.printStackTrace(); + String dylibPath = ANTLR_RUNTIME_PATH + "/.build/debug/"; +// System.err.println(dylibPath); + Pair buildResult = runProcess(projectDir, SWIFT_CMD, "build", + "-Xswiftc", "-I"+dylibPath, + "-Xlinker", "-L"+dylibPath, + "-Xlinker", "-lAntlr4", + "-Xlinker", "-rpath", + "-Xlinker", dylibPath); + if (buildResult.b.length() > 0) { + throw new IOException("unit test build failed: " + buildResult.a + "\n" + buildResult.b); } } @@ -214,20 +227,22 @@ public class BaseSwiftTest implements RuntimeTestSupport { StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); stdoutVacuum.start(); stderrVacuum.start(); - process.waitFor(); + int status = process.waitFor(); stdoutVacuum.join(); stderrVacuum.join(); + if (status != 0) { + throw new IOException("Process exited with status " + status + ":\n" + stdoutVacuum.toString() + "\n" + stderrVacuum.toString()); + } return new Pair<>(stdoutVacuum.toString(), stderrVacuum.toString()); } - private static void fastFailRunProcess(String workingDir, String... command) { + private static void fastFailRunProcess(String workingDir, String... command) throws IOException, InterruptedException { ProcessBuilder builder = new ProcessBuilder(command); builder.directory(new File(workingDir)); - try { - Process p = builder.start(); - p.waitFor(); - } catch (Exception e) { - e.printStackTrace(); + Process p = builder.start(); + int status = p.waitFor(); + if (status != 0) { + throw new IOException("Process exited with status " + status); } } @@ -251,8 +266,14 @@ public class BaseSwiftTest implements RuntimeTestSupport { addSourceFiles("main.swift"); String projectName = "testcase-" + System.currentTimeMillis(); String projectDir = getTmpDir() + "/" + projectName; - buildProject(projectDir, projectName); - return execTest(projectDir, projectName); + try { + buildProject(projectDir, projectName); + return execTest(projectDir, projectName); + } + catch (IOException | InterruptedException e) { + e.printStackTrace(); + return null; + } } private void writeParserTestFile(String parserName, From 509eeefa4afa8c04e6f29bf18f95ab5aed022a6f Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Thu, 28 Mar 2019 15:24:27 -0700 Subject: [PATCH 070/365] [Swift] Migrate Swift runtime to Swift 5. This switches from using the deprecated hashValue to hash(into:). It also switches from using index to firstIndex (matching the change in the standard library). In the test template, we switch to using String directly instead of String.characters. This also switches all the Travis macOS tests to use the Xcode 10.2 / Mojave image and changes the Linux Swift tests to download Swift 5.0.1. --- .travis.yml | 18 +++++----- .travis/run-tests-swift.sh | 2 +- .../v4/test/runtime/templates/Swift.test.stg | 6 ++-- .../Sources/Antlr4/UnbufferedCharStream.swift | 2 ++ .../Sources/Antlr4/VocabularySingle.swift | 6 ++-- .../Swift/Sources/Antlr4/atn/ATNConfig.swift | 24 ++++++------- .../Sources/Antlr4/atn/ATNConfigSet.swift | 12 +++---- .../Sources/Antlr4/atn/ATNDeserializer.swift | 4 +-- .../Swift/Sources/Antlr4/atn/ATNState.swift | 5 ++- .../Sources/Antlr4/atn/LexerATNConfig.swift | 22 ++++-------- .../Sources/Antlr4/atn/LexerAction.swift | 2 +- .../Antlr4/atn/LexerActionExecutor.swift | 6 ++-- .../Antlr4/atn/LexerChannelAction.swift | 9 ++--- .../Antlr4/atn/LexerCustomAction.swift | 13 ++----- .../Antlr4/atn/LexerIndexedCustomAction.swift | 13 ++----- .../Sources/Antlr4/atn/LexerModeAction.swift | 13 +++---- .../Sources/Antlr4/atn/LexerMoreAction.swift | 12 ++----- .../Antlr4/atn/LexerPopModeAction.swift | 12 ++----- .../Antlr4/atn/LexerPushModeAction.swift | 13 ++----- .../Sources/Antlr4/atn/LexerSkipAction.swift | 10 ++---- .../Sources/Antlr4/atn/LexerTypeAction.swift | 12 ++----- .../Sources/Antlr4/atn/LookupATNConfig.swift | 13 +++---- .../Antlr4/atn/PredictionContext.swift | 4 +-- .../Sources/Antlr4/atn/SemanticContext.swift | 36 ++++++------------- .../Swift/Sources/Antlr4/dfa/DFAState.swift | 6 ++-- .../Swift/Sources/Antlr4/misc/BitSet.swift | 6 +++- .../Swift/Sources/Antlr4/misc/Interval.swift | 11 +++--- .../Sources/Antlr4/misc/IntervalSet.swift | 20 +++-------- 28 files changed, 107 insertions(+), 205 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0220f4ca2..cd774ceae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,42 +74,42 @@ matrix: - clang-3.7 - os: osx compiler: clang - osx_image: xcode10.1 + osx_image: xcode10.2 env: - TARGET=cpp - GROUP=LEXER stage: extended-test - os: osx compiler: clang - osx_image: xcode10.1 + osx_image: xcode10.2 env: - TARGET=cpp - GROUP=PARSER stage: extended-test - os: osx compiler: clang - osx_image: xcode10.1 + osx_image: xcode10.2 env: - TARGET=cpp - GROUP=RECURSION stage: extended-test - os: osx compiler: clang - osx_image: xcode10.1 + osx_image: xcode10.2 env: - TARGET=swift - GROUP=LEXER stage: main-test - os: osx compiler: clang - osx_image: xcode10.1 + osx_image: xcode10.2 env: - TARGET=swift - GROUP=PARSER stage: main-test - os: osx compiler: clang - osx_image: xcode10.1 + osx_image: xcode10.2 env: - TARGET=swift - GROUP=RECURSION @@ -122,19 +122,19 @@ matrix: - GROUP=ALL stage: extended-test - os: osx - osx_image: xcode10.1 + osx_image: xcode10.2 env: - TARGET=dotnet - GROUP=LEXER stage: extended-test - os: osx - osx_image: xcode10.1 + osx_image: xcode10.2 env: - TARGET=dotnet - GROUP=PARSER stage: extended-test - os: osx - osx_image: xcode10.1 + osx_image: xcode10.2 env: - TARGET=dotnet - GROUP=RECURSION diff --git a/.travis/run-tests-swift.sh b/.travis/run-tests-swift.sh index 0e297c503..677f356b2 100755 --- a/.travis/run-tests-swift.sh +++ b/.travis/run-tests-swift.sh @@ -6,7 +6,7 @@ set -euo pipefail # here since environment variables doesn't pass # across scripts if [ $TRAVIS_OS_NAME == "linux" ]; then - export SWIFT_VERSION=swift-4.2.1 + export SWIFT_VERSION=swift-5.0.1 export SWIFT_HOME=$(pwd)/swift/$SWIFT_VERSION-RELEASE-ubuntu16.04/usr/bin/ export PATH=$SWIFT_HOME:$PATH diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg index d45168517..3cc31d141 100755 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg @@ -133,7 +133,7 @@ open func emit() -> Token { private func handleAcceptPositionForIdentifier() -> Bool { let tokenText = getText() var identifierLength = 0 - while ((identifierLength \< tokenText.characters.count) && isIdentifierChar(tokenText[tokenText.characters.index(tokenText.startIndex, offsetBy: identifierLength)])) { + while ((identifierLength \< tokenText.count) && isIdentifierChar(tokenText[tokenText.index(tokenText.startIndex, offsetBy: identifierLength)])) { identifierLength += 1 } @@ -147,8 +147,8 @@ private func handleAcceptPositionForIdentifier() -> Bool { } private func handleAcceptPositionForKeyword(_ keyword:String) -> Bool { - if getInputStream()!.index() > _tokenStartCharIndex + keyword.characters.count { - let offset = keyword.characters.count - 1 + if getInputStream()!.index() > _tokenStartCharIndex + keyword.count { + let offset = keyword.count - 1 (getInterpreter() as! PositionAdjustingLexerATNSimulator).resetAcceptPosition(getInputStream()!, _tokenStartCharIndex + offset, _tokenStartLine, _tokenStartCharPositionInLine + offset) return true } diff --git a/runtime/Swift/Sources/Antlr4/UnbufferedCharStream.swift b/runtime/Swift/Sources/Antlr4/UnbufferedCharStream.swift index 7ef18facb..6b2841d2a 100644 --- a/runtime/Swift/Sources/Antlr4/UnbufferedCharStream.swift +++ b/runtime/Swift/Sources/Antlr4/UnbufferedCharStream.swift @@ -343,6 +343,8 @@ fileprivate struct UInt8StreamIterator: IteratorProtocol { return nil case .opening, .open, .reading: break + @unknown default: + fatalError() } let count = stream.read(&buffer, maxLength: buffer.count) diff --git a/runtime/Swift/Sources/Antlr4/VocabularySingle.swift b/runtime/Swift/Sources/Antlr4/VocabularySingle.swift index 663ff5372..37cd3b23b 100644 --- a/runtime/Swift/Sources/Antlr4/VocabularySingle.swift +++ b/runtime/Swift/Sources/Antlr4/VocabularySingle.swift @@ -155,11 +155,9 @@ public class Vocabulary: Hashable { return String(tokenType) } - public var hashValue: Int { - return Unmanaged.passUnretained(self).toOpaque().hashValue -// return unsafeAddress(of: self).hashValue + public func hash(into hasher: inout Hasher) { + hasher.combine(ObjectIdentifier(self)) } - } public func ==(lhs: Vocabulary, rhs: Vocabulary) -> Bool { diff --git a/runtime/Swift/Sources/Antlr4/atn/ATNConfig.swift b/runtime/Swift/Sources/Antlr4/atn/ATNConfig.swift index b1069a126..cfbe8b615 100644 --- a/runtime/Swift/Sources/Antlr4/atn/ATNConfig.swift +++ b/runtime/Swift/Sources/Antlr4/atn/ATNConfig.swift @@ -126,20 +126,11 @@ public class ATNConfig: Hashable, CustomStringConvertible { } } - /// - /// An ATN configuration is equal to another if both have - /// the same state, they predict the same alternative, and - /// syntactic/semantic contexts are the same. - /// - - public var hashValue: Int { - var hashCode = MurmurHash.initialize(7) - hashCode = MurmurHash.update(hashCode, state.stateNumber) - hashCode = MurmurHash.update(hashCode, alt) - hashCode = MurmurHash.update(hashCode, context) - hashCode = MurmurHash.update(hashCode, semanticContext) - return MurmurHash.finish(hashCode, 4) - + public func hash(into hasher: inout Hasher) { + hasher.combine(state.stateNumber) + hasher.combine(alt) + hasher.combine(context) + hasher.combine(semanticContext) } public var description: String { @@ -166,6 +157,11 @@ public class ATNConfig: Hashable, CustomStringConvertible { } } +/// +/// An ATN configuration is equal to another if both have +/// the same state, they predict the same alternative, and +/// syntactic/semantic contexts are the same. +/// public func ==(lhs: ATNConfig, rhs: ATNConfig) -> Bool { if lhs === rhs { diff --git a/runtime/Swift/Sources/Antlr4/atn/ATNConfigSet.swift b/runtime/Swift/Sources/Antlr4/atn/ATNConfigSet.swift index d1f6a1bb3..7da94377d 100644 --- a/runtime/Swift/Sources/Antlr4/atn/ATNConfigSet.swift +++ b/runtime/Swift/Sources/Antlr4/atn/ATNConfigSet.swift @@ -203,16 +203,16 @@ public final class ATNConfigSet: Hashable, CustomStringConvertible { return false } - public var hashValue: Int { + public func hash(into hasher: inout Hasher) { if isReadonly() { if cachedHashCode == -1 { - cachedHashCode = configsHashValue//configs.hashValue ; + cachedHashCode = configsHashValue } - - return cachedHashCode + hasher.combine(cachedHashCode) + } + else { + hasher.combine(configsHashValue) } - - return configsHashValue // configs.hashValue; } private var configsHashValue: Int { diff --git a/runtime/Swift/Sources/Antlr4/atn/ATNDeserializer.swift b/runtime/Swift/Sources/Antlr4/atn/ATNDeserializer.swift index 65b02efc6..ce8349bc8 100644 --- a/runtime/Swift/Sources/Antlr4/atn/ATNDeserializer.swift +++ b/runtime/Swift/Sources/Antlr4/atn/ATNDeserializer.swift @@ -78,8 +78,8 @@ public class ATNDeserializer { /// internal func isFeatureSupported(_ feature: UUID, _ actualUuid: UUID) -> Bool { let supported = ATNDeserializer.SUPPORTED_UUIDS - guard let featureIndex = supported.index(of: feature), - let actualIndex = supported.index(of: actualUuid) else { + guard let featureIndex = supported.firstIndex(of: feature), + let actualIndex = supported.firstIndex(of: actualUuid) else { return false } return actualIndex >= featureIndex diff --git a/runtime/Swift/Sources/Antlr4/atn/ATNState.swift b/runtime/Swift/Sources/Antlr4/atn/ATNState.swift index 02481d381..24f2dfab3 100644 --- a/runtime/Swift/Sources/Antlr4/atn/ATNState.swift +++ b/runtime/Swift/Sources/Antlr4/atn/ATNState.swift @@ -123,11 +123,10 @@ public class ATNState: Hashable, CustomStringConvertible { public internal(set) final var nextTokenWithinRule: IntervalSet? - public var hashValue: Int { - return stateNumber + public func hash(into hasher: inout Hasher) { + hasher.combine(stateNumber) } - public func isNonGreedyExitState() -> Bool { return false } diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerATNConfig.swift b/runtime/Swift/Sources/Antlr4/atn/LexerATNConfig.swift index 3cacb48cb..1c4a0bb13 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerATNConfig.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerATNConfig.swift @@ -72,22 +72,14 @@ public class LexerATNConfig: ATNConfig { return passedThroughNonGreedyDecision } - override - /*public func hashCode() -> Int { - - }*/ - public var hashValue: Int { - var hashCode = MurmurHash.initialize(7) - hashCode = MurmurHash.update(hashCode, state.stateNumber) - hashCode = MurmurHash.update(hashCode, alt) - hashCode = MurmurHash.update(hashCode, context) - hashCode = MurmurHash.update(hashCode, semanticContext) - hashCode = MurmurHash.update(hashCode, passedThroughNonGreedyDecision ? 1 : 0) - hashCode = MurmurHash.update(hashCode, lexerActionExecutor) - return MurmurHash.finish(hashCode, 6) - + public override func hash(into hasher: inout Hasher) { + hasher.combine(state.stateNumber) + hasher.combine(alt) + hasher.combine(context) + hasher.combine(semanticContext) + hasher.combine(passedThroughNonGreedyDecision) + hasher.combine(lexerActionExecutor) } - } //useless diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerAction.swift b/runtime/Swift/Sources/Antlr4/atn/LexerAction.swift index 99d604da5..5920e405b 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerAction.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerAction.swift @@ -56,7 +56,7 @@ public class LexerAction: Hashable { fatalError(#function + " must be overridden") } - public var hashValue: Int { + public func hash(into hasher: inout Hasher) { fatalError(#function + " must be overridden") } diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerActionExecutor.swift b/runtime/Swift/Sources/Antlr4/atn/LexerActionExecutor.swift index e33e92e9b..365f41a58 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerActionExecutor.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerActionExecutor.swift @@ -176,11 +176,9 @@ public class LexerActionExecutor: Hashable { } - public var hashValue: Int { - return self.hashCode + public func hash(into hasher: inout Hasher) { + hasher.combine(hashCode) } - - } public func ==(lhs: LexerActionExecutor, rhs: LexerActionExecutor) -> Bool { diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerChannelAction.swift b/runtime/Swift/Sources/Antlr4/atn/LexerChannelAction.swift index 4d099f28c..a8fd17db6 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerChannelAction.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerChannelAction.swift @@ -63,12 +63,9 @@ public final class LexerChannelAction: LexerAction, CustomStringConvertible { } - override - public var hashValue: Int { - var hash = MurmurHash.initialize() - hash = MurmurHash.update(hash, getActionType().rawValue) - hash = MurmurHash.update(hash, channel) - return MurmurHash.finish(hash, 2) + public override func hash(into hasher: inout Hasher) { + hasher.combine(getActionType()) + hasher.combine(channel) } public var description: String { diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerCustomAction.swift b/runtime/Swift/Sources/Antlr4/atn/LexerCustomAction.swift index 3fe954236..fc6741ec2 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerCustomAction.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerCustomAction.swift @@ -92,24 +92,17 @@ public final class LexerCustomAction: LexerAction { try lexer.action(nil, ruleIndex, actionIndex) } - override - public var hashValue: Int { - var hash = MurmurHash.initialize() - hash = MurmurHash.update(hash, getActionType().rawValue) - hash = MurmurHash.update(hash, ruleIndex) - hash = MurmurHash.update(hash, actionIndex) - return MurmurHash.finish(hash, 3) + public override func hash(into hasher: inout Hasher) { + hasher.combine(ruleIndex) + hasher.combine(actionIndex) } - } public func ==(lhs: LexerCustomAction, rhs: LexerCustomAction) -> Bool { - if lhs === rhs { return true } - return lhs.ruleIndex == rhs.ruleIndex && lhs.actionIndex == rhs.actionIndex } diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerIndexedCustomAction.swift b/runtime/Swift/Sources/Antlr4/atn/LexerIndexedCustomAction.swift index 501b2e637..b9db97289 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerIndexedCustomAction.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerIndexedCustomAction.swift @@ -96,24 +96,17 @@ public final class LexerIndexedCustomAction: LexerAction { } - public override var hashValue: Int { - var hash = MurmurHash.initialize() - hash = MurmurHash.update(hash, offset) - hash = MurmurHash.update(hash, action) - return MurmurHash.finish(hash, 2) + public override func hash(into hasher: inout Hasher) { + hasher.combine(offset) + hasher.combine(action) } - - } public func ==(lhs: LexerIndexedCustomAction, rhs: LexerIndexedCustomAction) -> Bool { - if lhs === rhs { return true } - return lhs.offset == rhs.offset && lhs.action == rhs.action - } diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerModeAction.swift b/runtime/Swift/Sources/Antlr4/atn/LexerModeAction.swift index bdcaf6fdc..fca51619a 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerModeAction.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerModeAction.swift @@ -62,25 +62,20 @@ public final class LexerModeAction: LexerAction, CustomStringConvertible { public func execute(_ lexer: Lexer) { lexer.mode(mode) } - override - public var hashValue: Int { - var hash = MurmurHash.initialize() - hash = MurmurHash.update(hash, getActionType().rawValue) - hash = MurmurHash.update(hash, mode) - return MurmurHash.finish(hash, 2) + + public override func hash(into hasher: inout Hasher) { + hasher.combine(mode) } + public var description: String { return "mode(\(mode))" } } public func ==(lhs: LexerModeAction, rhs: LexerModeAction) -> Bool { - if lhs === rhs { return true } - return lhs.mode == rhs.mode - } diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerMoreAction.swift b/runtime/Swift/Sources/Antlr4/atn/LexerMoreAction.swift index bb9f197f3..c84f54ffd 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerMoreAction.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerMoreAction.swift @@ -56,23 +56,15 @@ public final class LexerMoreAction: LexerAction, CustomStringConvertible { } - override - public var hashValue: Int { - var hash = MurmurHash.initialize() - hash = MurmurHash.update(hash, getActionType().rawValue) - return MurmurHash.finish(hash, 1) - + public override func hash(into hasher: inout Hasher) { + hasher.combine(ObjectIdentifier(self)) } - public var description: String { return "more" } } public func ==(lhs: LexerMoreAction, rhs: LexerMoreAction) -> Bool { - return lhs === rhs - - } diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerPopModeAction.swift b/runtime/Swift/Sources/Antlr4/atn/LexerPopModeAction.swift index f35e78304..68fbe3fa6 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerPopModeAction.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerPopModeAction.swift @@ -57,21 +57,15 @@ public final class LexerPopModeAction: LexerAction, CustomStringConvertible { } - override - public var hashValue: Int { - var hash = MurmurHash.initialize() - hash = MurmurHash.update(hash, getActionType().rawValue) - return MurmurHash.finish(hash, 1) - + public override func hash(into hasher: inout Hasher) { + hasher.combine(ObjectIdentifier(self)) } + public var description: String { return "popMode" } } public func ==(lhs: LexerPopModeAction, rhs: LexerPopModeAction) -> Bool { - return lhs === rhs - - } diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerPushModeAction.swift b/runtime/Swift/Sources/Antlr4/atn/LexerPushModeAction.swift index fb432c497..ffac047aa 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerPushModeAction.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerPushModeAction.swift @@ -63,15 +63,10 @@ public final class LexerPushModeAction: LexerAction, CustomStringConvertible { lexer.pushMode(mode) } - - override - public var hashValue: Int { - var hash = MurmurHash.initialize() - hash = MurmurHash.update(hash, getActionType().rawValue) - hash = MurmurHash.update(hash, mode) - return MurmurHash.finish(hash, 2) - + public override func hash(into hasher: inout Hasher) { + hasher.combine(mode) } + public var description: String { return "pushMode(\(mode))" } @@ -79,10 +74,8 @@ public final class LexerPushModeAction: LexerAction, CustomStringConvertible { public func ==(lhs: LexerPushModeAction, rhs: LexerPushModeAction) -> Bool { - if lhs === rhs { return true } - return lhs.mode == rhs.mode } diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerSkipAction.swift b/runtime/Swift/Sources/Antlr4/atn/LexerSkipAction.swift index bbdd06d2f..db8b1d8e7 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerSkipAction.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerSkipAction.swift @@ -56,19 +56,15 @@ public final class LexerSkipAction: LexerAction, CustomStringConvertible { } - override - public var hashValue: Int { - var hash = MurmurHash.initialize() - hash = MurmurHash.update(hash, getActionType().rawValue) - return MurmurHash.finish(hash, 1) + public override func hash(into hasher: inout Hasher) { + hasher.combine(ObjectIdentifier(self)) } + public var description: String { return "skip" } - } public func ==(lhs: LexerSkipAction, rhs: LexerSkipAction) -> Bool { - return lhs === rhs } diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerTypeAction.swift b/runtime/Swift/Sources/Antlr4/atn/LexerTypeAction.swift index c08649ea3..8c7157be2 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerTypeAction.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerTypeAction.swift @@ -62,24 +62,18 @@ public class LexerTypeAction: LexerAction, CustomStringConvertible { } - override - public var hashValue: Int { - var hash = MurmurHash.initialize() - hash = MurmurHash.update(hash, getActionType().rawValue) - hash = MurmurHash.update(hash, type) - return MurmurHash.finish(hash, 2) + public override func hash(into hasher: inout Hasher) { + hasher.combine(type) } + public var description: String { return "type(\(type))" } - } public func ==(lhs: LexerTypeAction, rhs: LexerTypeAction) -> Bool { - if lhs === rhs { return true } - return lhs.type == rhs.type } diff --git a/runtime/Swift/Sources/Antlr4/atn/LookupATNConfig.swift b/runtime/Swift/Sources/Antlr4/atn/LookupATNConfig.swift index aef419b1d..d3a04a1cc 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LookupATNConfig.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LookupATNConfig.swift @@ -20,17 +20,12 @@ public class LookupATNConfig: Hashable { // dup config = old } - public var hashValue: Int { - - var hashCode: Int = 7 - hashCode = 31 * hashCode + config.state.stateNumber - hashCode = 31 * hashCode + config.alt - hashCode = 31 * hashCode + config.semanticContext.hashValue - return hashCode + public func hash(into hasher: inout Hasher) { + hasher.combine(config.state.stateNumber) + hasher.combine(config.alt) + hasher.combine(config.semanticContext) } - - } public func ==(lhs: LookupATNConfig, rhs: LookupATNConfig) -> Bool { diff --git a/runtime/Swift/Sources/Antlr4/atn/PredictionContext.swift b/runtime/Swift/Sources/Antlr4/atn/PredictionContext.swift index 376088905..78b0614b5 100644 --- a/runtime/Swift/Sources/Antlr4/atn/PredictionContext.swift +++ b/runtime/Swift/Sources/Antlr4/atn/PredictionContext.swift @@ -105,8 +105,8 @@ public class PredictionContext: Hashable, CustomStringConvertible { return getReturnState(size() - 1) == PredictionContext.EMPTY_RETURN_STATE } - public final var hashValue: Int { - return cachedHashCode + public func hash(into hasher: inout Hasher) { + hasher.combine(cachedHashCode) } static func calculateEmptyHashCode() -> Int { diff --git a/runtime/Swift/Sources/Antlr4/atn/SemanticContext.swift b/runtime/Swift/Sources/Antlr4/atn/SemanticContext.swift index 118f549f5..dffe24a1f 100644 --- a/runtime/Swift/Sources/Antlr4/atn/SemanticContext.swift +++ b/runtime/Swift/Sources/Antlr4/atn/SemanticContext.swift @@ -61,7 +61,7 @@ public class SemanticContext: Hashable, CustomStringConvertible { return self } - public var hashValue: Int { + public func hash(into hasher: inout Hasher) { fatalError(#function + " must be overridden") } @@ -94,16 +94,12 @@ public class SemanticContext: Hashable, CustomStringConvertible { return try parser.sempred(localctx, ruleIndex, predIndex) } - override - public var hashValue: Int { - var hashCode = MurmurHash.initialize() - hashCode = MurmurHash.update(hashCode, ruleIndex) - hashCode = MurmurHash.update(hashCode, predIndex) - hashCode = MurmurHash.update(hashCode, isCtxDependent ? 1 : 0) - return MurmurHash.finish(hashCode, 3) + public override func hash(into hasher: inout Hasher) { + hasher.combine(ruleIndex) + hasher.combine(predIndex) + hasher.combine(isCtxDependent) } - override public var description: String { return "{\(ruleIndex):\(predIndex)}?" @@ -138,11 +134,8 @@ public class SemanticContext: Hashable, CustomStringConvertible { } - override - public var hashValue: Int { - var hashCode: Int = 1 - hashCode = 31 * hashCode + precedence - return hashCode + public override func hash(into hasher: inout Hasher) { + hasher.combine(precedence) } override @@ -214,12 +207,8 @@ public class SemanticContext: Hashable, CustomStringConvertible { } - override - public var hashValue: Int { - //MurmurHash.hashCode(opnds, AND.class.hashCode()); - let seed = 1554547125 - //NSStringFromClass(AND.self).hashValue - return MurmurHash.hashCode(opnds, seed) + public override func hash(into hasher: inout Hasher) { + hasher.combine(opnds) } /// @@ -323,11 +312,8 @@ public class SemanticContext: Hashable, CustomStringConvertible { return opnds } - - override - public var hashValue: Int { - - return MurmurHash.hashCode(opnds, NSStringFromClass(OR.self).hashValue) + public override func hash(into hasher: inout Hasher) { + hasher.combine(opnds) } /// diff --git a/runtime/Swift/Sources/Antlr4/dfa/DFAState.swift b/runtime/Swift/Sources/Antlr4/dfa/DFAState.swift index d44dd34ec..8a705872c 100644 --- a/runtime/Swift/Sources/Antlr4/dfa/DFAState.swift +++ b/runtime/Swift/Sources/Antlr4/dfa/DFAState.swift @@ -109,10 +109,8 @@ public final class DFAState: Hashable, CustomStringConvertible { } - public var hashValue: Int { - var hash = MurmurHash.initialize(7) - hash = MurmurHash.update(hash, configs.hashValue) - return MurmurHash.finish(hash, 1) + public func hash(into hasher: inout Hasher) { + hasher.combine(configs) } public var description: String { diff --git a/runtime/Swift/Sources/Antlr4/misc/BitSet.swift b/runtime/Swift/Sources/Antlr4/misc/BitSet.swift index ef03b8aa7..5e611efc2 100644 --- a/runtime/Swift/Sources/Antlr4/misc/BitSet.swift +++ b/runtime/Swift/Sources/Antlr4/misc/BitSet.swift @@ -1053,7 +1053,7 @@ public class BitSet: Hashable, CustomStringConvertible { /// /// - returns: the hash code value for this bit set /// - public var hashValue: Int { + private var hashCode: Int { var h: Int64 = 1234 var i: Int = wordsInUse i -= 1 @@ -1065,6 +1065,10 @@ public class BitSet: Hashable, CustomStringConvertible { return Int(Int32((h >> 32) ^ h)) } + public func hash(into hasher: inout Hasher) { + hasher.combine(hashCode) + } + /// /// Returns the number of bits of space actually in use by this /// `BitSet` to represent bit values. diff --git a/runtime/Swift/Sources/Antlr4/misc/Interval.swift b/runtime/Swift/Sources/Antlr4/misc/Interval.swift index e3b4ad49f..f03ad9b08 100644 --- a/runtime/Swift/Sources/Antlr4/misc/Interval.swift +++ b/runtime/Swift/Sources/Antlr4/misc/Interval.swift @@ -62,13 +62,12 @@ public class Interval: Hashable { } - public var hashValue: Int { - var hash: Int = 23 - hash = hash * 31 + a - hash = hash * 31 + b - return hash + public func hash(into hasher: inout Hasher) { + hasher.combine(a) + hasher.combine(b) } - /// + + /// /// Does this start completely before other? Disjoint /// public func startsBeforeDisjoint(_ other: Interval) -> Bool { diff --git a/runtime/Swift/Sources/Antlr4/misc/IntervalSet.swift b/runtime/Swift/Sources/Antlr4/misc/IntervalSet.swift index 04c590db0..522952250 100644 --- a/runtime/Swift/Sources/Antlr4/misc/IntervalSet.swift +++ b/runtime/Swift/Sources/Antlr4/misc/IntervalSet.swift @@ -470,25 +470,13 @@ public class IntervalSet: IntSet, Hashable, CustomStringConvertible { return intervals } - - public func hashCode() -> Int { - var hash = MurmurHash.initialize() - for I: Interval in intervals { - hash = MurmurHash.update(hash, I.a) - hash = MurmurHash.update(hash, I.b) + public func hash(into hasher: inout Hasher) { + for interval in intervals { + hasher.combine(interval.a) + hasher.combine(interval.b) } - - return MurmurHash.finish(hash, intervals.count * 2) } - public var hashValue: Int { - var hash = MurmurHash.initialize() - for I: Interval in intervals { - hash = MurmurHash.update(hash, I.a) - hash = MurmurHash.update(hash, I.b) - } - return MurmurHash.finish(hash, intervals.count * 2) - } /// /// Are two IntervalSets equal? Because all intervals are sorted /// and disjoint, equals is a simple linear walk over both lists From 10d182cf6634504560e0c019aa53f5de4061fad4 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Thu, 11 Jul 2019 17:40:36 -0700 Subject: [PATCH 071/365] [Swift] Change the generation of the copyFrom method to work around SR-10260. This bug in Swift 5's compiler causes a SIL verification error when calling super.foo without overriding foo in the subclass. In our case, we are generating a copyFrom implementation in the subclass, and calling super.copyFrom. Crucially, the subclass implementation uses the context subtype for the argument, which means that it doesn't actually override the superclass's implementation. This then tickles the SIL verification bug. Work around this by changing our autogenerated copyFrom definition to use ParserRuleContext as the argument type. This means that it actually overrides the implementation in ParserRuleContext, and so the super.copyFrom call is generated correctly. --- .../org/antlr/v4/tool/templates/codegen/Swift/Swift.stg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg index 7d74dc5d0..cefb1e248 100755 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg @@ -848,11 +848,11 @@ StructDecl(struct,ctorAttrs,attrs,getters,dispatchMethods,interfaces,extensionMe func getRuleIndex() -> Int { return .RULE_ } - - init() { }!> + - func copyFrom(_ ctx: ) { - super.copyFrom(ctx) + override func copyFrom(_ ctx_: ParserRuleContext) { + super.copyFrom(ctx_) + let ctx = ctx_ as! = ctx.;}; separator="\n"> } From a701f2b099cd126206bb72e48ea6f4143f0ef469 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Wed, 28 Aug 2019 14:07:25 -0700 Subject: [PATCH 072/365] [Swift] Replace all uses of NSStringFromClass. Bridging from NSString to String in Swift 5 is a transcoding operation, so it is expensive. Use String(describing:) instead of NSStringFromClass to avoid this cost. --- .../Antlr4/atn/PredictionContext.swift | 2 +- .../Swift/Sources/Antlr4/atn/Transition.swift | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/runtime/Swift/Sources/Antlr4/atn/PredictionContext.swift b/runtime/Swift/Sources/Antlr4/atn/PredictionContext.swift index 78b0614b5..0e7026df4 100644 --- a/runtime/Swift/Sources/Antlr4/atn/PredictionContext.swift +++ b/runtime/Swift/Sources/Antlr4/atn/PredictionContext.swift @@ -668,7 +668,7 @@ public class PredictionContext: Hashable, CustomStringConvertible { } public func toString(_ recog: Recognizer) -> String { - return NSStringFromClass(PredictionContext.self) + return String(describing: PredictionContext.self) // return toString(recog, ParserRuleContext.EMPTY); } diff --git a/runtime/Swift/Sources/Antlr4/atn/Transition.swift b/runtime/Swift/Sources/Antlr4/atn/Transition.swift index 5af20e22a..af31c134e 100644 --- a/runtime/Swift/Sources/Antlr4/atn/Transition.swift +++ b/runtime/Swift/Sources/Antlr4/atn/Transition.swift @@ -54,16 +54,16 @@ public class Transition { public static let serializationTypes: Dictionary = [ - NSStringFromClass(EpsilonTransition.self): EPSILON, - NSStringFromClass(RangeTransition.self): RANGE, - NSStringFromClass(RuleTransition.self): RULE, - NSStringFromClass(PredicateTransition.self): PREDICATE, - NSStringFromClass(AtomTransition.self): ATOM, - NSStringFromClass(ActionTransition.self): ACTION, - NSStringFromClass(SetTransition.self): SET, - NSStringFromClass(NotSetTransition.self): NOT_SET, - NSStringFromClass(WildcardTransition.self): WILDCARD, - NSStringFromClass(PrecedencePredicateTransition.self): PRECEDENCE, + String(describing: EpsilonTransition.self): EPSILON, + String(describing: RangeTransition.self): RANGE, + String(describing: RuleTransition.self): RULE, + String(describing: PredicateTransition.self): PREDICATE, + String(describing: AtomTransition.self): ATOM, + String(describing: ActionTransition.self): ACTION, + String(describing: SetTransition.self): SET, + String(describing: NotSetTransition.self): NOT_SET, + String(describing: WildcardTransition.self): WILDCARD, + String(describing: PrecedencePredicateTransition.self): PRECEDENCE, ] From 3bd5465881f52681909b193978f753fad94c77b1 Mon Sep 17 00:00:00 2001 From: Mark Lilback Date: Mon, 18 Nov 2019 13:24:40 -0500 Subject: [PATCH 073/365] [Swift] fixed compiler warnings about using var instead of let --- contributors.txt | 1 + .../Sources/Antlr4/DiagnosticErrorListener.swift | 2 +- runtime/Swift/Sources/Antlr4/Parser.swift | 2 +- runtime/Swift/Sources/Antlr4/atn/ParseInfo.swift | 14 +++++++------- .../Sources/Antlr4/atn/ParserATNSimulator.swift | 2 +- .../Swift/Sources/Antlr4/misc/IntervalSet.swift | 4 ++-- 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/contributors.txt b/contributors.txt index 2c82eaffb..84c9c7799 100644 --- a/contributors.txt +++ b/contributors.txt @@ -233,3 +233,4 @@ YYYY/MM/DD, github id, Full name, email 2019/09/28, lmy269, Mingyang Liu, lmy040758@gmail.com 2019/10/31, a-square, Alexei Averchenko, lex.aver@gmail.com 2019/11/11, foxeverl, Liu Xinfeng, liuxf1986[at]gmail[dot]com +2019/11/18, mlilback, Mark Lilback, mark@lilback.com diff --git a/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift b/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift index 4109f672b..144d34921 100644 --- a/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift +++ b/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift @@ -101,7 +101,7 @@ public class DiagnosticErrorListener: BaseErrorListener { let decision: Int = dfa.decision let ruleIndex: Int = dfa.atnStartState.ruleIndex! - var ruleNames: [String] = recognizer.getRuleNames() + let ruleNames: [String] = recognizer.getRuleNames() if ruleIndex < 0 || ruleIndex >= ruleNames.count { return String(decision) } diff --git a/runtime/Swift/Sources/Antlr4/Parser.swift b/runtime/Swift/Sources/Antlr4/Parser.swift index 1f70f2eea..75c134ce5 100644 --- a/runtime/Swift/Sources/Antlr4/Parser.swift +++ b/runtime/Swift/Sources/Antlr4/Parser.swift @@ -941,7 +941,7 @@ open class Parser: Recognizer { public func getRuleInvocationStack(_ p: RuleContext?) -> [String] { var p = p - var ruleNames = getRuleNames() + let ruleNames = getRuleNames() var stack = [String]() while let pWrap = p { // compute what follows who invoked us diff --git a/runtime/Swift/Sources/Antlr4/atn/ParseInfo.swift b/runtime/Swift/Sources/Antlr4/atn/ParseInfo.swift index 244226a5c..2eb404592 100644 --- a/runtime/Swift/Sources/Antlr4/atn/ParseInfo.swift +++ b/runtime/Swift/Sources/Antlr4/atn/ParseInfo.swift @@ -40,7 +40,7 @@ public class ParseInfo { /// full-context predictions during parsing. /// public func getLLDecisions() -> Array { - var decisions: [DecisionInfo] = atnSimulator.getDecisionInfo() + let decisions: [DecisionInfo] = atnSimulator.getDecisionInfo() var LL: Array = Array() let length = decisions.count for i in 0.. Int64 { - var decisions: [DecisionInfo] = atnSimulator.getDecisionInfo() + let decisions: [DecisionInfo] = atnSimulator.getDecisionInfo() var t: Int64 = 0 let length = decisions.count for i in 0.. Int64 { - var decisions: [DecisionInfo] = atnSimulator.getDecisionInfo() + let decisions: [DecisionInfo] = atnSimulator.getDecisionInfo() var k: Int64 = 0 let length = decisions.count for i in 0.. Int64 { - var decisions: [DecisionInfo] = atnSimulator.getDecisionInfo() + let decisions: [DecisionInfo] = atnSimulator.getDecisionInfo() var k: Int64 = 0 let length = decisions.count for i in 0.. Int64 { - var decisions: [DecisionInfo] = atnSimulator.getDecisionInfo() + let decisions: [DecisionInfo] = atnSimulator.getDecisionInfo() var k: Int64 = 0 let length = decisions.count for i in 0.. Int64 { - var decisions: [DecisionInfo] = atnSimulator.getDecisionInfo() + let decisions: [DecisionInfo] = atnSimulator.getDecisionInfo() var k: Int64 = 0 let length = decisions.count for i in 0.. Int64 { - var decisions: [DecisionInfo] = atnSimulator.getDecisionInfo() + let decisions: [DecisionInfo] = atnSimulator.getDecisionInfo() var k: Int64 = 0 let length = decisions.count for i in 0.. DFAState? { - var edges = previousD.edges + let edges = previousD.edges if edges == nil || (t + 1) < 0 || (t + 1) >= (edges!.count) { return nil } diff --git a/runtime/Swift/Sources/Antlr4/misc/IntervalSet.swift b/runtime/Swift/Sources/Antlr4/misc/IntervalSet.swift index 522952250..16bbb929a 100644 --- a/runtime/Swift/Sources/Antlr4/misc/IntervalSet.swift +++ b/runtime/Swift/Sources/Antlr4/misc/IntervalSet.swift @@ -326,8 +326,8 @@ public class IntervalSet: IntSet, Hashable, CustomStringConvertible { return nil // nothing in common with null set } - var myIntervals = self.intervals - var theirIntervals = (other as! IntervalSet).intervals + let myIntervals = self.intervals + let theirIntervals = (other as! IntervalSet).intervals var intersection: IntervalSet? = nil let mySize = myIntervals.count let theirSize = theirIntervals.count From f650a0be67e9957041a80cc19f65b315226fbb6b Mon Sep 17 00:00:00 2001 From: WalterCouto Date: Wed, 11 Dec 2019 00:40:20 -0500 Subject: [PATCH 074/365] Separate bug fix to Java runtimes --- .../Java/src/org/antlr/v4/runtime/BufferedTokenStream.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index 907d59a3b..e1dc4b197 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -441,17 +441,15 @@ public class BufferedTokenStream implements TokenStream { @Override public String getText() { - fill(); return getText(Interval.of(0,size()-1)); } @Override public String getText(Interval interval) { - lazyInit(); int start = interval.a; int stop = interval.b; if ( start<0 || stop<0 ) return ""; - sync(stop); + fill(); if ( stop>=tokens.size() ) stop = tokens.size()-1; StringBuilder buf = new StringBuilder(); From ce59e171cb5c8d57471dae8ca893fff0d974011c Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Tue, 17 Dec 2019 09:53:59 -0800 Subject: [PATCH 075/365] [PHP] Fix AppVeyor tests Install PHP to C:\tools\php. This fixes the AppVeyor tests, because they were looking in C:\tools\php73, but the Chocolatey package has recently moved to 7.4. --- appveyor.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d58657c98..5788acadf 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,7 +6,8 @@ image: Visual Studio 2017 build: off install: - git submodule update --init --recursive - - cinst -y php composer + - cinst -y php --params "/InstallDir:C:\tools\php" + - cinst -y composer build_script: - mvn -DskipTests install --batch-mode - msbuild /target:restore /target:rebuild /property:Configuration=Release /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /verbosity:detailed runtime/CSharp/runtime/CSharp/Antlr4.dotnet.sln @@ -14,7 +15,7 @@ build_script: after_build: - msbuild /target:pack /property:Configuration=Release /verbosity:detailed runtime/CSharp/runtime/CSharp/Antlr4.dotnet.sln test_script: - - mvn install -Dantlr-php-php="C:\tools\php73\php.exe" -Dantlr-python2-python="C:\Python27\python.exe" -Dantlr-python3-python="C:\Python35\python.exe" -Dantlr-javascript-nodejs="C:\Program Files (x86)\nodejs\node.exe" --batch-mode + - mvn install -Dantlr-php-php="C:\tools\php\php.exe" -Dantlr-python2-python="C:\Python27\python.exe" -Dantlr-python3-python="C:\Python35\python.exe" -Dantlr-javascript-nodejs="C:\Program Files (x86)\nodejs\node.exe" --batch-mode artifacts: - path: 'runtime\**\*.nupkg' name: NuGet \ No newline at end of file From cc0882c8cb389943c387a0bd648f78f995200ec5 Mon Sep 17 00:00:00 2001 From: Marcos Passos Date: Tue, 17 Dec 2019 16:12:16 -0300 Subject: [PATCH 076/365] Add the submodule update step to the release docs --- doc/releasing-antlr.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index a44e48e7d..6baa0c748 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -22,6 +22,14 @@ $ git push origin 4.7-rc1 $ git push upstream 4.7-rc1 ``` +## Update submodules + +Update the runtime submodules by running the following command: + +```bash + git submodule update --recursive + ``` + ## Bump version Edit the repository looking for 4.5 or whatever and update it. Bump version in the following files: From c487b1a58012247ff5490fc1d4689d5de361cdf3 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Fri, 3 Jan 2020 09:01:13 +0100 Subject: [PATCH 077/365] Create pre-built binaries for VS 2017/19 instead of older VS versions --- runtime/Cpp/deploy-windows.cmd | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/runtime/Cpp/deploy-windows.cmd b/runtime/Cpp/deploy-windows.cmd index 5660f26a2..bb21af7d9 100644 --- a/runtime/Cpp/deploy-windows.cmd +++ b/runtime/Cpp/deploy-windows.cmd @@ -5,52 +5,51 @@ if exist bin rmdir /S /Q runtime\bin if exist obj rmdir /S /Q runtime\obj if exist lib rmdir /S /Q lib if exist antlr4-runtime rmdir /S /Q antlr4-runtime -if exist antlr4-cpp-runtime-vs2013.zip erase antlr4-cpp-runtime-vs2013.zip -if exist antlr4-cpp-runtime-vs2015.zip erase antlr4-cpp-runtime-vs2015.zip +if exist antlr4-cpp-runtime-vs2017.zip erase antlr4-cpp-runtime-vs2017.zip +if exist antlr4-cpp-runtime-vs2019.zip erase antlr4-cpp-runtime-vs2019.zip rem Headers xcopy runtime\src\*.h antlr4-runtime\ /s rem Binaries -rem VS 2013 disabled by default. Change the X to a C to enable it. -if exist "X:\Program Files (x86)\Microsoft Visual Studio 12.0\Common7\Tools\VsDevCmd.bat" ( - call "C:\Program Files (x86)\Microsoft Visual Studio 12.0\Common7\Tools\VsDevCmd.bat" +if exist "C:\Program Files (x86)\Microsoft Visual Studio 15.0\Common7\Tools\VsDevCmd.bat" ( + call "C:\Program Files (x86)\Microsoft Visual Studio 15.0\Common7\Tools\VsDevCmd.bat" pushd runtime - msbuild antlr4cpp-vs2013.vcxproj /p:configuration="Release DLL" /p:platform=Win32 - msbuild antlr4cpp-vs2013.vcxproj /p:configuration="Release DLL" /p:platform=x64 + msbuild antlr4cpp-vs2017.vcxproj /p:configuration="Release DLL" /p:platform=Win32 + msbuild antlr4cpp-vs2017.vcxproj /p:configuration="Release DLL" /p:platform=x64 popd - 7z a antlr4-cpp-runtime-vs2013.zip antlr4-runtime + 7z a antlr4-cpp-runtime-vs2017.zip antlr4-runtime xcopy runtime\bin\*.dll lib\ /s xcopy runtime\bin\*.lib lib\ /s - 7z a antlr4-cpp-runtime-vs2013.zip lib + 7z a antlr4-cpp-runtime-vs2017.zip lib rmdir /S /Q lib rmdir /S /Q runtime\bin rmdir /S /Q runtime\obj - rem if exist antlr4-cpp-runtime-vs2013.zip copy antlr4-cpp-runtime-vs2013.zip ~/antlr/sites/website-antlr4/download + rem if exist antlr4-cpp-runtime-vs2017.zip copy antlr4-cpp-runtime-vs2017.zip ~/antlr/sites/website-antlr4/download ) -if exist "C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\Tools\VsDevCmd.bat" ( - call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\Tools\VsDevCmd.bat" +if exist "C:\Program Files (x86)\Microsoft Visual Studio 16.0\Common7\Tools\VsDevCmd.bat" ( + call "C:\Program Files (x86)\Microsoft Visual Studio 16.0\Common7\Tools\VsDevCmd.bat" pushd runtime - msbuild antlr4cpp-vs2015.vcxproj /p:configuration="Release DLL" /p:platform=Win32 - msbuild antlr4cpp-vs2015.vcxproj /p:configuration="Release DLL" /p:platform=x64 + msbuild antlr4cpp-vs2019.vcxproj /p:configuration="Release DLL" /p:platform=Win32 + msbuild antlr4cpp-vs2019.vcxproj /p:configuration="Release DLL" /p:platform=x64 popd - 7z a antlr4-cpp-runtime-vs2015.zip antlr4-runtime + 7z a antlr4-cpp-runtime-vs2019.zip antlr4-runtime xcopy runtime\bin\*.dll lib\ /s xcopy runtime\bin\*.lib lib\ /s - 7z a antlr4-cpp-runtime-vs2015.zip lib + 7z a antlr4-cpp-runtime-vs2019.zip lib rmdir /S /Q lib rmdir /S /Q runtime\bin rmdir /S /Q runtime\obj - rem if exist antlr4-cpp-runtime-vs2015.zip copy antlr4-cpp-runtime-vs2015.zip ~/antlr/sites/website-antlr4/download + rem if exist antlr4-cpp-runtime-vs2019.zip copy antlr4-cpp-runtime-vs2019.zip ~/antlr/sites/website-antlr4/download ) rmdir /S /Q antlr4-runtime From d295462e2e8558e1945742b925f1bdee47887039 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Fri, 3 Jan 2020 11:54:30 +0100 Subject: [PATCH 078/365] Doc update for C++ target and fixes for VS 2019 builds --- doc/releasing-antlr.md | 8 ++--- runtime/Cpp/README.md | 14 ++++---- runtime/Cpp/deploy-windows.cmd | 32 ++++++++++++++++--- runtime/Cpp/runtime/antlr4cpp-vs2017.vcxproj | 16 +++++----- runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj | 3 +- .../runtime/antlr4cpp-vs2019.vcxproj.filters | 9 ++++-- .../runtime/src/atn/ParserATNSimulator.cpp | 4 +++ 7 files changed, 58 insertions(+), 28 deletions(-) diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index 6baa0c748..8eb1d525d 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -379,7 +379,7 @@ There are links to the artifacts in [download.html](http://www.antlr.org/downloa The C++ target is the most complex one, because it addresses multiple platforms, which require individual handling. We have 4 scenarios to cover: -* **Windows**: static and dynamic libraries for the VC++ runtime 2013 or 2015 (corresponding to Visual Studio 2013 or 2015) + header files. All that in 32 and 64bit, debug + release. +* **Windows**: static and dynamic libraries for the VC++ runtime 2017 or 2019 (corresponding to Visual Studio 2017 or 2019) + header files. All that in 32 and 64bit, debug + release. * **MacOS**: static and dynamic release libraries + header files. * **iOS**: no prebuilt binaries, but just a zip of the source, including the XCode project to build everything from source. * **Linux**: no prebuilt binaries, but just a zip of the source code, including the cmake file to build everything from source there. @@ -404,12 +404,12 @@ cd runtime/Cpp cp antlr4-cpp-runtime-source.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-source.zip ``` -On a Windows machine the build scripts checks if VS 2013 and/or VS 2015 are installed and builds binaries for each, if found. This script requires 7z to be installed (http://7-zip.org then do `set PATH=%PATH%;C:\Program Files\7-Zip\` from DOS not powershell). +On a Windows machine the build scripts checks if VS 2017 and/or VS 2019 are installed and builds binaries for each, if found. This script requires 7z to be installed (http://7-zip.org then do `set PATH=%PATH%;C:\Program Files\7-Zip\` from DOS not powershell). ```bash cd runtime/Cpp -deploy-windows.cmd -cp runtime\bin\vs-2015\x64\Release DLL\antlr4-cpp-runtime-vs2015.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-vs2015.zip +deploy-windows.cmd Community +cp antlr4-cpp-runtime-vs2019.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-vs2019.zip ``` Move target to website (**_rename to a specific ANTLR version first if needed_**): diff --git a/runtime/Cpp/README.md b/runtime/Cpp/README.md index 73fab634f..3a3523c79 100644 --- a/runtime/Cpp/README.md +++ b/runtime/Cpp/README.md @@ -25,30 +25,30 @@ The C++ target has been the work of the following people: ## Project Status -* Building on OS X, Windows, Android and Linux +* Building on macOS, Windows, Android and Linux * No errors and warnings * Library linking -* Some unit tests in the OSX project, for important base classes with almost 100% code coverage. +* Some unit tests in the macOS project, for important base classes with almost 100% code coverage. * All memory allocations checked * Simple command line demo application working on all supported platforms. * All runtime tests pass. ### Build + Usage Notes -The minimum C++ version to compile the ANTLR C++ runtime with is C++11. The supplied projects can built the runtime either as static or dynamic library, as both 32bit and 64bit arch. The OSX project contains a target for iOS and can also be built using cmake (instead of XCode). +The minimum C++ version to compile the ANTLR C++ runtime with is C++11. The supplied projects can built the runtime either as static or dynamic library, as both 32bit and 64bit arch. The macOS project contains a target for iOS and can also be built using cmake (instead of XCode). Include the antlr4-runtime.h umbrella header in your target application to get everything needed to use the library. If you are compiling with cmake, the minimum version required is cmake 2.8. -#### Compiling on Windows with Visual Studio prior to 2017 -Simply open the VS solution (VS 2013+) and build it. +#### Compiling on Windows with Visual Studio using he Visual Studio projects +Simply open the VS project from the runtime folder (VS 2013+) and build it. -#### Compiling on Windows with Visual Studio VS2017 +#### Compiling on Windows using cmake with Visual Studio VS2017 and later Use the "Open Folder" Feature from the File->Open->Folder menu to open the runtime/Cpp directory. It will automatically use the CMake description to open up a Visual Studio Solution. -#### Compiling on OSX +#### Compiling on macOS Either open the included XCode project and build that or use the cmake compilation as described for linux. #### Compiling on Android diff --git a/runtime/Cpp/deploy-windows.cmd b/runtime/Cpp/deploy-windows.cmd index bb21af7d9..2c5931836 100644 --- a/runtime/Cpp/deploy-windows.cmd +++ b/runtime/Cpp/deploy-windows.cmd @@ -1,4 +1,7 @@ @echo off +setlocal + +if [%1] == [] goto Usage rem Clean left overs from previous builds if there are any if exist bin rmdir /S /Q runtime\bin @@ -9,11 +12,14 @@ if exist antlr4-cpp-runtime-vs2017.zip erase antlr4-cpp-runtime-vs2017.zip if exist antlr4-cpp-runtime-vs2019.zip erase antlr4-cpp-runtime-vs2019.zip rem Headers -xcopy runtime\src\*.h antlr4-runtime\ /s +echo Copying header files ... +xcopy runtime\src\*.h antlr4-runtime\ /s /q rem Binaries -if exist "C:\Program Files (x86)\Microsoft Visual Studio 15.0\Common7\Tools\VsDevCmd.bat" ( - call "C:\Program Files (x86)\Microsoft Visual Studio 15.0\Common7\Tools\VsDevCmd.bat" +if exist "X:\Program Files (x86)\Microsoft Visual Studio\2017\%1\Common7\Tools\VsDevCmd.bat" ( + echo. + + call "C:\Program Files (x86)\Microsoft Visual Studio\2017\%1\Common7\Tools\VsDevCmd.bat" pushd runtime msbuild antlr4cpp-vs2017.vcxproj /p:configuration="Release DLL" /p:platform=Win32 @@ -32,8 +38,11 @@ if exist "C:\Program Files (x86)\Microsoft Visual Studio 15.0\Common7\Tools\VsDe rem if exist antlr4-cpp-runtime-vs2017.zip copy antlr4-cpp-runtime-vs2017.zip ~/antlr/sites/website-antlr4/download ) -if exist "C:\Program Files (x86)\Microsoft Visual Studio 16.0\Common7\Tools\VsDevCmd.bat" ( - call "C:\Program Files (x86)\Microsoft Visual Studio 16.0\Common7\Tools\VsDevCmd.bat" +set VCTargetsPath=C:\Program Files (x86)\Microsoft Visual Studio\2019\%1\MSBuild\Microsoft\VC\v160\ +if exist "C:\Program Files (x86)\Microsoft Visual Studio\2019\%1\Common7\Tools\VsDevCmd.bat" ( + echo. + + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\%1\Common7\Tools\VsDevCmd.bat" pushd runtime msbuild antlr4cpp-vs2019.vcxproj /p:configuration="Release DLL" /p:platform=Win32 @@ -53,5 +62,18 @@ if exist "C:\Program Files (x86)\Microsoft Visual Studio 16.0\Common7\Tools\VsDe ) rmdir /S /Q antlr4-runtime +echo. +echo === Build done === + +goto end + +:Usage + +echo This script builds Visual Studio 2017 and/or 2019 libraries of the ANTLR4 runtime. +echo You have to specify the name(s) of the VS type you have installed (Community, Professional etc.). +echo. +echo Example: +echo %0 Professional +echo. :end diff --git a/runtime/Cpp/runtime/antlr4cpp-vs2017.vcxproj b/runtime/Cpp/runtime/antlr4cpp-vs2017.vcxproj index 630ee34e9..2c3611c86 100644 --- a/runtime/Cpp/runtime/antlr4cpp-vs2017.vcxproj +++ b/runtime/Cpp/runtime/antlr4cpp-vs2017.vcxproj @@ -123,49 +123,49 @@ true - $(SolutionDir)bin\vs-2015\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)bin\vs-2017\$(PlatformTarget)\$(Configuration)\ $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ antlr4-runtime true - $(SolutionDir)bin\vs-2015\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)bin\vs-2017\$(PlatformTarget)\$(Configuration)\ $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ antlr4-runtime true - $(SolutionDir)bin\vs-2015\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)bin\vs-2017\$(PlatformTarget)\$(Configuration)\ $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ antlr4-runtime true - $(SolutionDir)bin\vs-2015\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)bin\vs-2017\$(PlatformTarget)\$(Configuration)\ $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ antlr4-runtime false - $(SolutionDir)bin\vs-2015\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)bin\vs-2017\$(PlatformTarget)\$(Configuration)\ $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ antlr4-runtime false - $(SolutionDir)bin\vs-2015\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)bin\vs-2017\$(PlatformTarget)\$(Configuration)\ $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ antlr4-runtime false - $(SolutionDir)bin\vs-2015\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)bin\vs-2017\$(PlatformTarget)\$(Configuration)\ $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ antlr4-runtime false - $(SolutionDir)bin\vs-2015\$(PlatformTarget)\$(Configuration)\ + $(SolutionDir)bin\vs-2017\$(PlatformTarget)\$(Configuration)\ $(SolutionDir)obj\$(PlatformTarget)\$(Configuration)\$(ProjectName)\ antlr4-runtime diff --git a/runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj b/runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj index b93d8f641..42a81fc06 100644 --- a/runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj +++ b/runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj @@ -1,5 +1,5 @@  - + Debug Static @@ -586,6 +586,7 @@ + diff --git a/runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj.filters b/runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj.filters index eb3b687be..cc1986923 100644 --- a/runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj.filters +++ b/runtime/Cpp/runtime/antlr4cpp-vs2019.vcxproj.filters @@ -336,6 +336,9 @@ Header Files\misc + + Header Files\misc + Header Files\support @@ -528,15 +531,15 @@ Header Files + + Source Files\support + Header Files\tree Header Files - - Header Files\support - diff --git a/runtime/Cpp/runtime/src/atn/ParserATNSimulator.cpp b/runtime/Cpp/runtime/src/atn/ParserATNSimulator.cpp index 30918ff0d..e0b85f695 100755 --- a/runtime/Cpp/runtime/src/atn/ParserATNSimulator.cpp +++ b/runtime/Cpp/runtime/src/atn/ParserATNSimulator.cpp @@ -1348,6 +1348,8 @@ Parser* ParserATNSimulator::getParser() { return parser; } +#pragma warning (disable:4996) // 'getenv': This function or variable may be unsafe. Consider using _dupenv_s instead. + bool ParserATNSimulator::getLrLoopSetting() { char *var = std::getenv("TURN_OFF_LR_LOOP_ENTRY_BRANCH_OPT"); if (var == nullptr) @@ -1356,6 +1358,8 @@ bool ParserATNSimulator::getLrLoopSetting() { return value == "true" || value == "1"; } +#pragma warning (default:4996) + void ParserATNSimulator::InitializeInstanceFields() { _mode = PredictionMode::LL; _startIndex = 0; From e47fd9d918d897c0b1096f9947bee80f9c6bc225 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Fri, 3 Jan 2020 14:57:04 +0100 Subject: [PATCH 079/365] Brought back a lost comment and fixed a description. --- runtime/Cpp/deploy-windows.cmd | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime/Cpp/deploy-windows.cmd b/runtime/Cpp/deploy-windows.cmd index 2c5931836..8fc22ab5b 100644 --- a/runtime/Cpp/deploy-windows.cmd +++ b/runtime/Cpp/deploy-windows.cmd @@ -16,6 +16,7 @@ echo Copying header files ... xcopy runtime\src\*.h antlr4-runtime\ /s /q rem Binaries +rem VS 2017 disabled by default. Change the X to a C to enable it. if exist "X:\Program Files (x86)\Microsoft Visual Studio\2017\%1\Common7\Tools\VsDevCmd.bat" ( echo. @@ -70,7 +71,8 @@ goto end :Usage echo This script builds Visual Studio 2017 and/or 2019 libraries of the ANTLR4 runtime. -echo You have to specify the name(s) of the VS type you have installed (Community, Professional etc.). +echo You have to specify the type of your VS installation (Community, Professional etc.) to construct +echo the correct build tools path. echo. echo Example: echo %0 Professional From 441fea52b1f66eed7d4c99df80e58ebb12507c60 Mon Sep 17 00:00:00 2001 From: parrt Date: Thu, 16 Jan 2020 12:05:03 -0800 Subject: [PATCH 080/365] update to 4.3 version --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/annotations/pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime-testsuite/processors/pom.xml | 2 +- .../org/antlr/v4/test/runtime/python3/BasePython3Test.java | 4 ++-- runtime/Java/pom.xml | 2 +- scripts/github_release_notes.py | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 4 ++-- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 6e57f5e79..1740d8906 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -8,7 +8,7 @@ org.antlr antlr4-master - 4.7.3-SNAPSHOT + 4.8-SNAPSHOT antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index e636655cf..b88bacaaa 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ org.antlr antlr4-master - 4.7.3-SNAPSHOT + 4.8-SNAPSHOT pom ANTLR 4 diff --git a/runtime-testsuite/annotations/pom.xml b/runtime-testsuite/annotations/pom.xml index 3d39f719f..616b7b827 100644 --- a/runtime-testsuite/annotations/pom.xml +++ b/runtime-testsuite/annotations/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.7.3-SNAPSHOT + 4.8-SNAPSHOT ../../pom.xml antlr4-runtime-test-annotations diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index b9e6bd936..898f20bde 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.7.3-SNAPSHOT + 4.8-SNAPSHOT antlr4-runtime-testsuite ANTLR 4 Runtime Tests (2nd generation) diff --git a/runtime-testsuite/processors/pom.xml b/runtime-testsuite/processors/pom.xml index 24fd9b786..fbba53189 100644 --- a/runtime-testsuite/processors/pom.xml +++ b/runtime-testsuite/processors/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.7.3-SNAPSHOT + 4.8-SNAPSHOT ../../pom.xml antlr4-runtime-test-annotation-processors diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java index f9e037fe8..44b0c1294 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java @@ -19,8 +19,8 @@ public class BasePython3Test extends BasePythonTest { @Override protected String getPythonExecutable() { - return "python3.6"; - } // force 3.6 + return "python3.7"; + } // force 3.7 @Override protected void writeLexerTestFile(String lexerName, boolean showDFA) { diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index 92605583d..845f6a618 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.7.3-SNAPSHOT + 4.8-SNAPSHOT ../../pom.xml antlr4-runtime diff --git a/scripts/github_release_notes.py b/scripts/github_release_notes.py index 10b8cca8c..0d5a0a492 100644 --- a/scripts/github_release_notes.py +++ b/scripts/github_release_notes.py @@ -5,7 +5,7 @@ from github import Github from collections import Counter import sys -TARGETS = ['csharp', 'cpp', 'go', 'java', 'javascript', 'python2', 'python3', 'swift'] +TARGETS = ['csharp', 'cpp', 'go', 'java', 'javascript', 'python2', 'python3', 'swift', 'php'] TOKEN=sys.argv[1] MILESTONE=sys.argv[2] diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index 4ef8bb482..757971d84 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.7.3-SNAPSHOT + 4.8-SNAPSHOT antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index f7ca644dc..8c99c238f 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.7.3-SNAPSHOT + 4.8-SNAPSHOT antlr4 ANTLR 4 Tool @@ -29,7 +29,7 @@ org.antlr ST4 - 4.1 + 4.3 org.abego.treelayout From 74c92f2e64b082107e8371fcf708c96897afd030 Mon Sep 17 00:00:00 2001 From: parrt Date: Thu, 16 Jan 2020 12:07:07 -0800 Subject: [PATCH 081/365] update doc to 4.3 --- doc/releasing-antlr.md | 80 +++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index 8eb1d525d..4e1fb9631 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -9,17 +9,17 @@ Create a pre-release or full release at github; [Example 4.5-rc-1](https://githu Wack any existing tag as mvn will create one and it fails if already there. ``` -$ git tag -d 4.7 -$ git push origin :refs/tags/4.7 -$ git push upstream :refs/tags/4.7 +$ git tag -d 4.8 +$ git push origin :refs/tags/4.8 +$ git push upstream :refs/tags/4.8 ``` ### Create release candidate tag ```bash -$ git tag -a 4.7-rc1 -m 'heading towards 4.7' -$ git push origin 4.7-rc1 -$ git push upstream 4.7-rc1 +$ git tag -a 4.8-rc1 -m 'heading towards 4.8' +$ git push origin 4.8-rc1 +$ git push upstream 4.8-rc1 ``` ## Update submodules @@ -114,7 +114,7 @@ Here is the file template ## Maven deploy snapshot -The goal is to get a snapshot, such as `4.7-SNAPSHOT`, to the staging server: [antlr4 tool](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4) and [antlr4 java runtime](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-runtime). +The goal is to get a snapshot, such as `4.8-SNAPSHOT`, to the staging server: [antlr4 tool](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4) and [antlr4 java runtime](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-runtime). Do this: @@ -122,15 +122,15 @@ Do this: $ mvn deploy -DskipTests ... [INFO] --- maven-deploy-plugin:2.7:deploy (default-deploy) @ antlr4-tool-testsuite --- -Downloading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/maven-metadata.xml -Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/antlr4-tool-testsuite-4.7-20161211.173752-1.jar -Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/antlr4-tool-testsuite-4.7-20161211.173752-1.jar (3 KB at 3.4 KB/sec) -Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/antlr4-tool-testsuite-4.7-20161211.173752-1.pom -Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/antlr4-tool-testsuite-4.7-20161211.173752-1.pom (3 KB at 6.5 KB/sec) +Downloading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.8-SNAPSHOT/maven-metadata.xml +Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.8-SNAPSHOT/antlr4-tool-testsuite-4.8-20161211.173752-1.jar +Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.8-SNAPSHOT/antlr4-tool-testsuite-4.8-20161211.173752-1.jar (3 KB at 3.4 KB/sec) +Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.8-SNAPSHOT/antlr4-tool-testsuite-4.8-20161211.173752-1.pom +Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.8-SNAPSHOT/antlr4-tool-testsuite-4.8-20161211.173752-1.pom (3 KB at 6.5 KB/sec) Downloading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml Downloaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml (371 B at 1.4 KB/sec) -Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/maven-metadata.xml -Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/maven-metadata.xml (774 B at 1.8 KB/sec) +Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.8-SNAPSHOT/maven-metadata.xml +Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.8-SNAPSHOT/maven-metadata.xml (774 B at 1.8 KB/sec) Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml (388 B at 0.9 KB/sec) [INFO] ------------------------------------------------------------------------ @@ -200,18 +200,18 @@ It will start out by asking you the version number: ``` ... -What is the release version for "ANTLR 4"? (org.antlr:antlr4-master) 4.7: : 4.7 -What is the release version for "ANTLR 4 Runtime"? (org.antlr:antlr4-runtime) 4.7: : -What is the release version for "ANTLR 4 Tool"? (org.antlr:antlr4) 4.7: : -What is the release version for "ANTLR 4 Maven plugin"? (org.antlr:antlr4-maven-plugin) 4.7: : -What is the release version for "ANTLR 4 Runtime Test Generator"? (org.antlr:antlr4-runtime-testsuite) 4.7: : -What is the release version for "ANTLR 4 Tool Tests"? (org.antlr:antlr4-tool-testsuite) 4.7: : -What is SCM release tag or label for "ANTLR 4"? (org.antlr:antlr4-master) antlr4-master-4.7: : 4.7 -What is the new development version for "ANTLR 4"? (org.antlr:antlr4-master) 4.7.1-SNAPSHOT: +What is the release version for "ANTLR 4"? (org.antlr:antlr4-master) 4.8: : 4.8 +What is the release version for "ANTLR 4 Runtime"? (org.antlr:antlr4-runtime) 4.8: : +What is the release version for "ANTLR 4 Tool"? (org.antlr:antlr4) 4.8: : +What is the release version for "ANTLR 4 Maven plugin"? (org.antlr:antlr4-maven-plugin) 4.8: : +What is the release version for "ANTLR 4 Runtime Test Generator"? (org.antlr:antlr4-runtime-testsuite) 4.8: : +What is the release version for "ANTLR 4 Tool Tests"? (org.antlr:antlr4-tool-testsuite) 4.8: : +What is SCM release tag or label for "ANTLR 4"? (org.antlr:antlr4-master) antlr4-master-4.8: : 4.8 +What is the new development version for "ANTLR 4"? (org.antlr:antlr4-master) 4.8.1-SNAPSHOT: ... ``` -Maven will go through your pom.xml files to update versions from 4.7-SNAPSHOT to 4.7 for release and then to 4.7.1-SNAPSHOT after release, which is done with: +Maven will go through your pom.xml files to update versions from 4.8-SNAPSHOT to 4.8 for release and then to 4.8.1-SNAPSHOT after release, which is done with: ```bash mvn release:perform -Darguments="-DskipTests" @@ -230,11 +230,11 @@ and on the left click "Staging Repositories". You click the staging repo and clo Copy the jars to antlr.org site and update download/index.html ```bash -cp ~/.m2/repository/org/antlr/antlr4-runtime/4.7/antlr4-runtime-4.7.jar ~/antlr/sites/website-antlr4/download/antlr-runtime-4.7.jar -cp ~/.m2/repository/org/antlr/antlr4/4.7/antlr4-4.7-complete.jar ~/antlr/sites/website-antlr4/download/antlr-4.7-complete.jar +cp ~/.m2/repository/org/antlr/antlr4-runtime/4.8/antlr4-runtime-4.8.jar ~/antlr/sites/website-antlr4/download/antlr-runtime-4.8.jar +cp ~/.m2/repository/org/antlr/antlr4/4.8/antlr4-4.8-complete.jar ~/antlr/sites/website-antlr4/download/antlr-4.8-complete.jar cd ~/antlr/sites/website-antlr4/download -git add antlr-4.7-complete.jar -git add antlr-runtime-4.7.jar +git add antlr-4.8-complete.jar +git add antlr-runtime-4.8.jar ``` Update on site: @@ -246,7 +246,7 @@ Update on site: * scripts/topnav.js ``` -git commit -a -m 'add 4.7 jars' +git commit -a -m 'add 4.8 jars' git push origin gh-pages ``` @@ -256,8 +256,8 @@ git push origin gh-pages ```bash cd runtime/JavaScript/src -zip -r /tmp/antlr-javascript-runtime-4.7.zip antlr4 -cp /tmp/antlr-javascript-runtime-4.7.zip ~/antlr/sites/website-antlr4/download +zip -r /tmp/antlr-javascript-runtime-4.8.zip antlr4 +cp /tmp/antlr-javascript-runtime-4.8.zip ~/antlr/sites/website-antlr4/download # git add, commit, push ``` @@ -273,7 +273,7 @@ Move target to website ```bash pushd ~/antlr/sites/website-antlr4/download -git add antlr-javascript-runtime-4.7.zip +git add antlr-javascript-runtime-4.8.zip git commit -a -m 'update JS runtime' git push origin gh-pages popd @@ -317,7 +317,7 @@ Copyright (C) Microsoft Corporation. All rights reserved. Restore completed in 427.62 ms for C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\Antlr4.Runtime.dotnet.csproj. Antlr4.Runtime.dotnet -> C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\netstandard1.3\Antlr4.Runtime.Standard.dll Antlr4.Runtime.dotnet -> C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\net35\Antlr4.Runtime.Standard.dll - Successfully created package 'C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\Antlr4.Runtime.Standard.4.7.2.nupkg'. + Successfully created package 'C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\Antlr4.Runtime.Standard.4.8.2.nupkg'. ``` **Publishing to NuGet** @@ -393,7 +393,7 @@ On a Mac (with XCode 7+ installed): ```bash cd runtime/Cpp ./deploy-macos.sh -cp antlr4-cpp-runtime-macos.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-macos.zip +cp antlr4-cpp-runtime-macos.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.8-macos.zip ``` On any Mac or Linux machine: @@ -401,7 +401,7 @@ On any Mac or Linux machine: ```bash cd runtime/Cpp ./deploy-source.sh -cp antlr4-cpp-runtime-source.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-source.zip +cp antlr4-cpp-runtime-source.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.8-source.zip ``` On a Windows machine the build scripts checks if VS 2017 and/or VS 2019 are installed and builds binaries for each, if found. This script requires 7z to be installed (http://7-zip.org then do `set PATH=%PATH%;C:\Program Files\7-Zip\` from DOS not powershell). @@ -409,7 +409,7 @@ On a Windows machine the build scripts checks if VS 2017 and/or VS 2019 are inst ```bash cd runtime/Cpp deploy-windows.cmd Community -cp antlr4-cpp-runtime-vs2019.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-vs2019.zip +cp antlr4-cpp-runtime-vs2019.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.8-vs2019.zip ``` Move target to website (**_rename to a specific ANTLR version first if needed_**): @@ -417,9 +417,9 @@ Move target to website (**_rename to a specific ANTLR version first if needed_** ```bash pushd ~/antlr/sites/website-antlr4/download # vi index.html -git add antlr4cpp-runtime-4.7-macos.zip -git add antlr4cpp-runtime-4.7-windows.zip -git add antlr4cpp-runtime-4.7-source.zip +git add antlr4cpp-runtime-4.8-macos.zip +git add antlr4cpp-runtime-4.8-windows.zip +git add antlr4cpp-runtime-4.8-source.zip git commit -a -m 'update C++ runtime' git push origin gh-pages popd @@ -441,9 +441,9 @@ cd ~/antlr/sites/website-antlr4/api git checkout gh-pages git pull origin gh-pages cd Java -jar xvf ~/.m2/repository/org/antlr/antlr4-runtime/4.7/antlr4-runtime-4.7-javadoc.jar +jar xvf ~/.m2/repository/org/antlr/antlr4-runtime/4.8/antlr4-runtime-4.8-javadoc.jar cd ../JavaTool -jar xvf ~/.m2/repository/org/antlr/antlr4/4.7/antlr4-4.7-javadoc.jar +jar xvf ~/.m2/repository/org/antlr/antlr4/4.8/antlr4-4.8-javadoc.jar git commit -a -m 'freshen api doc' git push origin gh-pages ``` From 6ad9646382d3d6fc022f58ca7acb18f1bfb328e7 Mon Sep 17 00:00:00 2001 From: parrt Date: Thu, 16 Jan 2020 12:18:02 -0800 Subject: [PATCH 082/365] update version in code to 4.8 --- runtime/CSharp/README.md | 2 +- .../CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj | 2 +- .../CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs | 6 +++--- .../runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathLexer.cs | 2 +- runtime/Cpp/VERSION | 2 +- runtime/Cpp/cmake/Antlr4Package.md | 2 +- runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake | 2 +- runtime/Cpp/cmake/FindANTLR.cmake | 2 +- runtime/Cpp/cmake/README.md | 2 +- runtime/Cpp/demo/generate.cmd | 2 +- runtime/Cpp/runtime/src/RuntimeMetaData.cpp | 2 +- runtime/Go/antlr/recognizer.go | 2 +- runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java | 2 +- runtime/JavaScript/src/antlr4/Recognizer.js | 2 +- runtime/JavaScript/src/antlr4/package.json | 2 +- runtime/Python2/setup.py | 4 ++-- runtime/Python2/src/antlr4/Recognizer.py | 2 +- runtime/Python2/tests/mocks/TestLexer.py | 4 ++-- runtime/Python3/setup.py | 4 ++-- runtime/Python3/src/antlr4/Recognizer.py | 2 +- runtime/Python3/src/antlr4/xpath/XPath.py | 2 +- runtime/Python3/test/expr/ExprLexer.py | 2 +- runtime/Python3/test/expr/ExprParser.py | 2 +- runtime/Python3/test/mocks/TestLexer.py | 4 ++-- runtime/Python3/test/parser/clexer.py | 2 +- runtime/Python3/test/parser/cparser.py | 2 +- runtime/Swift/Sources/Antlr4/RuntimeMetaData.swift | 2 +- runtime/Swift/Tests/Antlr4Tests/RuntimeMetaDataTests.swift | 1 + .../org/antlr/v4/tool/templates/codegen/Swift/Swift.stg | 2 +- tool/src/org/antlr/v4/codegen/target/CSharpTarget.java | 2 +- tool/src/org/antlr/v4/codegen/target/CppTarget.java | 2 +- tool/src/org/antlr/v4/codegen/target/GoTarget.java | 2 +- tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java | 2 +- tool/src/org/antlr/v4/codegen/target/PHPTarget.java | 2 +- tool/src/org/antlr/v4/codegen/target/Python2Target.java | 2 +- tool/src/org/antlr/v4/codegen/target/Python3Target.java | 2 +- tool/src/org/antlr/v4/codegen/target/SwiftTarget.java | 2 +- 37 files changed, 43 insertions(+), 42 deletions(-) diff --git a/runtime/CSharp/README.md b/runtime/CSharp/README.md index d272a6a06..94ea0440a 100644 --- a/runtime/CSharp/README.md +++ b/runtime/CSharp/README.md @@ -43,7 +43,7 @@ See the docs and the book to learn about writing lexer and parser grammars. ### Step 4: Generate the C# code This can be done either from the cmd line, or by adding a custom pre-build command in your project. -At minimal, the cmd line should look as follows: ``java -jar antlr4-4.7.2.jar -Dlanguage=CSharp grammar.g4`` +At minimal, the cmd line should look as follows: ``java -jar antlr4-4.8.jar -Dlanguage=CSharp grammar.g4`` This will generate the files, which you can then integrate in your project. This is just a quick start. The tool has many useful options to control generation, please refer to its documentation. diff --git a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj index 23cbfa49f..50814fc50 100644 --- a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj +++ b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj @@ -1,7 +1,7 @@  The ANTLR Organization - 4.7.2 + 4.8 en-US netstandard1.3;net35 $(NoWarn);CS1591;CS1574;CS1580 diff --git a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs index 325bf9076..1a8459929 100644 --- a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs +++ b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs @@ -42,8 +42,8 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("4.7.2")] +[assembly: AssemblyVersion("4.8")] #if !COMPACT -[assembly: AssemblyFileVersion("4.7.2")] -[assembly: AssemblyInformationalVersion("4.7.2")] +[assembly: AssemblyFileVersion("4.8")] +[assembly: AssemblyInformationalVersion("4.8")] #endif diff --git a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathLexer.cs b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathLexer.cs index 0fd80c339..abe5a1239 100644 --- a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathLexer.cs +++ b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathLexer.cs @@ -24,7 +24,7 @@ using Antlr4.Runtime.Atn; using Antlr4.Runtime.Misc; using DFA = Antlr4.Runtime.Dfa.DFA; -[System.CodeDom.Compiler.GeneratedCode("ANTLR", "4.7")] +[System.CodeDom.Compiler.GeneratedCode("ANTLR", "4.8")] [System.CLSCompliant(false)] public partial class XPathLexer : Lexer { diff --git a/runtime/Cpp/VERSION b/runtime/Cpp/VERSION index af9764a59..ef216a53f 100644 --- a/runtime/Cpp/VERSION +++ b/runtime/Cpp/VERSION @@ -1 +1 @@ -4.7.2 +4.8 diff --git a/runtime/Cpp/cmake/Antlr4Package.md b/runtime/Cpp/cmake/Antlr4Package.md index 984564203..17a630379 100644 --- a/runtime/Cpp/cmake/Antlr4Package.md +++ b/runtime/Cpp/cmake/Antlr4Package.md @@ -96,7 +96,7 @@ target_link_libraries( Parsertest PRIVATE find_package(antlr4-generator REQUIRED) # Set path to generator - set(ANTLR4_JAR_LOCATION ${PROJECT_SOURCE_DIR}/thirdparty/antlr/antlr-4.7.2-complete.jar) + set(ANTLR4_JAR_LOCATION ${PROJECT_SOURCE_DIR}/thirdparty/antlr/antlr-4.8-complete.jar) # generate lexer antlr4_generate( diff --git a/runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake b/runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake index 3523cf2d3..db8ed6f40 100644 --- a/runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake +++ b/runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake @@ -38,7 +38,7 @@ else() set(ANTLR4_SHARED_LIBRARIES ${ANTLR4_OUTPUT_DIR}/libantlr4-runtime.dll.a) set(ANTLR4_RUNTIME_LIBRARIES - ${ANTLR4_OUTPUT_DIR}/cygantlr4-runtime-4.7.2.dll) + ${ANTLR4_OUTPUT_DIR}/cygantlr4-runtime-4.8.dll) elseif(APPLE) set(ANTLR4_RUNTIME_LIBRARIES ${ANTLR4_OUTPUT_DIR}/libantlr4-runtime.dylib) diff --git a/runtime/Cpp/cmake/FindANTLR.cmake b/runtime/Cpp/cmake/FindANTLR.cmake index 511064173..5ff866fc2 100644 --- a/runtime/Cpp/cmake/FindANTLR.cmake +++ b/runtime/Cpp/cmake/FindANTLR.cmake @@ -2,7 +2,7 @@ find_package(Java QUIET COMPONENTS Runtime) if(NOT ANTLR_EXECUTABLE) find_program(ANTLR_EXECUTABLE - NAMES antlr.jar antlr4.jar antlr-4.jar antlr-4.7.2-complete.jar) + NAMES antlr.jar antlr4.jar antlr-4.jar antlr-4.8-complete.jar) endif() if(ANTLR_EXECUTABLE AND Java_JAVA_EXECUTABLE) diff --git a/runtime/Cpp/cmake/README.md b/runtime/Cpp/cmake/README.md index 1db315321..77e9da6c6 100644 --- a/runtime/Cpp/cmake/README.md +++ b/runtime/Cpp/cmake/README.md @@ -31,7 +31,7 @@ include_directories(${ANTLR4_INCLUDE_DIRS}) # set variable pointing to the antlr tool that supports C++ # this is not required if the jar file can be found under PATH environment -set(ANTLR_EXECUTABLE /home/user/antlr-4.7.2-complete.jar) +set(ANTLR_EXECUTABLE /home/user/antlr-4.8-complete.jar) # add macros to generate ANTLR Cpp code from grammar find_package(ANTLR REQUIRED) diff --git a/runtime/Cpp/demo/generate.cmd b/runtime/Cpp/demo/generate.cmd index 826c1f614..a7130c2b1 100644 --- a/runtime/Cpp/demo/generate.cmd +++ b/runtime/Cpp/demo/generate.cmd @@ -6,7 +6,7 @@ :: Download the ANLTR jar and place it in the same folder as this script (or adjust the LOCATION var accordingly). -set LOCATION=antlr-4.7.2-complete.jar +set LOCATION=antlr-4.8-complete.jar java -jar %LOCATION% -Dlanguage=Cpp -listener -visitor -o generated/ -package antlrcpptest TLexer.g4 TParser.g4 ::java -jar %LOCATION% -Dlanguage=Cpp -listener -visitor -o generated/ -package antlrcpptest -XdbgST TLexer.g4 TParser.g4 ::java -jar %LOCATION% -Dlanguage=Java -listener -visitor -o generated/ -package antlrcpptest TLexer.g4 TParser.g4 diff --git a/runtime/Cpp/runtime/src/RuntimeMetaData.cpp b/runtime/Cpp/runtime/src/RuntimeMetaData.cpp index 011f12a51..3ba799806 100755 --- a/runtime/Cpp/runtime/src/RuntimeMetaData.cpp +++ b/runtime/Cpp/runtime/src/RuntimeMetaData.cpp @@ -7,7 +7,7 @@ using namespace antlr4; -const std::string RuntimeMetaData::VERSION = "4.7.2"; +const std::string RuntimeMetaData::VERSION = "4.8"; std::string RuntimeMetaData::getRuntimeVersion() { return VERSION; diff --git a/runtime/Go/antlr/recognizer.go b/runtime/Go/antlr/recognizer.go index 9ea9f6f59..d114800f4 100644 --- a/runtime/Go/antlr/recognizer.go +++ b/runtime/Go/antlr/recognizer.go @@ -49,7 +49,7 @@ var tokenTypeMapCache = make(map[string]int) var ruleIndexMapCache = make(map[string]int) func (b *BaseRecognizer) checkVersion(toolVersion string) { - runtimeVersion := "4.7.2" + runtimeVersion := "4.8" if runtimeVersion != toolVersion { fmt.Println("ANTLR runtime and generated code versions disagree: " + runtimeVersion + "!=" + toolVersion) } diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java b/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java index 05e81441b..4b38bded8 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java @@ -67,7 +67,7 @@ public class RuntimeMetaData { * omitted. * */ - public static final String VERSION = "4.7.2"; + public static final String VERSION = "4.8"; /** * Gets the currently executing version of the ANTLR 4 runtime library. diff --git a/runtime/JavaScript/src/antlr4/Recognizer.js b/runtime/JavaScript/src/antlr4/Recognizer.js index bc2392529..fa787b10a 100644 --- a/runtime/JavaScript/src/antlr4/Recognizer.js +++ b/runtime/JavaScript/src/antlr4/Recognizer.js @@ -21,7 +21,7 @@ Recognizer.ruleIndexMapCache = {}; Recognizer.prototype.checkVersion = function(toolVersion) { - var runtimeVersion = "4.7.2"; + var runtimeVersion = "4.8"; if (runtimeVersion!==toolVersion) { console.log("ANTLR runtime and generated code versions disagree: "+runtimeVersion+"!="+toolVersion); } diff --git a/runtime/JavaScript/src/antlr4/package.json b/runtime/JavaScript/src/antlr4/package.json index d8b6fe3d4..67ebf6526 100644 --- a/runtime/JavaScript/src/antlr4/package.json +++ b/runtime/JavaScript/src/antlr4/package.json @@ -1,6 +1,6 @@ { "name": "antlr4", - "version": "4.7.2", + "version": "4.8", "description": "JavaScript runtime for ANTLR4", "main": "index.js", "repository": "antlr/antlr4.git", diff --git a/runtime/Python2/setup.py b/runtime/Python2/setup.py index 78491005e..ab6a41dc4 100644 --- a/runtime/Python2/setup.py +++ b/runtime/Python2/setup.py @@ -2,12 +2,12 @@ from setuptools import setup setup( name='antlr4-python2-runtime', - version='4.7.2', + version='4.8', url='http://www.antlr.org', license='BSD', packages=['antlr4', 'antlr4.atn', 'antlr4.dfa', 'antlr4.tree', 'antlr4.error', 'antlr4.xpath'], package_dir={'': 'src'}, author='Eric Vergnaud, Terence Parr, Sam Harwell', author_email='eric.vergnaud@wanadoo.fr', - description='ANTLR 4.7.2 runtime for Python 2.7.12' + description='ANTLR 4.8 runtime for Python 2.7.12' ) diff --git a/runtime/Python2/src/antlr4/Recognizer.py b/runtime/Python2/src/antlr4/Recognizer.py index 23693a915..bae837c99 100644 --- a/runtime/Python2/src/antlr4/Recognizer.py +++ b/runtime/Python2/src/antlr4/Recognizer.py @@ -30,7 +30,7 @@ class Recognizer(object): return major, minor def checkVersion(self, toolVersion): - runtimeVersion = "4.7.2" + runtimeVersion = "4.8" rvmajor, rvminor = self.extractVersion(runtimeVersion) tvmajor, tvminor = self.extractVersion(toolVersion) if rvmajor!=tvmajor or rvminor!=tvminor: diff --git a/runtime/Python2/tests/mocks/TestLexer.py b/runtime/Python2/tests/mocks/TestLexer.py index 9c54007b2..63147241a 100644 --- a/runtime/Python2/tests/mocks/TestLexer.py +++ b/runtime/Python2/tests/mocks/TestLexer.py @@ -39,7 +39,7 @@ class TestLexer(Lexer): def __init__(self, input=None): super(TestLexer, self).__init__(input) - self.checkVersion("4.7.2") + self.checkVersion("4.8") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None self._predicates = None @@ -95,7 +95,7 @@ class TestLexer2(Lexer): def __init__(self, input=None): super(TestLexer2, self).__init__(input) - self.checkVersion("4.7.2") + self.checkVersion("4.8") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None self._predicates = None diff --git a/runtime/Python3/setup.py b/runtime/Python3/setup.py index 5753df0cb..b8f42f169 100644 --- a/runtime/Python3/setup.py +++ b/runtime/Python3/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup( name='antlr4-python3-runtime', - version='4.7.2', + version='4.8', packages=['antlr4', 'antlr4.atn', 'antlr4.dfa', 'antlr4.tree', 'antlr4.error', 'antlr4.xpath'], package_dir={'': 'src'}, install_requires=[ @@ -12,5 +12,5 @@ setup( license='BSD', author='Eric Vergnaud, Terence Parr, Sam Harwell', author_email='eric.vergnaud@wanadoo.fr', - description='ANTLR 4.7.2 runtime for Python 3.6.3' + description='ANTLR 4.8 runtime for Python 3.7' ) diff --git a/runtime/Python3/src/antlr4/Recognizer.py b/runtime/Python3/src/antlr4/Recognizer.py index 01017351f..ca9ac49e4 100644 --- a/runtime/Python3/src/antlr4/Recognizer.py +++ b/runtime/Python3/src/antlr4/Recognizer.py @@ -33,7 +33,7 @@ class Recognizer(object): return major, minor def checkVersion(self, toolVersion): - runtimeVersion = "4.7.2" + runtimeVersion = "4.8" rvmajor, rvminor = self.extractVersion(runtimeVersion) tvmajor, tvminor = self.extractVersion(toolVersion) if rvmajor!=tvmajor or rvminor!=tvminor: diff --git a/runtime/Python3/src/antlr4/xpath/XPath.py b/runtime/Python3/src/antlr4/xpath/XPath.py index 58b05c466..3ac7d0c82 100644 --- a/runtime/Python3/src/antlr4/xpath/XPath.py +++ b/runtime/Python3/src/antlr4/xpath/XPath.py @@ -119,7 +119,7 @@ class XPathLexer(Lexer): def __init__(self, input=None): super().__init__(input) - self.checkVersion("4.7.2") + self.checkVersion("4.8") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None self._predicates = None diff --git a/runtime/Python3/test/expr/ExprLexer.py b/runtime/Python3/test/expr/ExprLexer.py index e338b0b9e..67df69b89 100644 --- a/runtime/Python3/test/expr/ExprLexer.py +++ b/runtime/Python3/test/expr/ExprLexer.py @@ -86,7 +86,7 @@ class ExprLexer(Lexer): def __init__(self, input=None, output:TextIO = sys.stdout): super().__init__(input, output) - self.checkVersion("4.7.2") + self.checkVersion("4.8") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None self._predicates = None diff --git a/runtime/Python3/test/expr/ExprParser.py b/runtime/Python3/test/expr/ExprParser.py index 598c778d1..a642f189e 100644 --- a/runtime/Python3/test/expr/ExprParser.py +++ b/runtime/Python3/test/expr/ExprParser.py @@ -86,7 +86,7 @@ class ExprParser ( Parser ): def __init__(self, input:TokenStream, output:TextIO = sys.stdout): super().__init__(input, output) - self.checkVersion("4.7.2") + self.checkVersion("4.8") self._interp = ParserATNSimulator(self, self.atn, self.decisionsToDFA, self.sharedContextCache) self._predicates = None diff --git a/runtime/Python3/test/mocks/TestLexer.py b/runtime/Python3/test/mocks/TestLexer.py index 9c54007b2..63147241a 100644 --- a/runtime/Python3/test/mocks/TestLexer.py +++ b/runtime/Python3/test/mocks/TestLexer.py @@ -39,7 +39,7 @@ class TestLexer(Lexer): def __init__(self, input=None): super(TestLexer, self).__init__(input) - self.checkVersion("4.7.2") + self.checkVersion("4.8") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None self._predicates = None @@ -95,7 +95,7 @@ class TestLexer2(Lexer): def __init__(self, input=None): super(TestLexer2, self).__init__(input) - self.checkVersion("4.7.2") + self.checkVersion("4.8") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None self._predicates = None diff --git a/runtime/Python3/test/parser/clexer.py b/runtime/Python3/test/parser/clexer.py index c35320593..345572c98 100644 --- a/runtime/Python3/test/parser/clexer.py +++ b/runtime/Python3/test/parser/clexer.py @@ -792,7 +792,7 @@ class CLexer(Lexer): def __init__(self, input=None): super().__init__(input) - self.checkVersion("4.7.2") + self.checkVersion("4.8") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None self._predicates = None diff --git a/runtime/Python3/test/parser/cparser.py b/runtime/Python3/test/parser/cparser.py index 1eb1d741b..17016be12 100644 --- a/runtime/Python3/test/parser/cparser.py +++ b/runtime/Python3/test/parser/cparser.py @@ -915,7 +915,7 @@ class CParser ( Parser ): def __init__(self, input:TokenStream): super().__init__(input) - self.checkVersion("4.7.2") + self.checkVersion("4.8") self._interp = ParserATNSimulator(self, self.atn, self.decisionsToDFA, self.sharedContextCache) self._predicates = None diff --git a/runtime/Swift/Sources/Antlr4/RuntimeMetaData.swift b/runtime/Swift/Sources/Antlr4/RuntimeMetaData.swift index b5ca8ceda..3ca97f244 100644 --- a/runtime/Swift/Sources/Antlr4/RuntimeMetaData.swift +++ b/runtime/Swift/Sources/Antlr4/RuntimeMetaData.swift @@ -63,7 +63,7 @@ public class RuntimeMetaData { /// omitted, the `-` (hyphen-minus) appearing before it is also /// omitted. /// - public static let VERSION: String = "4.7.2" + public static let VERSION: String = "4.8" /// /// Gets the currently executing version of the ANTLR 4 runtime library. diff --git a/runtime/Swift/Tests/Antlr4Tests/RuntimeMetaDataTests.swift b/runtime/Swift/Tests/Antlr4Tests/RuntimeMetaDataTests.swift index c3845c922..97ea0880a 100644 --- a/runtime/Swift/Tests/Antlr4Tests/RuntimeMetaDataTests.swift +++ b/runtime/Swift/Tests/Antlr4Tests/RuntimeMetaDataTests.swift @@ -15,6 +15,7 @@ class RuntimeMetaDataTests: XCTestCase { doGetMajorMinorVersionTest("4.7", "4.7") doGetMajorMinorVersionTest("4.7.1", "4.7") doGetMajorMinorVersionTest("4.7.2", "4.7") + doGetMajorMinorVersionTest("4.8", "4.8") doGetMajorMinorVersionTest("4-SNAPSHOT", "4") doGetMajorMinorVersionTest("4.-SNAPSHOT", "4.") doGetMajorMinorVersionTest("4.7-SNAPSHOT", "4.7") diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg index cefb1e248..a70d6f2d9 100755 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg @@ -354,7 +354,7 @@ func getVocabulary() -> Vocabulary { override init(_ input:TokenStream) throws { - RuntimeMetaData.checkVersion("4.7.2", RuntimeMetaData.VERSION) + RuntimeMetaData.checkVersion("4.8", RuntimeMetaData.VERSION) try super.init(input) _interp = ParserATNSimulator(self,._ATN,._decisionToDFA, ._sharedContextCache) } diff --git a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java index 82f65c63d..bd8f579f9 100644 --- a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java @@ -28,7 +28,7 @@ public class CSharpTarget extends Target { @Override public String getVersion() { - return "4.7.2"; + return "4.8"; } @Override diff --git a/tool/src/org/antlr/v4/codegen/target/CppTarget.java b/tool/src/org/antlr/v4/codegen/target/CppTarget.java index 839a362c1..8b5d7ac04 100644 --- a/tool/src/org/antlr/v4/codegen/target/CppTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/CppTarget.java @@ -50,7 +50,7 @@ public class CppTarget extends Target { } public String getVersion() { - return "4.7.2"; + return "4.8"; } public boolean needsHeader() { return true; } diff --git a/tool/src/org/antlr/v4/codegen/target/GoTarget.java b/tool/src/org/antlr/v4/codegen/target/GoTarget.java index 89a40de99..9b20b6ef2 100644 --- a/tool/src/org/antlr/v4/codegen/target/GoTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/GoTarget.java @@ -71,7 +71,7 @@ public class GoTarget extends Target { @Override public String getVersion() { - return "4.7.2"; + return "4.8"; } public Set getBadWords() { diff --git a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java index 75886fd0d..9d9cdbed9 100644 --- a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java @@ -51,7 +51,7 @@ public class JavaScriptTarget extends Target { @Override public String getVersion() { - return "4.7.2"; + return "4.8"; } public Set getBadWords() { diff --git a/tool/src/org/antlr/v4/codegen/target/PHPTarget.java b/tool/src/org/antlr/v4/codegen/target/PHPTarget.java index 2617565fb..686e10e9c 100644 --- a/tool/src/org/antlr/v4/codegen/target/PHPTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/PHPTarget.java @@ -54,7 +54,7 @@ public class PHPTarget extends Target { @Override public String getVersion() { - return "4.7.2"; + return "4.8"; } @Override diff --git a/tool/src/org/antlr/v4/codegen/target/Python2Target.java b/tool/src/org/antlr/v4/codegen/target/Python2Target.java index 1beaf8c20..582e0f192 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python2Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python2Target.java @@ -95,7 +95,7 @@ public class Python2Target extends Target { @Override public String getVersion() { - return "4.7.2"; + return "4.8"; } public Set getBadWords() { diff --git a/tool/src/org/antlr/v4/codegen/target/Python3Target.java b/tool/src/org/antlr/v4/codegen/target/Python3Target.java index 753f67cdf..b386e4d4b 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python3Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python3Target.java @@ -97,7 +97,7 @@ public class Python3Target extends Target { @Override public String getVersion() { - return "4.7.2"; + return "4.8"; } /** Avoid grammar symbols in this set to prevent conflicts in gen'd code. */ diff --git a/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java b/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java index b19234439..2e78f8487 100644 --- a/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java @@ -87,7 +87,7 @@ public class SwiftTarget extends Target { @Override public String getVersion() { - return "4.7.2"; // Java and tool versions move in lock step + return "4.8"; // Java and tool versions move in lock step } public Set getBadWords() { From d569f917954406e599d780c009224aa14f1920de Mon Sep 17 00:00:00 2001 From: parrt Date: Thu, 16 Jan 2020 12:24:22 -0800 Subject: [PATCH 083/365] [maven-release-plugin] prepare release 4.8 --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/annotations/pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime-testsuite/processors/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 1740d8906..d218e2714 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -8,7 +8,7 @@ org.antlr antlr4-master - 4.8-SNAPSHOT + 4.8 antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index b88bacaaa..82bfe12cf 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ org.antlr antlr4-master - 4.8-SNAPSHOT + 4.8 pom ANTLR 4 diff --git a/runtime-testsuite/annotations/pom.xml b/runtime-testsuite/annotations/pom.xml index 616b7b827..b13b23f35 100644 --- a/runtime-testsuite/annotations/pom.xml +++ b/runtime-testsuite/annotations/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8-SNAPSHOT + 4.8 ../../pom.xml antlr4-runtime-test-annotations diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 898f20bde..5b07d3f09 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.8-SNAPSHOT + 4.8 antlr4-runtime-testsuite ANTLR 4 Runtime Tests (2nd generation) diff --git a/runtime-testsuite/processors/pom.xml b/runtime-testsuite/processors/pom.xml index fbba53189..00a1da210 100644 --- a/runtime-testsuite/processors/pom.xml +++ b/runtime-testsuite/processors/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8-SNAPSHOT + 4.8 ../../pom.xml antlr4-runtime-test-annotation-processors diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index 845f6a618..9489eacb9 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8-SNAPSHOT + 4.8 ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index 757971d84..9cbc512f0 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.8-SNAPSHOT + 4.8 antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index 8c99c238f..76325efac 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8-SNAPSHOT + 4.8 antlr4 ANTLR 4 Tool From 666131841c83c250abd5d4d95763de2cee583f2c Mon Sep 17 00:00:00 2001 From: parrt Date: Thu, 16 Jan 2020 12:24:30 -0800 Subject: [PATCH 084/365] [maven-release-plugin] prepare for next development iteration --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/annotations/pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime-testsuite/processors/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index d218e2714..55f064c3d 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -8,7 +8,7 @@ org.antlr antlr4-master - 4.8 + 4.9-SNAPSHOT antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index 82bfe12cf..c2bf608f7 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ org.antlr antlr4-master - 4.8 + 4.9-SNAPSHOT pom ANTLR 4 diff --git a/runtime-testsuite/annotations/pom.xml b/runtime-testsuite/annotations/pom.xml index b13b23f35..eaef576f3 100644 --- a/runtime-testsuite/annotations/pom.xml +++ b/runtime-testsuite/annotations/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8 + 4.9-SNAPSHOT ../../pom.xml antlr4-runtime-test-annotations diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 5b07d3f09..4636b54a8 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.8 + 4.9-SNAPSHOT antlr4-runtime-testsuite ANTLR 4 Runtime Tests (2nd generation) diff --git a/runtime-testsuite/processors/pom.xml b/runtime-testsuite/processors/pom.xml index 00a1da210..234cf66fd 100644 --- a/runtime-testsuite/processors/pom.xml +++ b/runtime-testsuite/processors/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8 + 4.9-SNAPSHOT ../../pom.xml antlr4-runtime-test-annotation-processors diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index 9489eacb9..08ae511a5 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8 + 4.9-SNAPSHOT ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index 9cbc512f0..30a493b9c 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.8 + 4.9-SNAPSHOT antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index 76325efac..359ac1215 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8 + 4.9-SNAPSHOT antlr4 ANTLR 4 Tool From 42512526f97181f00c96debbf6934cb22a4ee10c Mon Sep 17 00:00:00 2001 From: parrt Date: Thu, 16 Jan 2020 12:39:52 -0800 Subject: [PATCH 085/365] update javascript target to 4.3 --- runtime/JavaScript/src/antlr4/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/JavaScript/src/antlr4/package.json b/runtime/JavaScript/src/antlr4/package.json index 67ebf6526..6536f2925 100644 --- a/runtime/JavaScript/src/antlr4/package.json +++ b/runtime/JavaScript/src/antlr4/package.json @@ -1,6 +1,6 @@ { "name": "antlr4", - "version": "4.8", + "version": "4.8.0", "description": "JavaScript runtime for ANTLR4", "main": "index.js", "repository": "antlr/antlr4.git", From 68fd66dd4973891bb125aab39078091fe4f0b02a Mon Sep 17 00:00:00 2001 From: parrt Date: Thu, 16 Jan 2020 15:46:58 -0800 Subject: [PATCH 086/365] add doc for releasing --- doc/releasing-antlr.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index 4e1fb9631..c65ab4d9e 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -24,6 +24,13 @@ $ git push upstream 4.8-rc1 ## Update submodules +Make sure you tell git to pull in the submodule (for every clone you do of antlr4): + +```bash +git submodule init +git submodule update +``` + Update the runtime submodules by running the following command: ```bash From 6c4b491d1bc2a69cb1210717f5290c57dfb2a717 Mon Sep 17 00:00:00 2001 From: parrt Date: Sat, 18 Jan 2020 09:39:46 -0800 Subject: [PATCH 087/365] set version to 4.8-1 to fix release --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/annotations/pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime-testsuite/processors/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 55f064c3d..00f7b545b 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -8,7 +8,7 @@ org.antlr antlr4-master - 4.9-SNAPSHOT + 4.8-1-SNAPSHOT antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index c2bf608f7..fad59f352 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ org.antlr antlr4-master - 4.9-SNAPSHOT + 4.8-1-SNAPSHOT pom ANTLR 4 diff --git a/runtime-testsuite/annotations/pom.xml b/runtime-testsuite/annotations/pom.xml index eaef576f3..97f485da4 100644 --- a/runtime-testsuite/annotations/pom.xml +++ b/runtime-testsuite/annotations/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.9-SNAPSHOT + 4.8-1-SNAPSHOT ../../pom.xml antlr4-runtime-test-annotations diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 4636b54a8..c93f0284d 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.9-SNAPSHOT + 4.8-1-SNAPSHOT antlr4-runtime-testsuite ANTLR 4 Runtime Tests (2nd generation) diff --git a/runtime-testsuite/processors/pom.xml b/runtime-testsuite/processors/pom.xml index 234cf66fd..6040ddb82 100644 --- a/runtime-testsuite/processors/pom.xml +++ b/runtime-testsuite/processors/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.9-SNAPSHOT + 4.8-1-SNAPSHOT ../../pom.xml antlr4-runtime-test-annotation-processors diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index 08ae511a5..3ad47a5eb 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.9-SNAPSHOT + 4.8-1-SNAPSHOT ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index 30a493b9c..bdc5197cc 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.9-SNAPSHOT + 4.8-1-SNAPSHOT antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index 359ac1215..7d2d4036a 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.9-SNAPSHOT + 4.8-1-SNAPSHOT antlr4 ANTLR 4 Tool From a872114f6165fb2c731f22cbb97fd4bc3bf8b87b Mon Sep 17 00:00:00 2001 From: parrt Date: Sat, 18 Jan 2020 10:07:26 -0800 Subject: [PATCH 088/365] trying to get 4.8-1 to fix release --- runtime-testsuite/pom.xml | 4 ++-- tool-testsuite/pom.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index c93f0284d..de396f619 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.8-1-SNAPSHOT + 4.8-SNAPSHOT antlr4-runtime-testsuite ANTLR 4 Runtime Tests (2nd generation) @@ -26,7 +26,7 @@ org.antlr ST4 - 4.1 + 4.3 test diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index bdc5197cc..9b29c824d 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.8-1-SNAPSHOT + 4.8-SNAPSHOT antlr4-tool-testsuite ANTLR 4 Tool Tests @@ -26,7 +26,7 @@ org.antlr ST4 - 4.1 + 4.3 test From f9cb658daab2c90a2b53a2e6a79e46cb9db8d364 Mon Sep 17 00:00:00 2001 From: parrt Date: Sat, 18 Jan 2020 10:20:39 -0800 Subject: [PATCH 089/365] Update PHP Runtime to latest version --- runtime/PHP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/PHP b/runtime/PHP index 9e1b759d0..62779f9c3 160000 --- a/runtime/PHP +++ b/runtime/PHP @@ -1 +1 @@ -Subproject commit 9e1b759d02220eedf477771169bdfb6a186a2984 +Subproject commit 62779f9c3457c57b3328c4b9608dfdd9968b5e61 From 54cb3f20183e8ddf8d0a006fe4963a62c5b99a76 Mon Sep 17 00:00:00 2001 From: parrt Date: Sat, 18 Jan 2020 10:34:27 -0800 Subject: [PATCH 090/365] trying to get 4.8-1 to fix release (all say 4.8-1-SNAPSHOT now) --- doc/releasing-antlr.md | 5 ++++- runtime-testsuite/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index c65ab4d9e..a7ff343db 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -28,7 +28,6 @@ Make sure you tell git to pull in the submodule (for every clone you do of antlr ```bash git submodule init -git submodule update ``` Update the runtime submodules by running the following command: @@ -74,6 +73,10 @@ find tool runtime -type f -exec grep -l '4\.6' {} \; Commit to repository. +## Building + +ugh. apparently you have to `mvn install` and then `mvn compile` or some such or subdir pom.xml's won't see the latest runtime build. + ## Maven Repository Settings First, make sure you have maven set up to communicate with staging servers etc... Create file `~/.m2/settings.xml` with appropriate username/password for staging server and gpg.keyname/passphrase for signing. Make sure it has strict visibility privileges to just you. On unix, it looks like: diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index de396f619..f2c89055c 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.8-SNAPSHOT + 4.8-1-SNAPSHOT antlr4-runtime-testsuite ANTLR 4 Runtime Tests (2nd generation) diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index 9b29c824d..2e3a0bd8e 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.8-SNAPSHOT + 4.8-1-SNAPSHOT antlr4-tool-testsuite ANTLR 4 Tool Tests From fa499e421528300fac54c795078dacae5ef5fe69 Mon Sep 17 00:00:00 2001 From: parrt Date: Sat, 18 Jan 2020 11:09:23 -0800 Subject: [PATCH 091/365] update travis to python 3.7 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cd774ceae..cae5dae3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -192,7 +192,7 @@ matrix: sources: - deadsnakes # source required so it finds the package definition below packages: - - python3.6 + - python3.7 stage: main-test - os: linux dist: trusty From 3f41f283d123731380ad549d0e3e89317d6bc3ae Mon Sep 17 00:00:00 2001 From: parrt Date: Sat, 18 Jan 2020 13:37:01 -0800 Subject: [PATCH 092/365] [maven-release-plugin] prepare release antlr4-master-4.8-1 --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/annotations/pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime-testsuite/processors/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 00f7b545b..4f954b0c2 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -8,7 +8,7 @@ org.antlr antlr4-master - 4.8-1-SNAPSHOT + 4.8-1 antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index fad59f352..9ab21d543 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ org.antlr antlr4-master - 4.8-1-SNAPSHOT + 4.8-1 pom ANTLR 4 diff --git a/runtime-testsuite/annotations/pom.xml b/runtime-testsuite/annotations/pom.xml index 97f485da4..75396a6b7 100644 --- a/runtime-testsuite/annotations/pom.xml +++ b/runtime-testsuite/annotations/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8-1-SNAPSHOT + 4.8-1 ../../pom.xml antlr4-runtime-test-annotations diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index f2c89055c..341e051f6 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.8-1-SNAPSHOT + 4.8-1 antlr4-runtime-testsuite ANTLR 4 Runtime Tests (2nd generation) diff --git a/runtime-testsuite/processors/pom.xml b/runtime-testsuite/processors/pom.xml index 6040ddb82..97aa4bc06 100644 --- a/runtime-testsuite/processors/pom.xml +++ b/runtime-testsuite/processors/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8-1-SNAPSHOT + 4.8-1 ../../pom.xml antlr4-runtime-test-annotation-processors diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index 3ad47a5eb..7743bf9d7 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8-1-SNAPSHOT + 4.8-1 ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index 2e3a0bd8e..a3513612f 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.8-1-SNAPSHOT + 4.8-1 antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index 7d2d4036a..0474a2aac 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8-1-SNAPSHOT + 4.8-1 antlr4 ANTLR 4 Tool From f1ffc017232455eb14f90b9d8cf13d2723fb9407 Mon Sep 17 00:00:00 2001 From: parrt Date: Sat, 18 Jan 2020 13:37:08 -0800 Subject: [PATCH 093/365] [maven-release-plugin] prepare for next development iteration --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/annotations/pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime-testsuite/processors/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 4f954b0c2..76e9bd37f 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -8,7 +8,7 @@ org.antlr antlr4-master - 4.8-1 + 4.8-2-SNAPSHOT antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index 9ab21d543..5286a1a8c 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ org.antlr antlr4-master - 4.8-1 + 4.8-2-SNAPSHOT pom ANTLR 4 diff --git a/runtime-testsuite/annotations/pom.xml b/runtime-testsuite/annotations/pom.xml index 75396a6b7..97bf78649 100644 --- a/runtime-testsuite/annotations/pom.xml +++ b/runtime-testsuite/annotations/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8-1 + 4.8-2-SNAPSHOT ../../pom.xml antlr4-runtime-test-annotations diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 341e051f6..33ea5e9f1 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.8-1 + 4.8-2-SNAPSHOT antlr4-runtime-testsuite ANTLR 4 Runtime Tests (2nd generation) diff --git a/runtime-testsuite/processors/pom.xml b/runtime-testsuite/processors/pom.xml index 97aa4bc06..c417c626d 100644 --- a/runtime-testsuite/processors/pom.xml +++ b/runtime-testsuite/processors/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8-1 + 4.8-2-SNAPSHOT ../../pom.xml antlr4-runtime-test-annotation-processors diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index 7743bf9d7..ce1e2063a 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8-1 + 4.8-2-SNAPSHOT ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index a3513612f..0b8bf4d66 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -10,7 +10,7 @@ org.antlr antlr4-master - 4.8-1 + 4.8-2-SNAPSHOT antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index 0474a2aac..11edf014d 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -9,7 +9,7 @@ org.antlr antlr4-master - 4.8-1 + 4.8-2-SNAPSHOT antlr4 ANTLR 4 Tool From 26ab547f1c9f803e32e608ed27fb94e8f89ddac7 Mon Sep 17 00:00:00 2001 From: parrt Date: Sat, 18 Jan 2020 13:58:23 -0800 Subject: [PATCH 094/365] bump dev version --- doc/releasing-antlr.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index a7ff343db..d028b68a8 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -235,7 +235,7 @@ Now, go here: and on the left click "Staging Repositories". You click the staging repo and close it, then you refresh, click it and release it. It's done when you see it here: -    [http://repo1.maven.org/maven2/org/antlr/antlr4-runtime/](http://repo1.maven.org/maven2/org/antlr/antlr4-runtime/) +    [https://oss.sonatype.org/service/local/repositories/releases/content/org/antlr/antlr4-runtime/4.8-1/antlr4-runtime-4.8-1.jar](https://oss.sonatype.org/service/local/repositories/releases/content/org/antlr/antlr4-runtime/4.8-1/antlr4-runtime-4.8-1.jar) Copy the jars to antlr.org site and update download/index.html From fc2e6636d6821aff13bc6617b501b05790a8159a Mon Sep 17 00:00:00 2001 From: parrt Date: Sat, 18 Jan 2020 14:00:52 -0800 Subject: [PATCH 095/365] tweak doc --- doc/releasing-antlr.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index d028b68a8..10a0f36a5 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -237,6 +237,8 @@ and on the left click "Staging Repositories". You click the staging repo and clo     [https://oss.sonatype.org/service/local/repositories/releases/content/org/antlr/antlr4-runtime/4.8-1/antlr4-runtime-4.8-1.jar](https://oss.sonatype.org/service/local/repositories/releases/content/org/antlr/antlr4-runtime/4.8-1/antlr4-runtime-4.8-1.jar) +All releases should be here: https://repo1.maven.org/maven2/org/antlr/antlr4-runtime/ + Copy the jars to antlr.org site and update download/index.html ```bash From 46bd9e55690e5607d5cbe94df988d97c384ed4f3 Mon Sep 17 00:00:00 2001 From: Larry Li Date: Sun, 19 Jan 2020 18:22:06 +1100 Subject: [PATCH 096/365] add dart target --- .travis.yml | 4 + .travis/before-install-linux-dart.sh | 10 + .travis/run-tests-dart.sh | 4 + contributors.txt | 3 +- runtime-testsuite/pom.xml | 1 + .../v4/test/runtime/templates/Dart.test.stg | 318 ++ .../v4/test/runtime/BaseRuntimeTest.java | 13 +- .../v4/test/runtime/dart/BaseDartTest.java | 1149 +++++++ .../runtime/dart/TestCompositeLexers.java | 26 + .../runtime/dart/TestCompositeParsers.java | 26 + .../runtime/dart/TestFullContextParsing.java | 26 + .../test/runtime/dart/TestLeftRecursion.java | 26 + .../v4/test/runtime/dart/TestLexerErrors.java | 26 + .../v4/test/runtime/dart/TestLexerExec.java | 26 + .../v4/test/runtime/dart/TestListeners.java | 26 + .../v4/test/runtime/dart/TestParseTrees.java | 26 + .../test/runtime/dart/TestParserErrors.java | 26 + .../v4/test/runtime/dart/TestParserExec.java | 26 + .../v4/test/runtime/dart/TestPerformance.java | 26 + .../runtime/dart/TestSemPredEvalLexer.java | 26 + .../runtime/dart/TestSemPredEvalParser.java | 26 + .../antlr/v4/test/runtime/dart/TestSets.java | 26 + .../descriptors/PerformanceDescriptors.java | 2 +- runtime/Dart/.gitignore | 23 + runtime/Dart/LICENSE.txt | 52 + runtime/Dart/README.md | 11 + runtime/Dart/lib/antlr4.dart | 21 + runtime/Dart/lib/src/atn/atn.dart | 18 + runtime/Dart/lib/src/atn/src/atn.dart | 189 ++ runtime/Dart/lib/src/atn/src/atn_config.dart | 248 ++ .../Dart/lib/src/atn/src/atn_config_set.dart | 281 ++ .../lib/src/atn/src/atn_deserializer.dart | 823 +++++ .../Dart/lib/src/atn/src/atn_simulator.dart | 101 + runtime/Dart/lib/src/atn/src/atn_state.dart | 298 ++ runtime/Dart/lib/src/atn/src/atn_type.dart | 18 + runtime/Dart/lib/src/atn/src/info.dart | 666 ++++ .../Dart/lib/src/atn/src/lexer_action.dart | 678 +++++ .../src/atn/src/lexer_action_executor.dart | 181 ++ .../lib/src/atn/src/lexer_atn_simulator.dart | 747 +++++ .../lib/src/atn/src/parser_atn_simulator.dart | 2701 +++++++++++++++++ .../src/atn/src/profiling_atn_simulator.dart | 228 ++ .../lib/src/atn/src/semantic_context.dart | 399 +++ runtime/Dart/lib/src/atn/src/transition.dart | 283 ++ runtime/Dart/lib/src/dfa/dfa.dart | 8 + runtime/Dart/lib/src/dfa/src/dfa.dart | 148 + .../Dart/lib/src/dfa/src/dfa_serializer.dart | 74 + runtime/Dart/lib/src/dfa/src/dfa_state.dart | 156 + runtime/Dart/lib/src/error/error.dart | 10 + .../error/src/diagnostic_error_listener.dart | 121 + .../lib/src/error/src/error_listener.dart | 250 ++ .../lib/src/error/src/error_strategy.dart | 955 ++++++ runtime/Dart/lib/src/error/src/errors.dart | 219 ++ runtime/Dart/lib/src/input_stream.dart | 350 +++ runtime/Dart/lib/src/interval_set.dart | 750 +++++ runtime/Dart/lib/src/lexer.dart | 342 +++ runtime/Dart/lib/src/ll1_analyzer.dart | 211 ++ runtime/Dart/lib/src/misc/multi_map.dart | 34 + runtime/Dart/lib/src/misc/pair.dart | 26 + runtime/Dart/lib/src/parser.dart | 839 +++++ runtime/Dart/lib/src/parser_interpreter.dart | 396 +++ runtime/Dart/lib/src/parser_rule_context.dart | 288 ++ runtime/Dart/lib/src/prediction_context.dart | 874 ++++++ runtime/Dart/lib/src/recognizer.dart | 197 ++ runtime/Dart/lib/src/rule_context.dart | 185 ++ runtime/Dart/lib/src/runtime_meta_data.dart | 199 ++ runtime/Dart/lib/src/token.dart | 479 +++ runtime/Dart/lib/src/token_factory.dart | 97 + runtime/Dart/lib/src/token_source.dart | 273 ++ runtime/Dart/lib/src/token_stream.dart | 650 ++++ .../Dart/lib/src/tree/src/pattern/chunk.dart | 108 + .../tree/src/pattern/parse_tree_match.dart | 701 +++++ runtime/Dart/lib/src/tree/src/tree.dart | 365 +++ runtime/Dart/lib/src/tree/src/trees.dart | 238 ++ runtime/Dart/lib/src/tree/tree.dart | 10 + runtime/Dart/lib/src/util/bit_set.dart | 301 ++ runtime/Dart/lib/src/util/murmur_hash.dart | 85 + runtime/Dart/lib/src/util/utils.dart | 33 + runtime/Dart/lib/src/vocabulary.dart | 268 ++ runtime/Dart/pubspec.yaml | 10 + .../v4/tool/templates/codegen/Dart/Dart.stg | 875 ++++++ .../antlr/v4/codegen/target/DartTarget.java | 112 + 81 files changed, 21068 insertions(+), 3 deletions(-) create mode 100755 .travis/before-install-linux-dart.sh create mode 100755 .travis/run-tests-dart.sh create mode 100644 runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Dart.test.stg create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestCompositeLexers.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestCompositeParsers.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestFullContextParsing.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestLeftRecursion.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestLexerErrors.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestLexerExec.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestListeners.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestParseTrees.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestParserErrors.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestParserExec.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestPerformance.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestSemPredEvalLexer.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestSemPredEvalParser.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestSets.java create mode 100644 runtime/Dart/.gitignore create mode 100644 runtime/Dart/LICENSE.txt create mode 100644 runtime/Dart/README.md create mode 100644 runtime/Dart/lib/antlr4.dart create mode 100644 runtime/Dart/lib/src/atn/atn.dart create mode 100644 runtime/Dart/lib/src/atn/src/atn.dart create mode 100644 runtime/Dart/lib/src/atn/src/atn_config.dart create mode 100644 runtime/Dart/lib/src/atn/src/atn_config_set.dart create mode 100644 runtime/Dart/lib/src/atn/src/atn_deserializer.dart create mode 100644 runtime/Dart/lib/src/atn/src/atn_simulator.dart create mode 100644 runtime/Dart/lib/src/atn/src/atn_state.dart create mode 100644 runtime/Dart/lib/src/atn/src/atn_type.dart create mode 100644 runtime/Dart/lib/src/atn/src/info.dart create mode 100644 runtime/Dart/lib/src/atn/src/lexer_action.dart create mode 100644 runtime/Dart/lib/src/atn/src/lexer_action_executor.dart create mode 100644 runtime/Dart/lib/src/atn/src/lexer_atn_simulator.dart create mode 100644 runtime/Dart/lib/src/atn/src/parser_atn_simulator.dart create mode 100644 runtime/Dart/lib/src/atn/src/profiling_atn_simulator.dart create mode 100644 runtime/Dart/lib/src/atn/src/semantic_context.dart create mode 100644 runtime/Dart/lib/src/atn/src/transition.dart create mode 100644 runtime/Dart/lib/src/dfa/dfa.dart create mode 100644 runtime/Dart/lib/src/dfa/src/dfa.dart create mode 100644 runtime/Dart/lib/src/dfa/src/dfa_serializer.dart create mode 100644 runtime/Dart/lib/src/dfa/src/dfa_state.dart create mode 100644 runtime/Dart/lib/src/error/error.dart create mode 100644 runtime/Dart/lib/src/error/src/diagnostic_error_listener.dart create mode 100644 runtime/Dart/lib/src/error/src/error_listener.dart create mode 100644 runtime/Dart/lib/src/error/src/error_strategy.dart create mode 100644 runtime/Dart/lib/src/error/src/errors.dart create mode 100644 runtime/Dart/lib/src/input_stream.dart create mode 100644 runtime/Dart/lib/src/interval_set.dart create mode 100644 runtime/Dart/lib/src/lexer.dart create mode 100644 runtime/Dart/lib/src/ll1_analyzer.dart create mode 100644 runtime/Dart/lib/src/misc/multi_map.dart create mode 100644 runtime/Dart/lib/src/misc/pair.dart create mode 100644 runtime/Dart/lib/src/parser.dart create mode 100644 runtime/Dart/lib/src/parser_interpreter.dart create mode 100644 runtime/Dart/lib/src/parser_rule_context.dart create mode 100644 runtime/Dart/lib/src/prediction_context.dart create mode 100644 runtime/Dart/lib/src/recognizer.dart create mode 100644 runtime/Dart/lib/src/rule_context.dart create mode 100644 runtime/Dart/lib/src/runtime_meta_data.dart create mode 100644 runtime/Dart/lib/src/token.dart create mode 100644 runtime/Dart/lib/src/token_factory.dart create mode 100644 runtime/Dart/lib/src/token_source.dart create mode 100644 runtime/Dart/lib/src/token_stream.dart create mode 100644 runtime/Dart/lib/src/tree/src/pattern/chunk.dart create mode 100644 runtime/Dart/lib/src/tree/src/pattern/parse_tree_match.dart create mode 100644 runtime/Dart/lib/src/tree/src/tree.dart create mode 100644 runtime/Dart/lib/src/tree/src/trees.dart create mode 100644 runtime/Dart/lib/src/tree/tree.dart create mode 100644 runtime/Dart/lib/src/util/bit_set.dart create mode 100644 runtime/Dart/lib/src/util/murmur_hash.dart create mode 100644 runtime/Dart/lib/src/util/utils.dart create mode 100644 runtime/Dart/lib/src/vocabulary.dart create mode 100644 runtime/Dart/pubspec.yaml create mode 100644 tool/resources/org/antlr/v4/tool/templates/codegen/Dart/Dart.stg create mode 100644 tool/src/org/antlr/v4/codegen/target/DartTarget.java diff --git a/.travis.yml b/.travis.yml index cae5dae3d..16665093c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -152,6 +152,10 @@ matrix: jdk: openjdk8 env: TARGET=csharp stage: main-test + - os: linux + jdk: openjdk8 + env: TARGET=dart + stage: main-test - os: linux language: php php: diff --git a/.travis/before-install-linux-dart.sh b/.travis/before-install-linux-dart.sh new file mode 100755 index 000000000..f9f8266ff --- /dev/null +++ b/.travis/before-install-linux-dart.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -euo pipefail + +sudo apt-get -y install apt-transport-https +sudo sh -c 'wget -qO- https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -' +sudo sh -c 'wget -qO- https://storage.googleapis.com/download.dartlang.org/linux/debian/dart_stable.list > /etc/apt/sources.list.d/dart_stable.list' +sudo apt-get -q update +sudo apt-get -y install dart +export PATH="$PATH:/usr/lib/dart/bin" diff --git a/.travis/run-tests-dart.sh b/.travis/run-tests-dart.sh new file mode 100755 index 000000000..8053a90d7 --- /dev/null +++ b/.travis/run-tests-dart.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +set -euo pipefail +mvn -q -Dparallel=classes -DthreadCount=4 -Dtest=dart.* test diff --git a/contributors.txt b/contributors.txt index 1fe97ab8b..fdc7a19ef 100644 --- a/contributors.txt +++ b/contributors.txt @@ -236,4 +236,5 @@ YYYY/MM/DD, github id, Full name, email 2019/10/31, a-square, Alexei Averchenko, lex.aver@gmail.com 2019/11/11, foxeverl, Liu Xinfeng, liuxf1986[at]gmail[dot]com 2019/11/17, felixn, Felix Nieuwenhuizhen, felix@tdlrali.com -2019/11/18, mlilback, Mark Lilback, mark@lilback.com \ No newline at end of file +2019/11/18, mlilback, Mark Lilback, mark@lilback.com +2020/01/19, lingyv-li, Lingyu Li, lingyv.li@gmail.com \ No newline at end of file diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 33ea5e9f1..2bb36dc58 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -116,6 +116,7 @@ **/python2/Test*.java **/python3/Test*.java **/php/Test*.java + **/dart/Test*.java ${antlr.tests.swift} diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Dart.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Dart.test.stg new file mode 100644 index 000000000..73bcb9523 --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Dart.test.stg @@ -0,0 +1,318 @@ +writeln(s) ::= <);>> +write(s) ::= <);>> +writeList(s) ::= <);>> + +False() ::= "false" + +True() ::= "true" + +Not(v) ::= "!" + +Assert(s) ::= <);>> + +Cast(t,v) ::= "( as )" + +Append(a,b) ::= ".toString() + .toString()" + +AppendStr(a,b) ::= <%%> + +Concat(a,b) ::= "" + +AssertIsList(v) ::= "assert ( is List);" // just use static type system + +AssignLocal(s,v) ::= " = ;" + +InitIntMember(n,v) ::= <%int = ;%> + +InitBooleanMember(n,v) ::= <%bool = ;%> + +InitIntVar(n,v) ::= <%%> + +IntArg(n) ::= "int " + +VarRef(n) ::= "" + +GetMember(n) ::= <%this.%> + +SetMember(n,v) ::= <%this. = ;%> + +AddMember(n,v) ::= <%this. += ;%> + +MemberEquals(n,v) ::= <%this. == %> + +ModMemberEquals(n,m,v) ::= <%this. % == %> + +ModMemberNotEquals(n,m,v) ::= <%this. % != %> + +DumpDFA() ::= "this.dumpDFA();" + +Pass() ::= "" + +StringList() ::= "List\" + +BuildParseTrees() ::= "buildParseTree = true;" + +BailErrorStrategy() ::= <%errorHandler = new BailErrorStrategy();%> + +ToStringTree(s) ::= <%.toStringTree(parser: this)%> + +Column() ::= "this.charPositionInLine" + +Text() ::= "this.text" + +ValEquals(a,b) ::= <%==%> + +TextEquals(a) ::= <%this.text == ""%> + +PlusText(a) ::= <%"" + this.text%> + +InputText() ::= "this.tokenStream.text" + +LTEquals(i, v) ::= <%this.tokenStream.LT().text == %> + +LANotEquals(i, v) ::= <%this.tokenStream.LA()!=%> + +TokenStartColumnEquals(i) ::= <%this.tokenStartCharPositionInLine==%> + +ImportListener(X) ::= "" + +GetExpectedTokenNames() ::= "this.expectedTokens.toString(vocabulary: this.vocabulary)" + +RuleInvocationStack() ::= "ruleInvocationStack" + +LL_EXACT_AMBIG_DETECTION() ::= <> + +ParserToken(parser, token) ::= <%.TOKEN_%> + +Production(p) ::= <%

%> + +Result(r) ::= <%%> + +ParserPropertyMember() ::= << +@members { +bool Property() { + return true; +} +} +>> + +ParserPropertyCall(p, call) ::= "

." + +PositionAdjustingLexerDef() ::= << +class PositionAdjustingLexerATNSimulator extends LexerATNSimulator { + PositionAdjustingLexerATNSimulator(Lexer recog, ATN atn, + List decisionToDFA, PredictionContextCache sharedContextCache) + : super(atn, decisionToDFA, sharedContextCache, recog: recog); + + void resetAcceptPosition(CharStream input, int index, int line, + int charPositionInLine) { + input.seek(index); + this.line = line; + this.charPositionInLine = charPositionInLine; + consume(input); + } +} +>> + +PositionAdjustingLexer() ::= << +@override +Token nextToken() { + if (!(super.interpreter is PositionAdjustingLexerATNSimulator)) { + interpreter = new PositionAdjustingLexerATNSimulator( + this, _ATN, _decisionToDFA, _sharedContextCache); + } + + return super.nextToken(); +} + +@override +Token emit() { + switch (type) { + case TOKEN_TOKENS: + handleAcceptPositionForKeyword("tokens"); + break; + + case TOKEN_LABEL: + handleAcceptPositionForIdentifier(); + break; + + default: + break; + } + + return super.emit(); +} + +bool handleAcceptPositionForIdentifier() { + String tokenText = text; + int identifierLength = 0; + while (identifierLength \< tokenText.length && + isIdentifierChar(tokenText[identifierLength])) { + identifierLength++; + } + + if (inputStream.index > tokenStartCharIndex + identifierLength) { + int offset = identifierLength - 1; + interpreter.resetAcceptPosition(inputStream, tokenStartCharIndex + offset, + tokenStartLine, tokenStartCharPositionInLine + offset); + return true; + } + + return false; +} + +bool handleAcceptPositionForKeyword(String keyword) { + if (inputStream.index > tokenStartCharIndex + keyword.length) { + int offset = keyword.length - 1; + interpreter.resetAcceptPosition(inputStream, tokenStartCharIndex + offset, + tokenStartLine, tokenStartCharPositionInLine + offset); + return true; + } + + return false; +} + +@override +PositionAdjustingLexerATNSimulator get interpreter { + return super.interpreter as PositionAdjustingLexerATNSimulator; +} + +static bool isIdentifierChar(String c) { + return isLetterOrDigit(c) || c == '_'; +} + +static const ZERO = 48; +static const LOWER_A = 97; +static const LOWER_Z = 122; +static const UPPER_A = 65; +static const UPPER_Z = 90; + +static bool isLetterOrDigit(String char) => isLetter(char) || isDigit(char); + +// Note: this is intentially ASCII only +static bool isLetter(String char) { + if (char == null) return false; + var cc = char.codeUnitAt(0); + return cc >= LOWER_A && cc \<= LOWER_Z || cc >= UPPER_A && cc \<= UPPER_Z; +} + +static bool isDigit(String char) { + if (char == null) return false; + var cc = char.codeUnitAt(0); + return cc >= ZERO && cc \< ZERO + 10; +} +>> + +BasicListener(X) ::= << +@parser::definitions { +class LeafListener extends TBaseListener { + void visitTerminal(TerminalNode node) { + print(node.symbol.text); + } +} +} +>> + +WalkListener(s) ::= << +ParseTreeWalker walker = new ParseTreeWalker(); +walker.walk(new LeafListener(), ); +>> + +TreeNodeWithAltNumField(X) ::= << +@parser::definitions { +class MyRuleNode extends ParserRuleContext { + int altNum; + + MyRuleNode(ParserRuleContext parent, int invokingStateNumber) + : super(parent, invokingStateNumber); + + @override int get altNumber { + return altNum; + } + + @override void set altNumber(int altNum) { + this.altNum = altNum; + } +} +} +>> + +TokenGetterListener(X) ::= << +@parser::definitions { +class LeafListener extends TBaseListener { + void exitA(AContext ctx) { + if (ctx.childCount==2) + stdout.write("${ctx.INT(0).symbol.text} ${ctx.INT(1).symbol.text} ${ctx.INTs()}"); + else + print(ctx.ID().symbol); + } +} +} +>> + +RuleGetterListener(X) ::= << +@parser::definitions { +class LeafListener extends TBaseListener { + void exitA(AContext ctx) { + if (ctx.childCount==2) { + stdout.write("${ctx.b(0).start.text} ${ctx.b(1).start.text} ${ctx.bs()[0].start.text}"); + } else + print(ctx.b(0).start.text); + } +} +} +>> + + +LRListener(X) ::= << +@parser::definitions { +class LeafListener extends TBaseListener { + void exitE(EContext ctx) { + if (ctx.childCount==3) { + stdout.write("${ctx.e(0).start.text} ${ctx.e(1).start.text} ${ctx.es()[0].start.text}\n"); + } else + print(ctx.INT().symbol.text); + } +} +} +>> + +LRWithLabelsListener(X) ::= << +@parser::definitions { +class LeafListener extends TBaseListener { + void exitCall(CallContext ctx) { + stdout.write("${ctx.e().start.text} ${ctx.eList()}"); + } + void exitInt(IntContext ctx) { + print(ctx.INT().symbol.text); + } +} +} +>> + +DeclareContextListGettersFunction() ::= << +void foo() { + SContext s = null; + List\ a = s.as(); + List\ b = s.bs(); +} +>> + +Declare_foo() ::= << + void foo() {print("foo");} +>> + +Invoke_foo() ::= "foo();" + +Declare_pred() ::= <> + +Invoke_pred(v) ::= <)>> + +ParserTokenType(t) ::= "Parser." +ContextRuleFunction(ctx, rule) ::= "." +StringType() ::= "String" +ContextMember(ctx, subctx, member) ::= ".." diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java index f7874d671..94acf1956 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java @@ -51,7 +51,8 @@ public abstract class BaseRuntimeTest { "CSharp", "Python2", "Python3", "PHP", - "Node", "Safari", "Firefox", "Explorer", "Chrome" + "Node", "Safari", "Firefox", "Explorer", "Chrome", + "Dart" }; public final static String[] JavaScriptTargets = { "Node", "Safari", "Firefox", "Explorer", "Chrome" @@ -302,6 +303,16 @@ public abstract class BaseRuntimeTest { } } + public static String readFile(String dir, String fileName) { + try { + return String.copyValueOf(Utils.readFile(dir+"/"+fileName, "UTF-8")); + } + catch (IOException ioe) { + System.err.println("can't read file"); + ioe.printStackTrace(System.err); + } + return null; + } protected static void assertCorrectOutput(RuntimeTestDescriptor descriptor, RuntimeTestSupport delegate, String actualOutput) { String actualParseErrors = delegate.getParseErrors(); diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java new file mode 100644 index 000000000..9d9df1f8a --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java @@ -0,0 +1,1149 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.Tool; +import org.antlr.v4.analysis.AnalysisPipeline; +import org.antlr.v4.automata.ATNFactory; +import org.antlr.v4.automata.ATNPrinter; +import org.antlr.v4.automata.LexerATNFactory; +import org.antlr.v4.automata.ParserATNFactory; +import org.antlr.v4.codegen.CodeGenerator; +import org.antlr.v4.misc.Utils; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.misc.IntegerList; +import org.antlr.v4.runtime.misc.Interval; +import org.antlr.v4.runtime.misc.Pair; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.semantics.SemanticPipeline; +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.ErrorQueue; +import org.antlr.v4.test.runtime.RuntimeTestSupport; +import org.antlr.v4.test.runtime.StreamVacuum; +import org.antlr.v4.tool.*; +import org.stringtemplate.v4.ST; +import org.stringtemplate.v4.STGroup; +import org.stringtemplate.v4.STGroupString; + +import java.io.*; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.*; + +import static junit.framework.TestCase.*; +import static org.antlr.v4.test.runtime.BaseRuntimeTest.readFile; +import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; +import static org.junit.Assert.assertArrayEquals; + + +public class BaseDartTest implements RuntimeTestSupport { + public static final String newline = System.getProperty("line.separator"); + public static final String pathSep = System.getProperty("path.separator"); + + + /** + * When the {@code antlr.preserve-test-dir} runtime property is set to + * {@code true}, the temporary directories created by the test run will not + * be removed at the end of the test run, even for tests that completed + * successfully. + *

+ *

+ * The default behavior (used in all other cases) is removing the temporary + * directories for all tests which completed successfully, and preserving + * the directories for tests which failed.

+ */ + public static final boolean PRESERVE_TEST_DIR = Boolean.parseBoolean(System.getProperty("antlr.preserve-test-dir", "false")); + + /** + * The base test directory is the directory where generated files get placed + * during unit test execution. + *

+ *

+ * The default value for this property is the {@code java.io.tmpdir} system + * property, and can be overridden by setting the + * {@code antlr.java-test-dir} property to a custom location. Note that the + * {@code antlr.java-test-dir} property directly affects the + * {@link #CREATE_PER_TEST_DIRECTORIES} value as well.

+ */ + public static final String BASE_TEST_DIR; + + /** + * When {@code true}, a temporary directory will be created for each test + * executed during the test run. + *

+ *

+ * This value is {@code true} when the {@code antlr.java-test-dir} system + * property is set, and otherwise {@code false}.

+ */ + public static final boolean CREATE_PER_TEST_DIRECTORIES; + + static { + String baseTestDir = System.getProperty("antlr.dart-test-dir"); + boolean perTestDirectories = false; + if (baseTestDir == null || baseTestDir.isEmpty()) { + baseTestDir = System.getProperty("java.io.tmpdir"); + perTestDirectories = true; + } + + if (!new File(baseTestDir).isDirectory()) { + throw new UnsupportedOperationException("The specified base test directory does not exist: " + baseTestDir); + } + + BASE_TEST_DIR = baseTestDir; + CREATE_PER_TEST_DIRECTORIES = perTestDirectories; + } + + /** + * Build up the full classpath we need, including the surefire path (if present) + */ + public static final String CLASSPATH = System.getProperty("java.class.path"); + + public String tmpdir = null; + + /** + * If error during parser execution, store stderr here; can't return + * stdout and stderr. This doesn't trap errors from running antlr. + */ + protected String stderrDuringParse; + + /** + * Errors found while running antlr + */ + protected StringBuilder antlrToolErrors; + + private static String cacheDartPackages; + + private String getPropertyPrefix() { + return "antlr-php"; + } + + @Override + public void testSetUp() throws Exception { + if (CREATE_PER_TEST_DIRECTORIES) { + // new output dir for each test + String threadName = Thread.currentThread().getName(); + String testDirectory = getClass().getSimpleName() + "-" + threadName + "-" + System.nanoTime(); + tmpdir = new File(BASE_TEST_DIR, testDirectory).getAbsolutePath(); + } else { + tmpdir = new File(BASE_TEST_DIR).getAbsolutePath(); + if (!PRESERVE_TEST_DIR && new File(tmpdir).exists()) { + eraseFiles(); + } + } + antlrToolErrors = new StringBuilder(); + } + + @Override + public void testTearDown() throws Exception { + } + + @Override + public String getTmpDir() { + return tmpdir; + } + + @Override + public String getStdout() { + return null; + } + + @Override + public String getParseErrors() { + return stderrDuringParse; + } + + @Override + public String getANTLRToolErrors() { + if (antlrToolErrors.length() == 0) { + return null; + } + return antlrToolErrors.toString(); + } + + protected Tool newTool(String[] args) { + Tool tool = new Tool(args); + return tool; + } + + protected ATN createATN(Grammar g, boolean useSerializer) { + if (g.atn == null) { + semanticProcess(g); + assertEquals(0, g.tool.getNumErrors()); + + ParserATNFactory f; + if (g.isLexer()) { + f = new LexerATNFactory((LexerGrammar) g); + } else { + f = new ParserATNFactory(g); + } + + g.atn = f.createATN(); + assertEquals(0, g.tool.getNumErrors()); + } + + ATN atn = g.atn; + if (useSerializer) { + char[] serialized = ATNSerializer.getSerializedAsChars(atn); + return new ATNDeserializer().deserialize(serialized); + } + + return atn; + } + + protected void semanticProcess(Grammar g) { + if (g.ast != null && !g.ast.hasErrors) { +// System.out.println(g.ast.toStringTree()); + Tool antlr = new Tool(); + SemanticPipeline sem = new SemanticPipeline(g); + sem.process(); + if (g.getImportedGrammars() != null) { // process imported grammars (if any) + for (Grammar imp : g.getImportedGrammars()) { + antlr.processNonCombinedGrammar(imp, false); + } + } + } + } + + public DFA createDFA(Grammar g, DecisionState s) { +// PredictionDFAFactory conv = new PredictionDFAFactory(g, s); +// DFA dfa = conv.createDFA(); +// conv.issueAmbiguityWarnings(); +// System.out.print("DFA="+dfa); +// return dfa; + return null; + } + +// public void minimizeDFA(DFA dfa) { +// DFAMinimizer dmin = new DFAMinimizer(dfa); +// dfa.minimized = dmin.minimize(); +// } + + IntegerList getTypesFromString(Grammar g, String expecting) { + IntegerList expectingTokenTypes = new IntegerList(); + if (expecting != null && !expecting.trim().isEmpty()) { + for (String tname : expecting.replace(" ", "").split(",")) { + int ttype = g.getTokenType(tname); + expectingTokenTypes.add(ttype); + } + } + return expectingTokenTypes; + } + + public IntegerList getTokenTypesViaATN(String input, LexerATNSimulator lexerATN) { + ANTLRInputStream in = new ANTLRInputStream(input); + IntegerList tokenTypes = new IntegerList(); + int ttype; + do { + ttype = lexerATN.match(in, Lexer.DEFAULT_MODE); + tokenTypes.add(ttype); + } while (ttype != Token.EOF); + return tokenTypes; + } + + public List getTokenTypes(LexerGrammar lg, + ATN atn, + CharStream input) { + LexerATNSimulator interp = new LexerATNSimulator(atn, new DFA[]{new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE))}, null); + List tokenTypes = new ArrayList(); + int ttype; + boolean hitEOF = false; + do { + if (hitEOF) { + tokenTypes.add("EOF"); + break; + } + int t = input.LA(1); + ttype = interp.match(input, Lexer.DEFAULT_MODE); + if (ttype == Token.EOF) { + tokenTypes.add("EOF"); + } else { + tokenTypes.add(lg.typeToTokenList.get(ttype)); + } + + if (t == IntStream.EOF) { + hitEOF = true; + } + } while (ttype != Token.EOF); + return tokenTypes; + } + + List checkRuleDFA(String gtext, String ruleName, String expecting) + throws Exception { + ErrorQueue equeue = new ErrorQueue(); + Grammar g = new Grammar(gtext, equeue); + ATN atn = createATN(g, false); + ATNState s = atn.ruleToStartState[g.getRule(ruleName).index]; + if (s == null) { + System.err.println("no such rule: " + ruleName); + return null; + } + ATNState t = s.transition(0).target; + if (!(t instanceof DecisionState)) { + System.out.println(ruleName + " has no decision"); + return null; + } + DecisionState blk = (DecisionState) t; + checkRuleDFA(g, blk, expecting); + return equeue.all; + } + + List checkRuleDFA(String gtext, int decision, String expecting) + throws Exception { + ErrorQueue equeue = new ErrorQueue(); + Grammar g = new Grammar(gtext, equeue); + ATN atn = createATN(g, false); + DecisionState blk = atn.decisionToState.get(decision); + checkRuleDFA(g, blk, expecting); + return equeue.all; + } + + void checkRuleDFA(Grammar g, DecisionState blk, String expecting) + throws Exception { + DFA dfa = createDFA(g, blk); + String result = null; + if (dfa != null) result = dfa.toString(); + assertEquals(expecting, result); + } + + List checkLexerDFA(String gtext, String expecting) + throws Exception { + return checkLexerDFA(gtext, LexerGrammar.DEFAULT_MODE_NAME, expecting); + } + + List checkLexerDFA(String gtext, String modeName, String expecting) + throws Exception { + ErrorQueue equeue = new ErrorQueue(); + LexerGrammar g = new LexerGrammar(gtext, equeue); + g.atn = createATN(g, false); +// LexerATNToDFAConverter conv = new LexerATNToDFAConverter(g); +// DFA dfa = conv.createDFA(modeName); +// g.setLookaheadDFA(0, dfa); // only one decision to worry about +// +// String result = null; +// if ( dfa!=null ) result = dfa.toString(); +// assertEquals(expecting, result); +// +// return equeue.all; + return null; + } + + protected String load(String fileName, String encoding) + throws IOException { + if (fileName == null) { + return null; + } + + String fullFileName = getClass().getPackage().getName().replace('.', '/') + '/' + fileName; + int size = 65000; + InputStreamReader isr; + InputStream fis = getClass().getClassLoader().getResourceAsStream(fullFileName); + if (encoding != null) { + isr = new InputStreamReader(fis, encoding); + } else { + isr = new InputStreamReader(fis); + } + try { + char[] data = new char[size]; + int n = isr.read(data); + return new String(data, 0, n); + } finally { + isr.close(); + } + } + + protected String execLexer(String grammarFileName, + String grammarStr, + String lexerName, + String input) { + return execLexer(grammarFileName, grammarStr, lexerName, input, false); + } + + @Override + public String execLexer(String grammarFileName, + String grammarStr, + String lexerName, + String input, + boolean showDFA) { + boolean success = rawGenerateAndBuildRecognizer(grammarFileName, + grammarStr, + null, + lexerName); + assertTrue(success); + writeFile(tmpdir, "input", input); + writeLexerTestFile(lexerName, showDFA); + String output = execClass("Test.dart"); + return output; + } + + public ParseTree execParser(String startRuleName, String input, + String parserName, String lexerName) + throws Exception { + Pair pl = getParserAndLexer(input, parserName, lexerName); + Parser parser = pl.a; + return execStartRule(startRuleName, parser); + } + + public ParseTree execStartRule(String startRuleName, Parser parser) + throws IllegalAccessException, InvocationTargetException, + NoSuchMethodException { + Method startRule = null; + Object[] args = null; + try { + startRule = parser.getClass().getMethod(startRuleName); + } catch (NoSuchMethodException nsme) { + // try with int _p arg for recursive func + startRule = parser.getClass().getMethod(startRuleName, int.class); + args = new Integer[]{0}; + } + ParseTree result = (ParseTree) startRule.invoke(parser, args); +// System.out.println("parse tree = "+result.toStringTree(parser)); + return result; + } + + public Pair getParserAndLexer(String input, + String parserName, String lexerName) + throws Exception { + final Class lexerClass = loadLexerClassFromTempDir(lexerName); + final Class parserClass = loadParserClassFromTempDir(parserName); + + ANTLRInputStream in = new ANTLRInputStream(new StringReader(input)); + + Class c = lexerClass.asSubclass(Lexer.class); + Constructor ctor = c.getConstructor(CharStream.class); + Lexer lexer = ctor.newInstance(in); + + Class pc = parserClass.asSubclass(Parser.class); + Constructor pctor = pc.getConstructor(TokenStream.class); + CommonTokenStream tokens = new CommonTokenStream(lexer); + Parser parser = pctor.newInstance(tokens); + return new Pair(parser, lexer); + } + + public Class loadClassFromTempDir(String name) throws Exception { + ClassLoader loader = + new URLClassLoader(new URL[]{new File(tmpdir).toURI().toURL()}, + ClassLoader.getSystemClassLoader()); + return loader.loadClass(name); + } + + public Class loadLexerClassFromTempDir(String name) throws Exception { + return loadClassFromTempDir(name).asSubclass(Lexer.class); + } + + public Class loadParserClassFromTempDir(String name) throws Exception { + return loadClassFromTempDir(name).asSubclass(Parser.class); + } + + @Override + public String execParser(String grammarFileName, + String grammarStr, + String parserName, + String lexerName, + String listenerName, + String visitorName, + String startRuleName, + String input, + boolean showDiagnosticErrors) { + return execParser(grammarFileName, grammarStr, parserName, lexerName, + listenerName, visitorName, startRuleName, input, showDiagnosticErrors, false); + } + + public String execParser(String grammarFileName, + String grammarStr, + String parserName, + String lexerName, + String listenerName, + String visitorName, + String startRuleName, + String input, + boolean showDiagnosticErrors, + boolean profile) { + boolean success = rawGenerateAndBuildRecognizer(grammarFileName, + grammarStr, + parserName, + lexerName, + "-visitor"); + assertTrue(success); + writeFile(tmpdir, "input", input); + return rawExecRecognizer(parserName, + lexerName, + startRuleName, + showDiagnosticErrors, + profile); + } + + /** + * Return true if all is well + */ + protected boolean rawGenerateAndBuildRecognizer(String grammarFileName, + String grammarStr, + String parserName, + String lexerName, + String... extraOptions) { + return rawGenerateAndBuildRecognizer(grammarFileName, grammarStr, parserName, lexerName, false, extraOptions); + } + + /** + * Return true if all is well + */ + protected boolean rawGenerateAndBuildRecognizer(String grammarFileName, + String grammarStr, + String parserName, + String lexerName, + boolean defaultListener, + String... extraOptions) { + ErrorQueue equeue = + BaseRuntimeTest.antlrOnString(getTmpDir(), "Dart", grammarFileName, grammarStr, defaultListener, extraOptions); + if (!equeue.errors.isEmpty()) { + return false; + } + + List files = new ArrayList(); + if (lexerName != null) { + files.add(lexerName + ".dart"); + } + if (parserName != null) { + files.add(parserName + ".dart"); + Set optionsSet = new HashSet(Arrays.asList(extraOptions)); + String grammarName = grammarFileName.substring(0, grammarFileName.lastIndexOf('.')); + if (!optionsSet.contains("-no-listener")) { + files.add(grammarName + "Listener.dart"); + files.add(grammarName + "BaseListener.dart"); + } + if (optionsSet.contains("-visitor")) { + files.add(grammarName + "Visitor.dart"); + files.add(grammarName + "BaseVisitor.dart"); + } + } + + String runtime = locateRuntime(); + writeFile(tmpdir, "pubspec.yaml", + "name: \"test\"\n" + + "dependencies:\n" + + " antlr4:\n" + + " path: " + runtime + "\n"); + if (cacheDartPackages == null) { + System.out.println("Not skipping" + tmpdir); + try { + Process process = Runtime.getRuntime().exec(new String[]{locatePub(), "get"}, null, new File(tmpdir)); + StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); + stderrVacuum.start(); + process.waitFor(); + stderrVacuum.join(); + System.out.println(stderrVacuum.toString()); + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + return false; + } + cacheDartPackages = readFile(tmpdir, ".packages"); + } else { + writeFile(tmpdir, ".packages", cacheDartPackages); + } + return true; // allIsWell: no compile + } + + protected String rawExecRecognizer(String parserName, + String lexerName, + String parserStartRuleName, + boolean debug, + boolean profile) { + this.stderrDuringParse = null; + if (parserName == null) { + writeLexerTestFile(lexerName, false); + } else { + writeTestFile(parserName, + lexerName, + parserStartRuleName, + debug, + profile); + } + + return execClass("Test.dart"); + } + + public String execRecognizer() { + return execClass("Test.dart"); + } + + public String execClass(String className) { + try { + String[] args = new String[]{ + locateDart(), + className, new File(tmpdir, "input").getAbsolutePath() + }; + String cmdLine = Utils.join(args, " "); + System.err.println("execParser: " + cmdLine); + Process process = + Runtime.getRuntime().exec(args, null, new File(tmpdir)); + StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); + StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); + stdoutVacuum.start(); + stderrVacuum.start(); + process.waitFor(); + stdoutVacuum.join(); + stderrVacuum.join(); + String output = stdoutVacuum.toString(); + if (output.length() == 0) { + output = null; + } + if (stderrVacuum.toString().length() > 0) { + this.stderrDuringParse = stderrVacuum.toString(); + } + return output; + } catch (Exception e) { + System.err.println("can't exec recognizer"); + e.printStackTrace(System.err); + } + return null; + } + + private String locateTool(String tool) { + final String phpPath = System.getProperty("DART_PATH"); + + if (phpPath != null && new File(phpPath).exists()) { + return phpPath; + } + + String[] roots = {"/usr/local/bin/", "/opt/local/bin/", "/usr/bin/", "/usr/lib/dart/bin/"}; + + for (String root : roots) { + if (new File(root + tool).exists()) { + return root + tool; + } + } + + throw new RuntimeException("Could not locate " + tool); + } + + protected String locatePub() { + String propName = getPropertyPrefix() + "-pub"; + String prop = System.getProperty(propName); + + if (prop == null || prop.length() == 0) { + prop = locateTool("pub"); + } + + File file = new File(prop); + + if (!file.exists()) { + throw new RuntimeException("Missing system property:" + propName); + } + + return file.getAbsolutePath(); + } + + protected String locateDart() { + String propName = getPropertyPrefix() + "-dart"; + String prop = System.getProperty(propName); + + if (prop == null || prop.length() == 0) { + prop = locateTool("dart"); + } + + File file = new File(prop); + + if (!file.exists()) { + throw new RuntimeException("Missing system property:" + propName); + } + + return file.getAbsolutePath(); + } + + private String locateRuntime() { + final ClassLoader loader = Thread.currentThread().getContextClassLoader(); + final URL runtimeSrc = loader.getResource("Dart"); + if (runtimeSrc == null) { + throw new RuntimeException("Cannot find Dart runtime"); + } + if (isWindows()) { + return runtimeSrc.getPath().replaceFirst("/", ""); + } + return runtimeSrc.getPath(); + } + + private boolean isWindows() { + return System.getProperty("os.name").toLowerCase().contains("windows"); + } + +// void ambig(List msgs, int[] expectedAmbigAlts, String expectedAmbigInput) +// throws Exception +// { +// ambig(msgs, 0, expectedAmbigAlts, expectedAmbigInput); +// } + +// void ambig(List msgs, int i, int[] expectedAmbigAlts, String expectedAmbigInput) +// throws Exception +// { +// List amsgs = getMessagesOfType(msgs, AmbiguityMessage.class); +// AmbiguityMessage a = (AmbiguityMessage)amsgs.get(i); +// if ( a==null ) assertNull(expectedAmbigAlts); +// else { +// assertEquals(a.conflictingAlts.toString(), Arrays.toString(expectedAmbigAlts)); +// } +// assertEquals(expectedAmbigInput, a.input); +// } + +// void unreachable(List msgs, int[] expectedUnreachableAlts) +// throws Exception +// { +// unreachable(msgs, 0, expectedUnreachableAlts); +// } + +// void unreachable(List msgs, int i, int[] expectedUnreachableAlts) +// throws Exception +// { +// List amsgs = getMessagesOfType(msgs, UnreachableAltsMessage.class); +// UnreachableAltsMessage u = (UnreachableAltsMessage)amsgs.get(i); +// if ( u==null ) assertNull(expectedUnreachableAlts); +// else { +// assertEquals(u.conflictingAlts.toString(), Arrays.toString(expectedUnreachableAlts)); +// } +// } + + List getMessagesOfType(List msgs, Class c) { + List filtered = new ArrayList(); + for (ANTLRMessage m : msgs) { + if (m.getClass() == c) filtered.add(m); + } + return filtered; + } + + public void checkRuleATN(Grammar g, String ruleName, String expecting) { +// DOTGenerator dot = new DOTGenerator(g); +// System.out.println(dot.getDOT(g.atn.ruleToStartState[g.getRule(ruleName).index])); + + Rule r = g.getRule(ruleName); + ATNState startState = g.getATN().ruleToStartState[r.index]; + ATNPrinter serializer = new ATNPrinter(g, startState); + String result = serializer.asString(); + + //System.out.print(result); + assertEquals(expecting, result); + } + + public void testActions(String templates, String actionName, String action, String expected) throws org.antlr.runtime.RecognitionException { + int lp = templates.indexOf('('); + String name = templates.substring(0, lp); + STGroup group = new STGroupString(templates); + ST st = group.getInstanceOf(name); + st.add(actionName, action); + String grammar = st.render(); + ErrorQueue equeue = new ErrorQueue(); + Grammar g = new Grammar(grammar, equeue); + if (g.ast != null && !g.ast.hasErrors) { + SemanticPipeline sem = new SemanticPipeline(g); + sem.process(); + + ATNFactory factory = new ParserATNFactory(g); + if (g.isLexer()) factory = new LexerATNFactory((LexerGrammar) g); + g.atn = factory.createATN(); + + AnalysisPipeline anal = new AnalysisPipeline(g); + anal.process(); + + CodeGenerator gen = new CodeGenerator(g); + ST outputFileST = gen.generateParser(false); + String output = outputFileST.render(); + //System.out.println(output); + String b = "#" + actionName + "#"; + int start = output.indexOf(b); + String e = "#end-" + actionName + "#"; + int end = output.indexOf(e); + String snippet = output.substring(start + b.length(), end); + assertEquals(expected, snippet); + } + if (equeue.size() > 0) { +// System.err.println(equeue.toString()); + } + } + + protected void checkGrammarSemanticsError(ErrorQueue equeue, + GrammarSemanticsMessage expectedMessage) + throws Exception { + ANTLRMessage foundMsg = null; + for (int i = 0; i < equeue.errors.size(); i++) { + ANTLRMessage m = equeue.errors.get(i); + if (m.getErrorType() == expectedMessage.getErrorType()) { + foundMsg = m; + } + } + assertNotNull("no error; " + expectedMessage.getErrorType() + " expected", foundMsg); + assertTrue("error is not a GrammarSemanticsMessage", + foundMsg instanceof GrammarSemanticsMessage); + assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs())); + if (equeue.size() != 1) { + System.err.println(equeue); + } + } + + protected void checkGrammarSemanticsWarning(ErrorQueue equeue, + GrammarSemanticsMessage expectedMessage) + throws Exception { + ANTLRMessage foundMsg = null; + for (int i = 0; i < equeue.warnings.size(); i++) { + ANTLRMessage m = equeue.warnings.get(i); + if (m.getErrorType() == expectedMessage.getErrorType()) { + foundMsg = m; + } + } + assertNotNull("no error; " + expectedMessage.getErrorType() + " expected", foundMsg); + assertTrue("error is not a GrammarSemanticsMessage", + foundMsg instanceof GrammarSemanticsMessage); + assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs())); + if (equeue.size() != 1) { + System.err.println(equeue); + } + } + + protected void checkError(ErrorQueue equeue, + ANTLRMessage expectedMessage) + throws Exception { + //System.out.println("errors="+equeue); + ANTLRMessage foundMsg = null; + for (int i = 0; i < equeue.errors.size(); i++) { + ANTLRMessage m = equeue.errors.get(i); + if (m.getErrorType() == expectedMessage.getErrorType()) { + foundMsg = m; + } + } + assertTrue("no error; " + expectedMessage.getErrorType() + " expected", !equeue.errors.isEmpty()); + assertTrue("too many errors; " + equeue.errors, equeue.errors.size() <= 1); + assertNotNull("couldn't find expected error: " + expectedMessage.getErrorType(), foundMsg); + /* + * assertTrue("error is not a GrammarSemanticsMessage", foundMsg + * instanceof GrammarSemanticsMessage); + */ + assertArrayEquals(expectedMessage.getArgs(), foundMsg.getArgs()); + } + + public static class FilteringTokenStream extends CommonTokenStream { + public FilteringTokenStream(TokenSource src) { + super(src); + } + + Set hide = new HashSet(); + + @Override + protected boolean sync(int i) { + if (!super.sync(i)) { + return false; + } + + Token t = get(i); + if (hide.contains(t.getType())) { + ((WritableToken) t).setChannel(Token.HIDDEN_CHANNEL); + } + + return true; + } + + public void setTokenTypeChannel(int ttype, int channel) { + hide.add(ttype); + } + } + + protected void writeTestFile(String parserName, + String lexerName, + String parserStartRuleName, + boolean debug, + boolean profile) { + ST outputFileST = new ST( + "import 'package:antlr4/antlr4.dart';\n" + + "\n" + + "import '.dart';\n" + + "import '.dart';\n" + + "\n" + + "void main(List args) async {\n" + + " CharStream input = await InputStream.fromPath(args[0]);\n" + + " lex = new (input);\n" + + " CommonTokenStream tokens = new CommonTokenStream(lex);\n" + + " \n" + + " parser.buildParseTree = true;\n" + + " \n" + + " ParserRuleContext tree = parser.();\n" + + " print('[${profiler.getDecisionInfo().join(', ')}]');\n" + + " ParseTreeWalker.DEFAULT.walk(new TreeShapeListener(), tree);\n" + + "}\n" + + "\n" + + "class TreeShapeListener implements ParseTreeListener {\n" + + " @override void visitTerminal(TerminalNode node) {}\n" + + "\n" + + " @override void visitErrorNode(ErrorNode node) {}\n" + + "\n" + + " @override void exitEveryRule(ParserRuleContext ctx) {}\n" + + "\n" + + " @override\n" + + " void enterEveryRule(ParserRuleContext ctx) {\n" + + " for (int i = 0; i \\< ctx.childCount; i++) {\n" + + " ParseTree parent = ctx.getChild(i).parent;\n" + + " if (!(parent is RuleNode) || (parent as RuleNode).ruleContext != ctx) {\n" + + " throw new StateError(\"Invalid parse tree shape detected.\");\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n" + ); + ST createParserST = new ST(" parser = new (tokens);\n"); + if (debug) { + createParserST = + new ST( + " parser = new (tokens);\n" + + " parser.addErrorListener(new DiagnosticErrorListener());\n"); + } + if (profile) { + outputFileST.add("profile", + "ProfilingATNSimulator profiler = new ProfilingATNSimulator(parser);\n" + + "parser.setInterpreter(profiler);"); + } else { + outputFileST.add("profile", new ArrayList()); + } + outputFileST.add("createParser", createParserST); + outputFileST.add("parserName", parserName); + outputFileST.add("lexerName", lexerName); + outputFileST.add("parserStartRuleName", parserStartRuleName); + writeFile(tmpdir, "Test.dart", outputFileST.render()); + } + + protected void writeLexerTestFile(String lexerName, boolean showDFA) { + ST outputFileST = new ST( + "import 'dart:io';\n" + + "\n" + + "import 'package:antlr4/antlr4.dart';\n" + + "\n" + + "import '.dart';\n" + + "\n" + + "void main(List args) async {\n" + + " CharStream input = await InputStream.fromPath(args[0]);\n" + + " lex = new (input);\n" + + " CommonTokenStream tokens = new CommonTokenStream(lex);\n" + + " tokens.fill();\n" + + " for (Object t in tokens.getTokens())\n" + + " print(t);\n" + + "\n" + + (showDFA ? "stdout.write(lex.interpreter.getDFA(Lexer.DEFAULT_MODE).toLexerString());\n" : "") + + "}\n" + ); + + outputFileST.add("lexerName", lexerName); + writeFile(tmpdir, "Test.dart", outputFileST.render()); + } + + public void writeRecognizerAndCompile(String parserName, String lexerName, + String parserStartRuleName, + boolean debug, + boolean profile) { + if (parserName == null) { + writeLexerTestFile(lexerName, debug); + } else { + writeTestFile(parserName, + lexerName, + parserStartRuleName, + debug, + profile); + } + } + + protected void eraseFiles(final String filesEndingWith) { + File tmpdirF = new File(tmpdir); + String[] files = tmpdirF.list(); + for (int i = 0; files != null && i < files.length; i++) { + if (files[i].endsWith(filesEndingWith)) { + new File(tmpdir + "/" + files[i]).delete(); + } + } + } + + protected void eraseFiles() { + if (tmpdir == null) { + return; + } + + File tmpdirF = new File(tmpdir); + String[] files = tmpdirF.list(); + for (int i = 0; files != null && i < files.length; i++) { + new File(tmpdir + "/" + files[i]).delete(); + } + } + + @Override + public void eraseTempDir() { + File tmpdirF = new File(tmpdir); + if (tmpdirF.exists()) { + eraseFiles(); + tmpdirF.delete(); + } + } + + public String getFirstLineOfException() { + if (this.stderrDuringParse == null) { + return null; + } + String[] lines = this.stderrDuringParse.split("\n"); + String prefix = "Exception in thread \"main\" "; + return lines[0].substring(prefix.length(), lines[0].length()); + } + + /** + * When looking at a result set that consists of a Map/HashTable + * we cannot rely on the output order, as the hashing algorithm or other aspects + * of the implementation may be different on differnt JDKs or platforms. Hence + * we take the Map, convert the keys to a List, sort them and Stringify the Map, which is a + * bit of a hack, but guarantees that we get the same order on all systems. We assume that + * the keys are strings. + * + * @param m The Map that contains keys we wish to return in sorted order + * @return A string that represents all the keys in sorted order. + */ + public String sortMapToString(Map m) { + // Pass in crap, and get nothing back + // + if (m == null) { + return null; + } + + System.out.println("Map toString looks like: " + m.toString()); + + // Sort the keys in the Map + // + TreeMap nset = new TreeMap(m); + + System.out.println("Tree map looks like: " + nset.toString()); + return nset.toString(); + } + + public List realElements(List elements) { + return elements.subList(Token.MIN_USER_TOKEN_TYPE, elements.size()); + } + + public void assertNotNullOrEmpty(String message, String text) { + assertNotNull(message, text); + assertFalse(message, text.isEmpty()); + } + + public void assertNotNullOrEmpty(String text) { + assertNotNull(text); + assertFalse(text.isEmpty()); + } + + public static class IntTokenStream implements TokenStream { + public IntegerList types; + int p = 0; + + public IntTokenStream(IntegerList types) { + this.types = types; + } + + @Override + public void consume() { + p++; + } + + @Override + public int LA(int i) { + return LT(i).getType(); + } + + @Override + public int mark() { + return index(); + } + + @Override + public int index() { + return p; + } + + @Override + public void release(int marker) { + seek(marker); + } + + @Override + public void seek(int index) { + p = index; + } + + @Override + public int size() { + return types.size(); + } + + @Override + public String getSourceName() { + return UNKNOWN_SOURCE_NAME; + } + + @Override + public Token LT(int i) { + CommonToken t; + int rawIndex = p + i - 1; + if (rawIndex >= types.size()) t = new CommonToken(Token.EOF); + else t = new CommonToken(types.get(rawIndex)); + t.setTokenIndex(rawIndex); + return t; + } + + @Override + public Token get(int i) { + return new CommonToken(types.get(i)); + } + + @Override + public TokenSource getTokenSource() { + return null; + } + + @Override + public String getText() { + throw new UnsupportedOperationException("can't give strings"); + } + + @Override + public String getText(Interval interval) { + throw new UnsupportedOperationException("can't give strings"); + } + + @Override + public String getText(RuleContext ctx) { + throw new UnsupportedOperationException("can't give strings"); + } + + @Override + public String getText(Token start, Token stop) { + throw new UnsupportedOperationException("can't give strings"); + } + } + + /** + * Sort a list + */ + public > List sort(List data) { + List dup = new ArrayList(); + dup.addAll(data); + Collections.sort(dup); + return dup; + } + + /** + * Return map sorted by key + */ + public , V> LinkedHashMap sort(Map data) { + LinkedHashMap dup = new LinkedHashMap(); + List keys = new ArrayList(); + keys.addAll(data.keySet()); + Collections.sort(keys); + for (K k : keys) { + dup.put(k, data.get(k)); + } + return dup; + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestCompositeLexers.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestCompositeLexers.java new file mode 100644 index 000000000..60aa4a35a --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestCompositeLexers.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.CompositeLexersDescriptors; +import org.antlr.v4.test.runtime.dart.BaseDartTest; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestCompositeLexers extends BaseRuntimeTest { + public TestCompositeLexers(RuntimeTestDescriptor descriptor) { + super(descriptor,new BaseDartTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(CompositeLexersDescriptors.class, "Dart"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestCompositeParsers.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestCompositeParsers.java new file mode 100644 index 000000000..638413f9a --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestCompositeParsers.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.CompositeParsersDescriptors; +import org.antlr.v4.test.runtime.dart.BaseDartTest; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestCompositeParsers extends BaseRuntimeTest { + public TestCompositeParsers(RuntimeTestDescriptor descriptor) { + super(descriptor,new BaseDartTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(CompositeParsersDescriptors.class, "Dart"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestFullContextParsing.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestFullContextParsing.java new file mode 100644 index 000000000..a0d7f9c1a --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestFullContextParsing.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.FullContextParsingDescriptors; +import org.antlr.v4.test.runtime.dart.BaseDartTest; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestFullContextParsing extends BaseRuntimeTest { + public TestFullContextParsing(RuntimeTestDescriptor descriptor) { + super(descriptor,new BaseDartTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(FullContextParsingDescriptors.class, "Dart"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestLeftRecursion.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestLeftRecursion.java new file mode 100644 index 000000000..e92f1b306 --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestLeftRecursion.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.LeftRecursionDescriptors; +import org.antlr.v4.test.runtime.dart.BaseDartTest; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestLeftRecursion extends BaseRuntimeTest { + public TestLeftRecursion(RuntimeTestDescriptor descriptor) { + super(descriptor,new BaseDartTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(LeftRecursionDescriptors.class, "Dart"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestLexerErrors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestLexerErrors.java new file mode 100644 index 000000000..b95cd59bf --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestLexerErrors.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.LexerErrorsDescriptors; +import org.antlr.v4.test.runtime.dart.BaseDartTest; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestLexerErrors extends BaseRuntimeTest { + public TestLexerErrors(RuntimeTestDescriptor descriptor) { + super(descriptor,new BaseDartTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(LexerErrorsDescriptors.class, "Dart"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestLexerExec.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestLexerExec.java new file mode 100644 index 000000000..1ed1d84ca --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestLexerExec.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.LexerExecDescriptors; +import org.antlr.v4.test.runtime.dart.BaseDartTest; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestLexerExec extends BaseRuntimeTest { + public TestLexerExec(RuntimeTestDescriptor descriptor) { + super(descriptor,new BaseDartTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(LexerExecDescriptors.class, "Dart"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestListeners.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestListeners.java new file mode 100644 index 000000000..e15dee9ae --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestListeners.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.ListenersDescriptors; +import org.antlr.v4.test.runtime.dart.BaseDartTest; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestListeners extends BaseRuntimeTest { + public TestListeners(RuntimeTestDescriptor descriptor) { + super(descriptor,new BaseDartTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(ListenersDescriptors.class, "Dart"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestParseTrees.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestParseTrees.java new file mode 100644 index 000000000..0115e384d --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestParseTrees.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.ParseTreesDescriptors; +import org.antlr.v4.test.runtime.dart.BaseDartTest; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestParseTrees extends BaseRuntimeTest { + public TestParseTrees(RuntimeTestDescriptor descriptor) { + super(descriptor,new BaseDartTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(ParseTreesDescriptors.class, "Dart"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestParserErrors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestParserErrors.java new file mode 100644 index 000000000..87b850fb9 --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestParserErrors.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.ParserErrorsDescriptors; +import org.antlr.v4.test.runtime.dart.BaseDartTest; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestParserErrors extends BaseRuntimeTest { + public TestParserErrors(RuntimeTestDescriptor descriptor) { + super(descriptor,new BaseDartTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(ParserErrorsDescriptors.class, "Dart"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestParserExec.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestParserExec.java new file mode 100644 index 000000000..c22aa8ceb --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestParserExec.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.ParserExecDescriptors; +import org.antlr.v4.test.runtime.dart.BaseDartTest; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestParserExec extends BaseRuntimeTest { + public TestParserExec(RuntimeTestDescriptor descriptor) { + super(descriptor,new BaseDartTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(ParserExecDescriptors.class, "Dart"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestPerformance.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestPerformance.java new file mode 100644 index 000000000..78e6942ac --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestPerformance.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.PerformanceDescriptors; +import org.antlr.v4.test.runtime.dart.BaseDartTest; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestPerformance extends BaseRuntimeTest { + public TestPerformance(RuntimeTestDescriptor descriptor) { + super(descriptor,new BaseDartTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(PerformanceDescriptors.class, "Dart"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestSemPredEvalLexer.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestSemPredEvalLexer.java new file mode 100644 index 000000000..8825042cf --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestSemPredEvalLexer.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.SemPredEvalLexerDescriptors; +import org.antlr.v4.test.runtime.dart.BaseDartTest; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestSemPredEvalLexer extends BaseRuntimeTest { + public TestSemPredEvalLexer(RuntimeTestDescriptor descriptor) { + super(descriptor,new BaseDartTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(SemPredEvalLexerDescriptors.class, "Dart"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestSemPredEvalParser.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestSemPredEvalParser.java new file mode 100644 index 000000000..87d6a9dea --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestSemPredEvalParser.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.SemPredEvalParserDescriptors; +import org.antlr.v4.test.runtime.dart.BaseDartTest; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestSemPredEvalParser extends BaseRuntimeTest { + public TestSemPredEvalParser(RuntimeTestDescriptor descriptor) { + super(descriptor,new BaseDartTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(SemPredEvalParserDescriptors.class, "Dart"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestSets.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestSets.java new file mode 100644 index 000000000..4fe603fd4 --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/TestSets.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.runtime.dart; + +import org.antlr.v4.test.runtime.BaseRuntimeTest; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; +import org.antlr.v4.test.runtime.descriptors.SetsDescriptors; +import org.antlr.v4.test.runtime.dart.BaseDartTest; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestSets extends BaseRuntimeTest { + public TestSets(RuntimeTestDescriptor descriptor) { + super(descriptor,new BaseDartTest()); + } + + @Parameterized.Parameters(name="{0}") + public static RuntimeTestDescriptor[] getAllTestDescriptors() { + return BaseRuntimeTest.getRuntimeTestDescriptors(SetsDescriptors.class, "Dart"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/PerformanceDescriptors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/PerformanceDescriptors.java index 2c2702fe5..01e0bd4a1 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/PerformanceDescriptors.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/PerformanceDescriptors.java @@ -113,7 +113,7 @@ public class PerformanceDescriptors { @Override public boolean ignore(String targetName) { - return !Arrays.asList("Java", "CSharp", "Python2", "Python3", "Node", "Cpp", "Swift").contains(targetName); + return !Arrays.asList("Java", "CSharp", "Python2", "Python3", "Node", "Cpp", "Swift", "Dart").contains(targetName); } } diff --git a/runtime/Dart/.gitignore b/runtime/Dart/.gitignore new file mode 100644 index 000000000..a45b12ec7 --- /dev/null +++ b/runtime/Dart/.gitignore @@ -0,0 +1,23 @@ +!lib + +# See https://www.dartlang.org/guides/libraries/private-files + +# Files and directories created by pub +.dart_tool/ +.packages +build/ +# If you're building an application, you may want to check-in your pubspec.lock +pubspec.lock + +# Directory created by dartdoc +# If you don't generate documentation locally you can remove this line. +doc/api/ + +# Avoid committing generated Javascript files: +*.dart.js +*.info.json # Produced by the --dump-info flag. +*.js # When generated by dart2js. Don't specify *.js if your + # project includes source files written in JavaScript. +*.js_ +*.js.deps +*.js.map \ No newline at end of file diff --git a/runtime/Dart/LICENSE.txt b/runtime/Dart/LICENSE.txt new file mode 100644 index 000000000..2042d1bda --- /dev/null +++ b/runtime/Dart/LICENSE.txt @@ -0,0 +1,52 @@ +[The "BSD 3-clause license"] +Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +===== + +MIT License for codepointat.js from https://git.io/codepointat +MIT License for fromcodepoint.js from https://git.io/vDW1m + +Copyright Mathias Bynens + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/runtime/Dart/README.md b/runtime/Dart/README.md new file mode 100644 index 000000000..b386f0ba1 --- /dev/null +++ b/runtime/Dart/README.md @@ -0,0 +1,11 @@ +# Dart target for ANTLR 4 + +Dart runtime libraries for ANTLR 4 + +This runtime is available through [pub](https://pub.dev). The package name is 'antlr4'. + +See www.antlr.org for more information on ANTLR. + +See https://github.com/antlr/antlr4/blob/master/doc/Dart-target.md for more information on using ANTLR in Dart. + + diff --git a/runtime/Dart/lib/antlr4.dart b/runtime/Dart/lib/antlr4.dart new file mode 100644 index 000000000..0a4b4f60e --- /dev/null +++ b/runtime/Dart/lib/antlr4.dart @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +library antlr4; + +export 'src/atn/atn.dart'; +export 'src/dfa/dfa.dart'; +export 'src/tree/tree.dart'; +export 'src/error/error.dart'; +export 'src/rule_context.dart'; +export 'src/input_stream.dart'; +export 'src/token_stream.dart'; +export 'src/lexer.dart'; +export 'src/parser.dart'; +export 'src/parser_rule_context.dart'; +export 'src/vocabulary.dart'; +export 'src/runtime_meta_data.dart'; +export 'src/token.dart'; diff --git a/runtime/Dart/lib/src/atn/atn.dart b/runtime/Dart/lib/src/atn/atn.dart new file mode 100644 index 000000000..a0400f41d --- /dev/null +++ b/runtime/Dart/lib/src/atn/atn.dart @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +export 'src/atn.dart'; +export 'src/atn_config.dart'; +export 'src/atn_config_set.dart'; +export 'src/atn_deserializer.dart'; +export 'src/atn_simulator.dart'; +export 'src/atn_state.dart'; +export 'src/info.dart'; +export 'src/lexer_action_executor.dart'; +export 'src/lexer_atn_simulator.dart'; +export 'src/parser_atn_simulator.dart'; +export 'src/profiling_atn_simulator.dart'; +export 'src/transition.dart'; diff --git a/runtime/Dart/lib/src/atn/src/atn.dart b/runtime/Dart/lib/src/atn/src/atn.dart new file mode 100644 index 000000000..fd881ce4f --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/atn.dart @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:collection'; + +import '../../interval_set.dart'; +import '../../ll1_analyzer.dart'; +import '../../rule_context.dart'; +import '../../token.dart'; +import 'atn_state.dart'; +import 'atn_type.dart'; +import 'lexer_action.dart'; +import 'transition.dart'; + +class ATN { + static final INVALID_ALT_NUMBER = 0; + + List states = []; + + /** Each subrule/rule is a decision point and we must track them so we + * can go back later and build DFA predictors for them. This includes + * all the rules, subrules, optional blocks, ()+, ()* etc... + */ + List decisionToState = []; + + /** + * Maps from rule index to starting state number. + */ + List ruleToStartState; + + /** + * Maps from rule index to stop state number. + */ + List ruleToStopState; + + Map modeNameToStartState = LinkedHashMap(); + + /** + * The type of the ATN. + */ + final ATNType grammarType; + + /** + * The maximum value for any symbol recognized by a transition in the ATN. + */ + final int maxTokenType; + + /** + * For lexer ATNs, this maps the rule index to the resulting token type. + * For parser ATNs, this maps the rule index to the generated bypass token + * type if the + * {@link ATNDeserializationOptions#isGenerateRuleBypassTransitions} + * deserialization option was specified; otherwise, this is null. + */ + List ruleToTokenType; + + /** + * For lexer ATNs, this is an array of [LexerAction] objects which may + * be referenced by action transitions in the ATN. + */ + List lexerActions; + + List modeToStartState = []; + + /** Used for runtime deserialization of ATNs from strings */ + ATN(this.grammarType, this.maxTokenType); + + /** + * TODO merge doc comment + * Compute the set of valid tokens that can occur starting in state [s]. + * If [ctx] is null, the set of tokens will not include what can follow + * the rule surrounding [s]. In other words, the set will be + * restricted to tokens reachable staying within [s]'s rule. + * + * Compute the set of valid tokens that can occur starting in [s] and + * staying in same rule. {@link Token#EPSILON} is in set if we reach end of + * rule. + */ + IntervalSet nextTokens(ATNState s, [RuleContext ctx]) { + if (ctx != null) { + return LL1Analyzer(this).LOOK(s, ctx); + } + if (s.nextTokenWithinRule != null) return s.nextTokenWithinRule; + s.nextTokenWithinRule = LL1Analyzer(this).LOOK(s, null); + s.nextTokenWithinRule.setReadonly(true); + return s.nextTokenWithinRule; + } + + void addState(ATNState state) { + if (state != null) { + state.atn = this; + state.stateNumber = states.length; + } + + states.add(state); + } + + void removeState(ATNState state) { + states[state.stateNumber] = + null; // just free mem, don't shift states in list + } + + int defineDecisionState(DecisionState s) { + decisionToState.add(s); + s.decision = decisionToState.length - 1; + return s.decision; + } + + DecisionState getDecisionState(int decision) { + if (!decisionToState.isEmpty) { + return decisionToState[decision]; + } + return null; + } + + int get numberOfDecisions { + return decisionToState.length; + } + + /** + * Computes the set of input symbols which could follow ATN state number + * [stateNumber] in the specified full [context]. This method + * considers the complete parser context, but does not evaluate semantic + * predicates (i.e. all predicates encountered during the calculation are + * assumed true). If a path in the ATN exists from the starting state to the + * [RuleStopState] of the outermost context without matching any + * symbols, {@link Token#EOF} is added to the returned set. + * + *

If [context] is null, it is treated as {@link ParserRuleContext#EMPTY}.

+ * + * Note that this does NOT give you the set of all tokens that could + * appear at a given token position in the input phrase. In other words, + * it does not answer: + * + * "Given a specific partial input phrase, return the set of all tokens + * that can follow the last token in the input phrase." + * + * The big difference is that with just the input, the parser could + * land right in the middle of a lookahead decision. Getting + * all *possible* tokens given a partial input stream is a separate + * computation. See https://github.com/antlr/antlr4/issues/1428 + * + * For this function, we are specifying an ATN state and call stack to compute + * what token(s) can come next and specifically: outside of a lookahead decision. + * That is what you want for error reporting and recovery upon parse error. + * + * @param stateNumber the ATN state number + * @param context the full parse context + * @return The set of potentially valid input symbols which could follow the + * specified state in the specified context. + * @throws IllegalArgumentException if the ATN does not contain a state with + * number [stateNumber] + */ + IntervalSet getExpectedTokens(int stateNumber, RuleContext context) { + if (stateNumber < 0 || stateNumber >= states.length) { + throw new RangeError.index(stateNumber, states, "stateNumber"); + } + + RuleContext ctx = context; + ATNState s = states[stateNumber]; + IntervalSet following = nextTokens(s); + if (!following.contains(Token.EPSILON)) { + return following; + } + + IntervalSet expected = new IntervalSet(); + expected.addAll(following); + expected.remove(Token.EPSILON); + while (ctx != null && + ctx.invokingState >= 0 && + following.contains(Token.EPSILON)) { + ATNState invokingState = states[ctx.invokingState]; + RuleTransition rt = invokingState.transition(0); + following = nextTokens(rt.followState); + expected.addAll(following); + expected.remove(Token.EPSILON); + ctx = ctx.parent; + } + + if (following.contains(Token.EPSILON)) { + expected.addOne(Token.EOF); + } + + return expected; + } +} diff --git a/runtime/Dart/lib/src/atn/src/atn_config.dart b/runtime/Dart/lib/src/atn/src/atn_config.dart new file mode 100644 index 000000000..70baa25c9 --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/atn_config.dart @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import '../../prediction_context.dart'; +import '../../recognizer.dart'; +import '../../util/murmur_hash.dart'; +import 'atn_state.dart'; +import 'lexer_action_executor.dart'; +import 'semantic_context.dart'; + +checkParams(params, isCfg) { + if (params == null) { + Map result = { + "state": null, + "alt": null, + "context": null, + "semanticContext": null + }; + if (isCfg) { + result["reachesIntoOuterContext"] = 0; + } + return result; + } else { + Map props = {}; + props["state"] = params.state ?? null; + props["alt"] = (params.alt == null) ? null : params.alt; + props["context"] = params.context ?? null; + props["semanticContext"] = params.semanticContext ?? null; + if (isCfg) { + props["reachesIntoOuterContext"] = params.reachesIntoOuterContext ?? 0; + props["precedenceFilterSuppressed"] = + params.precedenceFilterSuppressed ?? false; + } + return props; + } +} + +/** A tuple: (ATN state, predicted alt, syntactic, semantic context). + * The syntactic context is a graph-structured stack node whose + * path(s) to the root is the rule invocation(s) + * chain used to arrive at the state. The semantic context is + * the tree of semantic predicates encountered before reaching + * an ATN state. + */ +class ATNConfig { + /** + * This field stores the bit mask for implementing the + * {@link #isPrecedenceFilterSuppressed} property as a bit within the + * existing {@link #reachesIntoOuterContext} field. + */ + static final int SUPPRESS_PRECEDENCE_FILTER = 0x40000000; + + /** The ATN state associated with this configuration */ + ATNState state; + + /** What alt (or lexer rule) is predicted by this configuration */ + int alt; + + /** The stack of invoking states leading to the rule/states associated + * with this config. We track only those contexts pushed during + * execution of the ATN simulator. + */ + PredictionContext context; + + /** + * We cannot execute predicates dependent upon local context unless + * we know for sure we are in the correct context. Because there is + * no way to do this efficiently, we simply cannot evaluate + * dependent predicates unless we are in the rule that initially + * invokes the ATN simulator. + * + *

+ * closure() tracks the depth of how far we dip into the outer context: + * depth > 0. Note that it may not be totally accurate depth since I + * don't ever decrement. TODO: make it a bool then

+ * + *

+ * For memory efficiency, the {@link #isPrecedenceFilterSuppressed} method + * is also backed by this field. Since the field is ly accessible, the + * highest bit which would not cause the value to become negative is used to + * store this field. This choice minimizes the risk that code which only + * compares this value to 0 would be affected by the new purpose of the + * flag. It also ensures the performance of the existing [ATNConfig] + * constructors as well as certain operations like + * {@link ATNConfigSet#add(ATNConfig, DoubleKeyMap)} method are + * completely unaffected by the change.

+ */ + int reachesIntoOuterContext = 0; + + SemanticContext semanticContext; + + ATNConfig(this.state, this.alt, this.context, + [this.semanticContext = SemanticContext.NONE]); + + ATNConfig.dup(ATNConfig c, + {this.state, this.alt, this.context, this.semanticContext}) { + this.state = state ?? c.state; + this.alt = alt ?? c.alt; + this.context = context ?? c.context; + this.semanticContext = semanticContext ?? c.semanticContext; + this.reachesIntoOuterContext = + c.reachesIntoOuterContext ?? reachesIntoOuterContext; + } + + /** + * This method gets the value of the {@link #reachesIntoOuterContext} field + * as it existed prior to the introduction of the + * {@link #isPrecedenceFilterSuppressed} method. + */ + int get outerContextDepth { + return reachesIntoOuterContext & ~SUPPRESS_PRECEDENCE_FILTER; + } + + bool isPrecedenceFilterSuppressed() { + return (reachesIntoOuterContext & SUPPRESS_PRECEDENCE_FILTER) != 0; + } + + void setPrecedenceFilterSuppressed(bool value) { + if (value) { + this.reachesIntoOuterContext |= 0x40000000; + } else { + this.reachesIntoOuterContext &= ~SUPPRESS_PRECEDENCE_FILTER; + } + } + + /** An ATN configuration is equal to another if both have + * the same state, they predict the same alternative, and + * syntactic/semantic contexts are the same. + */ + operator ==(Object other) { + if (other is ATNConfig && other != null) { + return this.state.stateNumber == other.state.stateNumber && + this.alt == other.alt && + (this.context == other.context || + (this.context != null && this.context == other.context)) && + this.semanticContext == other.semanticContext && + this.isPrecedenceFilterSuppressed() == + other.isPrecedenceFilterSuppressed(); + } + return false; + } + + get hashCode { + int hashCode = MurmurHash.initialize(7); + hashCode = MurmurHash.update(hashCode, state.stateNumber); + hashCode = MurmurHash.update(hashCode, alt); + hashCode = MurmurHash.update(hashCode, context); + hashCode = MurmurHash.update(hashCode, semanticContext); + hashCode = MurmurHash.finish(hashCode, 4); + return hashCode; + } + + String toString([Recognizer recog, bool showAlt = true]) { + StringBuffer buf = new StringBuffer(); + // if ( state.ruleIndex>=0 ) { + // if ( recog!=null ) buf.write(recog.ruleNames[state.ruleIndex]+":"); + // else buf.write(state.ruleIndex+":"); + // } + buf.write('('); + buf.write(state); + if (showAlt) { + buf.write(","); + buf.write(alt); + } + if (context != null) { + buf.write(",["); + buf.write(context.toString()); + buf.write("]"); + } + if (semanticContext != null && semanticContext != SemanticContext.NONE) { + buf.write(","); + buf.write(semanticContext); + } + if (outerContextDepth > 0) { + buf.write(",up="); + buf.write(outerContextDepth); + } + buf.write(')'); + return buf.toString(); + } +} + +class LexerATNConfig extends ATNConfig { + /** + * Gets the [LexerActionExecutor] capable of executing the embedded + * action(s) for the current configuration. + */ + LexerActionExecutor lexerActionExecutor; + + bool passedThroughNonGreedyDecision = false; + + LexerATNConfig(ATNState state, int alt, PredictionContext context, + [this.lexerActionExecutor]) + : super(state, alt, context, SemanticContext.NONE) { + this.passedThroughNonGreedyDecision = false; + } + + LexerATNConfig.dup(LexerATNConfig c, ATNState state, + {this.lexerActionExecutor, PredictionContext context}) + : super.dup(c, state: state, context: context) { + this.lexerActionExecutor = lexerActionExecutor ?? c.lexerActionExecutor; + this.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state); + } + + bool hasPassedThroughNonGreedyDecision() { + return passedThroughNonGreedyDecision; + } + + int get hashCode { + int hashCode = MurmurHash.initialize(7); + hashCode = MurmurHash.update(hashCode, state.stateNumber); + hashCode = MurmurHash.update(hashCode, alt); + hashCode = MurmurHash.update(hashCode, context); + hashCode = MurmurHash.update(hashCode, semanticContext); + hashCode = + MurmurHash.update(hashCode, passedThroughNonGreedyDecision ? 1 : 0); + hashCode = MurmurHash.update(hashCode, lexerActionExecutor); + hashCode = MurmurHash.finish(hashCode, 6); + return hashCode; + } + + bool operator ==(Object other) { + if (identical(this, other)) { + return true; + } else if (other is LexerATNConfig) { + LexerATNConfig lexerOther = other; + if (passedThroughNonGreedyDecision != + lexerOther.passedThroughNonGreedyDecision) { + return false; + } + + if (lexerActionExecutor != lexerOther.lexerActionExecutor) { + return false; + } + + return super == other; + } + return false; + } + + static bool checkNonGreedyDecision(LexerATNConfig source, ATNState target) { + return source.passedThroughNonGreedyDecision || + target is DecisionState && target.nonGreedy; + } +} diff --git a/runtime/Dart/lib/src/atn/src/atn_config_set.dart b/runtime/Dart/lib/src/atn/src/atn_config_set.dart new file mode 100644 index 000000000..58385543f --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/atn_config_set.dart @@ -0,0 +1,281 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:collection'; +import 'dart:math'; + +import 'package:collection/collection.dart'; + +import '../../misc/pair.dart'; +import '../../prediction_context.dart'; +import '../../util/bit_set.dart'; +import '../../util/utils.dart'; +import 'atn.dart'; +import 'atn_config.dart'; +import 'atn_state.dart'; +import 'semantic_context.dart'; + +class ATNConfigSet extends Iterable { + /** + * Indicates that the set of configurations is read-only. Do not + * allow any code to manipulate the set; DFA states will point at + * the sets and they must not change. This does not protect the other + * fields; in particular, conflictingAlts is set after + * we've made this readonly. + */ + bool _readOnly = false; + + bool get readOnly => _readOnly; + + set readOnly(bool readOnly) { + this._readOnly = readOnly; + if (readOnly) { + this.configLookup = null; // can't mod, no need for lookup cache + } + } + + /// The reason that we need this is because we don't want the hash map to use + /// the standard hash code and equals. We need all configurations with the same + /// {@code (s,i,_,semctx)} to be equal. Unfortunately, this key effectively doubles + /// the number of objects associated with ATNConfigs. The other solution is to + /// use a hash table that lets us specify the equals/hashcode operation. + /// + /// All configs but hashed by (s, i, _, pi) not including context. Wiped out + /// when we go readonly as this set becomes a DFA state. + Set configLookup = new HashSet(equals: (a, b) { + if (a == null || b == null) return false; + return a.state.stateNumber == b.state.stateNumber && + a.alt == b.alt && + a.semanticContext == b.semanticContext; + }, hashCode: (ATNConfig o) { + int hashCode = 7; + hashCode = 31 * hashCode + o.state.stateNumber; + hashCode = 31 * hashCode + o.alt; + hashCode = 31 * hashCode + o.semanticContext.hashCode; + return hashCode; + }); + + /** Track the elements as they are added to the set; supports get(i) */ + final List configs = []; + + // TODO: these fields make me pretty uncomfortable but nice to pack up info together, saves recomputation + // TODO: can we track conflicts as they are added to save scanning configs later? + int uniqueAlt = 0; + + /** + * Currently this is only used when we detect SLL conflict; this does + * not necessarily represent the ambiguous alternatives. In fact, + * I should also point out that this seems to include predicated alternatives + * that have predicates that evaluate to false. Computed in computeTargetState(). + */ + BitSet conflictingAlts; + + // Used in parser and lexer. In lexer, it indicates we hit a pred + // while computing a closure operation. Don't make a DFA state from this. + bool hasSemanticContext = false; + bool dipsIntoOuterContext = false; + + /** Indicates that this configuration set is part of a full context + * LL prediction. It will be used to determine how to merge $. With SLL + * it's a wildcard whereas it is not for LL context merge. + */ + bool fullCtx; + + int cachedHashCode = -1; + + ATNConfigSet([this.fullCtx = true]); + + ATNConfigSet.dup(ATNConfigSet old) { + this.fullCtx = old.fullCtx; + addAll(old); + this.uniqueAlt = old.uniqueAlt; + this.conflictingAlts = old.conflictingAlts; + this.hasSemanticContext = old.hasSemanticContext; + this.dipsIntoOuterContext = old.dipsIntoOuterContext; + } + + /** + * Adding a new config means merging contexts with existing configs for + * {@code (s, i, pi, _)}, where [s] is the + * {@link ATNConfig#state}, [i] is the {@link ATNConfig#alt}, and + * [pi] is the {@link ATNConfig#semanticContext}. We use + * {@code (s,i,pi)} as key. + * + *

This method updates {@link #dipsIntoOuterContext} and + * {@link #hasSemanticContext} when necessary.

+ */ + bool add(ATNConfig config, + [Map, PredictionContext> + mergeCache = null]) { + if (readOnly) throw new StateError("This set is readonly"); + if (config.semanticContext != SemanticContext.NONE) { + hasSemanticContext = true; + } + if (config.outerContextDepth > 0) { + dipsIntoOuterContext = true; + } + final existing = configLookup.lookup(config) ?? config; + if (identical(existing, config)) { + // we added this new one + cachedHashCode = -1; + configLookup.add(config); + configs.add(config); // track order here + return true; + } + // a previous (s,i,pi,_), merge with it and save result + bool rootIsWildcard = !fullCtx; + PredictionContext merged = PredictionContext.merge( + existing.context, config.context, rootIsWildcard, mergeCache); + // no need to check for existing.context, config.context in cache + // since only way to create new graphs is "call rule" and here. We + // cache at both places. + existing.reachesIntoOuterContext = + max(existing.reachesIntoOuterContext, config.reachesIntoOuterContext); + + // make sure to preserve the precedence filter suppression during the merge + if (config.isPrecedenceFilterSuppressed()) { + existing.setPrecedenceFilterSuppressed(true); + } + + existing.context = merged; // replace context; no need to alt mapping + return true; + } + + /** Return a List holding list of configs */ + List get elements { + return configs; + } + + Set get states { + var states = new Set(); + for (var i = 0; i < this.configs.length; i++) { + states.add(this.configs[i].state); + } + return states; + } + + /** + * Gets the complete set of represented alternatives for the configuration + * set. + * + * @return the set of represented alternatives in this configuration set + * + * @since 4.3 + */ + BitSet get alts { + BitSet alts = new BitSet(); + for (ATNConfig config in configs) { + alts.set(config.alt); + } + return alts; + } + + List get predicates { + List preds = []; + for (ATNConfig c in configs) { + if (c.semanticContext != SemanticContext.NONE) { + preds.add(c.semanticContext); + } + } + return preds; + } + + ATNConfig get(int i) { + return configs[i]; + } + + optimizeConfigs(interpreter) { + if (this.readOnly) throw StateError("This set is readonly"); + + if (this.configLookup.isEmpty) return; + + for (ATNConfig config in configs) { +// int before = PredictionContext.getAllContextNodes(config.context).length; + config.context = interpreter.getCachedContext(config.context); +// int after = PredictionContext.getAllContextNodes(config.context).length; +// System.out.println("configs "+before+"->"+after); + } + } + + addAll(coll) { + for (ATNConfig c in coll) add(c); + return false; + } + + bool operator ==(other) { + return identical(this, other) || + (other is ATNConfigSet && + other != null && + ListEquality().equals(this.configs, other.configs) && + this.fullCtx == other.fullCtx && + this.uniqueAlt == other.uniqueAlt && + this.conflictingAlts == other.conflictingAlts && + this.hasSemanticContext == other.hasSemanticContext && + this.dipsIntoOuterContext == other.dipsIntoOuterContext); + } + + int get hashCode { + if (readOnly) { + if (cachedHashCode == -1) { + cachedHashCode = ListEquality().hash(configs); + } + + return cachedHashCode; + } + + return ListEquality().hash(configs); + } + + int get length { + return configs.length; + } + + bool get isEmpty => configs.isEmpty; + + updateHashCode(hash) { + if (this.readOnly) { + if (this.cachedHashCode == -1) { + this.cachedHashCode = this.hashCode; + } + hash.update(this.cachedHashCode); + } else { + hash.update(this.hashCode); + } + } + + bool contains(Object o) { + if (configLookup == null) { + throw new UnsupportedError( + "This method is not implemented for readonly sets."); + } + + return configLookup.contains(o); + } + + Iterator get iterator => configs.iterator; + + clear() { + if (readOnly) throw StateError("This set is readonly"); + configs.clear(); + cachedHashCode = -1; + configLookup.clear(); + } + + String toString() { + final buf = new StringBuffer(); + buf.write(arrayToString(elements)); + if (hasSemanticContext) + buf.write(",hasSemanticContext=$hasSemanticContext"); + if (uniqueAlt != ATN.INVALID_ALT_NUMBER) buf.write(",uniqueAlt=$uniqueAlt"); + if (conflictingAlts != null) buf.write(",conflictingAlts=$conflictingAlts"); + if (dipsIntoOuterContext) buf.write(",dipsIntoOuterContext"); + return buf.toString(); + } +} + +class OrderedATNConfigSet extends ATNConfigSet { + final configLookup = Set(); +} diff --git a/runtime/Dart/lib/src/atn/src/atn_deserializer.dart b/runtime/Dart/lib/src/atn/src/atn_deserializer.dart new file mode 100644 index 000000000..9d48780d1 --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/atn_deserializer.dart @@ -0,0 +1,823 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import '../../interval_set.dart'; +import '../../misc/pair.dart'; +import '../../token.dart'; +import 'atn.dart'; +import 'atn_state.dart'; +import 'atn_type.dart'; +import 'lexer_action.dart'; +import 'transition.dart'; + +class ATNDeserializationOptions { + static final ATNDeserializationOptions defaultOptions = + ATNDeserializationOptions()..makeReadOnly(); + + bool readOnly; + bool verifyATN; + bool generateRuleBypassTransitions; + + ATNDeserializationOptions([ATNDeserializationOptions options]) { + if (options == null) { + this.verifyATN = true; + this.generateRuleBypassTransitions = false; + } else { + this.verifyATN = options.verifyATN; + this.generateRuleBypassTransitions = + options.generateRuleBypassTransitions; + } + } + + bool isReadOnly() { + return readOnly; + } + + void makeReadOnly() { + readOnly = true; + } + + bool isVerifyATN() { + return verifyATN; + } + + void setVerifyATN(bool verifyATN) { + throwIfReadOnly(); + this.verifyATN = verifyATN; + } + + bool isGenerateRuleBypassTransitions() { + return generateRuleBypassTransitions; + } + + void setGenerateRuleBypassTransitions(bool generateRuleBypassTransitions) { + throwIfReadOnly(); + this.generateRuleBypassTransitions = generateRuleBypassTransitions; + } + + void throwIfReadOnly() { + if (isReadOnly()) { + throw new StateError("The object is read only."); + } + } +} + +class ATNDeserializer { + /** This value should never change. Updates following this version are + * reflected as change in the unique ID SERIALIZED_UUID. + */ + static final SERIALIZED_VERSION = 3; + + /** WARNING: DO NOT MERGE THESE LINES. If UUIDs differ during a merge, + * resolve the conflict by generating a new ID! + */ + /** + * This is the earliest supported serialized UUID. + */ + static final BASE_SERIALIZED_UUID = "33761B2D-78BB-4A43-8B0B-4F5BEE8AACF3"; + + /** + * This UUID indicates an extension of {@link BASE_SERIALIZED_UUID} for the + * addition of precedence predicates. + */ + static final ADDED_PRECEDENCE_TRANSITIONS = + "1DA0C57D-6C06-438A-9B27-10BCB3CE0F61"; + + /** + * This UUID indicates an extension of {@link #ADDED_PRECEDENCE_TRANSITIONS} + * for the addition of lexer actions encoded as a sequence of + * [LexerAction] instances. + */ + static final ADDED_LEXER_ACTIONS = "AADB8D7E-AEEF-4415-AD2B-8204D6CF042E"; + + /** + * This UUID indicates the serialized ATN contains two sets of + * IntervalSets, where the second set's values are encoded as + * 32-bit integers to support the full Unicode SMP range up to U+10FFFF. + */ + static final ADDED_UNICODE_SMP = "59627784-3BE5-417A-B9EB-8131A7286089"; + + /** + * This list contains all of the currently supported UUIDs, ordered by when + * the feature first appeared in this branch. + */ + static final SUPPORTED_UUIDS = [ + BASE_SERIALIZED_UUID, + ADDED_PRECEDENCE_TRANSITIONS, + ADDED_LEXER_ACTIONS, + ADDED_UNICODE_SMP + ]; + + /** + * This is the current serialized UUID. + */ + static final SERIALIZED_UUID = ADDED_UNICODE_SMP; + + ATNDeserializationOptions deserializationOptions; + List data; + var pos; + String uuid; + + ATNDeserializer([options = null]) { + this.deserializationOptions = + options ?? ATNDeserializationOptions.defaultOptions; + } + +// Determines if a particular serialized representation of an ATN supports +// a particular feature, identified by the [UUID] used for serializing +// the ATN at the time the feature was first introduced. +// +// @param feature The [UUID] marking the first time the feature was +// supported in the serialized ATN. +// @param actualUuid The [UUID] of the actual serialized ATN which is +// currently being deserialized. +// @return [true] if the [actualUuid] value represents a +// serialized ATN at or after the feature identified by [feature] was +// introduced; otherwise, [false]. + + isFeatureSupported(feature, actualUuid) { + var idx1 = SUPPORTED_UUIDS.indexOf(feature); + if (idx1 < 0) { + return false; + } + var idx2 = SUPPORTED_UUIDS.indexOf(actualUuid); + return idx2 >= idx1; + } + + deserialize(List data) { + this.reset(data); + this.checkVersion(); + this.checkUUID(); + final ATN atn = this.readATN(); + this.readStates(atn); + this.readRules(atn); + this.readModes(atn); + var sets = List(); + // First, deserialize sets with 16-bit arguments <= U+FFFF. + this.readSets(atn, sets, () => this.readInt()); + // Next, if the ATN was serialized with the Unicode SMP feature, + // deserialize sets with 32-bit arguments <= U+10FFFF. + if (this.isFeatureSupported(ADDED_UNICODE_SMP, this.uuid)) { + this.readSets(atn, sets, () => this.readInt32()); + } + this.readEdges(atn, sets); + this.readDecisions(atn); + this.readLexerActions(atn); + this.markPrecedenceDecisions(atn); + this.verifyATN(atn); + if (this.deserializationOptions.generateRuleBypassTransitions && + atn.grammarType == ATNType.PARSER) { + this.generateRuleBypassTransitions(atn); + // re-verify after modification + this.verifyATN(atn); + } + return atn; + } + + /// Each char value in data is shifted by +2 at the entry to this method. + /// This is an encoding optimization targeting the serialized values 0 + /// and -1 (serialized to 0xFFFF), each of which are very common in the + /// serialized form of the ATN. In the modified UTF-8 that Java uses for + /// compiled string literals, these two character values have multi-byte + /// forms. By shifting each value by +2, they become characters 2 and 1 + /// prior to writing the string, each of which have single-byte + /// representations. Since the shift occurs in the tool during ATN + /// serialization, each target is responsible for adjusting the values + /// during deserialization. + /// + /// As a special case, note that the first element of data is not + /// adjusted because it contains the major version number of the + /// serialized ATN, which was fixed at 3 at the time the value shifting + /// was implemented. + reset(List data) { + var adjust = (int c) { + var v = c; + return v > 1 ? v - 2 : v + 65534; + }; + final temp = data.map(adjust).toList(); + // don't adjust the first value since that's the version number + temp[0] = data[0]; + this.data = temp; + this.pos = 0; + } + + checkVersion() { + var version = this.readInt(); + if (version != SERIALIZED_VERSION) { + throw ("Could not deserialize ATN with version $version (expected $SERIALIZED_VERSION)."); + } + } + + checkUUID() { + var uuid = this.readUUID(); + if (SUPPORTED_UUIDS.indexOf(uuid) < 0) { + throw ("Could not deserialize ATN with UUID: $uuid (expected $SERIALIZED_UUID or a legacy UUID)."); + } + this.uuid = uuid; + } + + ATN readATN() { + var grammarType = this.readInt(); + var maxTokenType = this.readInt(); + return new ATN(ATNType.values[grammarType], maxTokenType); + } + + readStates(ATN atn) { + List> loopBackStateNumbers = []; + List> endStateNumbers = []; + int nstates = this.readInt(); + for (int i = 0; i < nstates; i++) { + StateType stype = StateType.values[readInt()]; + // ignore bad type of states + if (stype == StateType.INVALID_TYPE) { + atn.addState(null); + continue; + } + + int ruleIndex = readInt(); + if (ruleIndex == 0xFFFF) { + ruleIndex = -1; + } + + ATNState s = stateFactory(stype, ruleIndex); + if (s is LoopEndState) { + // special case + int loopBackStateNumber = readInt(); + loopBackStateNumbers.add(Pair(s, loopBackStateNumber)); + } else if (s is BlockStartState) { + int endStateNumber = readInt(); + endStateNumbers.add(new Pair(s, endStateNumber)); + } + atn.addState(s); + } + + // delay the assignment of loop back and end states until we know all the state instances have been initialized + for (final pair in loopBackStateNumbers) { + pair.a.loopBackState = atn.states[pair.b]; + } + + for (final pair in endStateNumbers) { + pair.a.endState = atn.states[pair.b] as BlockEndState; + } + + int numNonGreedyStates = readInt(); + for (int i = 0; i < numNonGreedyStates; i++) { + int stateNumber = readInt(); + (atn.states[stateNumber] as DecisionState).nonGreedy = true; + } + if (this.isFeatureSupported(ADDED_PRECEDENCE_TRANSITIONS, this.uuid)) { + int numPrecedenceStates = readInt(); + for (int i = 0; i < numPrecedenceStates; i++) { + int stateNumber = readInt(); + (atn.states[stateNumber] as RuleStartState).isLeftRecursiveRule = true; + } + } + } + + readRules(ATN atn) { + int nrules = readInt(); + if (atn.grammarType == ATNType.LEXER) { + atn.ruleToTokenType = new List(nrules); + } + + atn.ruleToStartState = new List(nrules); + for (int i = 0; i < nrules; i++) { + int s = readInt(); + RuleStartState startState = atn.states[s]; + atn.ruleToStartState[i] = startState; + if (atn.grammarType == ATNType.LEXER) { + int tokenType = readInt(); + if (tokenType == 0xFFFF) { + tokenType = Token.EOF; + } + + atn.ruleToTokenType[i] = tokenType; + + if (!isFeatureSupported(ADDED_LEXER_ACTIONS, uuid)) { + // this piece of unused metadata was serialized prior to the + // addition of LexerAction + int actionIndexIgnored = readInt(); + } + } + } + + atn.ruleToStopState = new List(nrules); + for (ATNState state in atn.states) { + if (!(state is RuleStopState)) { + continue; + } + + RuleStopState stopState = state; + atn.ruleToStopState[state.ruleIndex] = stopState; + atn.ruleToStartState[state.ruleIndex].stopState = stopState; + } + } + + readModes(ATN atn) { + int nmodes = readInt(); + for (int i = 0; i < nmodes; i++) { + int s = readInt(); + atn.modeToStartState.add(atn.states[s] as TokensStartState); + } + } + + readSets(ATN atn, List sets, readUnicode) { + int nsets = readInt(); + for (int i = 0; i < nsets; i++) { + int nintervals = readInt(); + IntervalSet set = new IntervalSet(); + sets.add(set); + + bool containsEof = readInt() != 0; + if (containsEof) { + set.addOne(-1); + } + + for (int j = 0; j < nintervals; j++) { + int a = readUnicode(); + int b = readUnicode(); + set.addRange(a, b); + } + } + } + + readEdges(ATN atn, sets) { + int nedges = readInt(); + for (int i = 0; i < nedges; i++) { + int src = readInt(); + int trg = readInt(); + TransitionType ttype = TransitionType.values[readInt()]; + int arg1 = readInt(); + int arg2 = readInt(); + int arg3 = readInt(); + Transition trans = + edgeFactory(atn, ttype, src, trg, arg1, arg2, arg3, sets); +// System.out.println("EDGE "+trans.getClass().getSimpleName()+" "+ +// src+"->"+trg+ +// " "+Transition.serializationNames[ttype]+ +// " "+arg1+","+arg2+","+arg3); + ATNState srcState = atn.states[src]; + srcState.addTransition(trans); + } + + // edges for rule stop states can be derived, so they aren't serialized + for (ATNState state in atn.states) { + for (int i = 0; i < state.numberOfTransitions; i++) { + Transition t = state.transition(i); + if (t is RuleTransition) { + final ruleTransition = t; + int outermostPrecedenceReturn = -1; + if (atn.ruleToStartState[ruleTransition.target.ruleIndex] + .isLeftRecursiveRule) { + if (ruleTransition.precedence == 0) { + outermostPrecedenceReturn = ruleTransition.target.ruleIndex; + } + } + + EpsilonTransition returnTransition = new EpsilonTransition( + ruleTransition.followState, outermostPrecedenceReturn); + atn.ruleToStopState[ruleTransition.target.ruleIndex] + .addTransition(returnTransition); + } + } + } + + for (ATNState state in atn.states) { + if (state is BlockStartState) { + // we need to know the end state to set its start state + if (state.endState == null) { + throw new StateError(""); + } + + // block end states can only be associated to a single block start state + if (state.endState.startState != null) { + throw new StateError(""); + } + + state.endState.startState = state; + } + + if (state is PlusLoopbackState) { + PlusLoopbackState loopbackState = state; + for (int i = 0; i < loopbackState.numberOfTransitions; i++) { + ATNState target = loopbackState.transition(i).target; + if (target is PlusBlockStartState) { + target.loopBackState = loopbackState; + } + } + } else if (state is StarLoopbackState) { + StarLoopbackState loopbackState = state; + for (int i = 0; i < loopbackState.numberOfTransitions; i++) { + ATNState target = loopbackState.transition(i).target; + if (target is StarLoopEntryState) { + target.loopBackState = loopbackState; + } + } + } + } + } + + readDecisions(ATN atn) { + int ndecisions = this.readInt(); + for (int i = 1; i <= ndecisions; i++) { + int s = this.readInt(); + DecisionState decState = atn.states[s]; + atn.decisionToState.add(decState); + decState.decision = i - 1; + } + } + + readLexerActions(ATN atn) { + if (atn.grammarType == ATNType.LEXER) { + if (isFeatureSupported(ADDED_LEXER_ACTIONS, this.uuid)) { + atn.lexerActions = new List(readInt()); + for (int i = 0; i < atn.lexerActions.length; i++) { + LexerActionType actionType = LexerActionType.values[readInt()]; + int data1 = readInt(); + if (data1 == 0xFFFF) { + data1 = -1; + } + + int data2 = readInt(); + if (data2 == 0xFFFF) { + data2 = -1; + } + LexerAction lexerAction = + lexerActionFactory(actionType, data1, data2); + + atn.lexerActions[i] = lexerAction; + } + } else { + // for compatibility with older serialized ATNs, convert the old + // serialized action index for action transitions to the new + // form, which is the index of a LexerCustomAction + List legacyLexerActions = []; + for (ATNState state in atn.states) { + for (int i = 0; i < state.numberOfTransitions; i++) { + Transition transition = state.transition(i); + if (transition is ActionTransition) { + int ruleIndex = transition.ruleIndex; + int actionIndex = transition.actionIndex; + LexerCustomAction lexerAction = + new LexerCustomAction(ruleIndex, actionIndex); + state.setTransition( + i, + new ActionTransition(transition.target, ruleIndex, + legacyLexerActions.length, false)); + legacyLexerActions.add(lexerAction); + } + } + } + + atn.lexerActions = legacyLexerActions; + } + } + } + + generateRuleBypassTransitions(ATN atn) { + for (int i = 0; i < atn.ruleToStartState.length; i++) { + atn.ruleToTokenType[i] = atn.maxTokenType + i + 1; + } + for (int i = 0; i < atn.ruleToStartState.length; i++) { + this.generateRuleBypassTransition(atn, i); + } + } + + generateRuleBypassTransition(ATN atn, int idx) { + BasicBlockStartState bypassStart = new BasicBlockStartState(); + bypassStart.ruleIndex = idx; + atn.addState(bypassStart); + + BlockEndState bypassStop = new BlockEndState(); + bypassStop.ruleIndex = idx; + atn.addState(bypassStop); + + bypassStart.endState = bypassStop; + atn.defineDecisionState(bypassStart); + + bypassStop.startState = bypassStart; + + ATNState endState; + Transition excludeTransition = null; + if (atn.ruleToStartState[idx].isLeftRecursiveRule) { + // wrap from the beginning of the rule to the StarLoopEntryState + endState = null; + for (ATNState state in atn.states) { + if (state.ruleIndex != idx) { + continue; + } + + if (!(state is StarLoopEntryState)) { + continue; + } + + ATNState maybeLoopEndState = + state.transition(state.numberOfTransitions - 1).target; + if (!(maybeLoopEndState is LoopEndState)) { + continue; + } + + if (maybeLoopEndState.epsilonOnlyTransitions && + maybeLoopEndState.transition(0).target is RuleStopState) { + endState = state; + break; + } + } + + if (endState == null) { + throw new UnsupportedError( + "Couldn't identify final state of the precedence rule prefix section."); + } + + excludeTransition = + (endState as StarLoopEntryState).loopBackState.transition(0); + } else { + endState = atn.ruleToStopState[idx]; + } + + // all non-excluded transitions that currently target end state need to target blockEnd instead + for (ATNState state in atn.states) { + for (Transition transition in state.transitions) { + if (transition == excludeTransition) { + continue; + } + + if (transition.target == endState) { + transition.target = bypassStop; + } + } + } + + // all transitions leaving the rule start state need to leave blockStart instead + while (atn.ruleToStartState[idx].numberOfTransitions > 0) { + Transition transition = atn.ruleToStartState[idx].removeTransition( + atn.ruleToStartState[idx].numberOfTransitions - 1); + bypassStart.addTransition(transition); + } + + // link the new states + atn.ruleToStartState[idx].addTransition(new EpsilonTransition(bypassStart)); + bypassStop.addTransition(new EpsilonTransition(endState)); + + ATNState matchState = new BasicState(); + atn.addState(matchState); + matchState.addTransition( + new AtomTransition(bypassStop, atn.ruleToTokenType[idx])); + bypassStart.addTransition(new EpsilonTransition(matchState)); + } + + /** + * Analyze the [StarLoopEntryState] states in the specified ATN to set + * the {@link StarLoopEntryState#isPrecedenceDecision} field to the + * correct value. + * + * @param atn The ATN. + */ + markPrecedenceDecisions(ATN atn) { + for (ATNState state in atn.states) { + if (state is StarLoopEntryState) { + /* We analyze the ATN to determine if this ATN decision state is the + * decision for the closure block that determines whether a + * precedence rule should continue or complete. + */ + if (atn.ruleToStartState[state.ruleIndex].isLeftRecursiveRule) { + ATNState maybeLoopEndState = + state.transition(state.numberOfTransitions - 1).target; + if (maybeLoopEndState is LoopEndState) { + if (maybeLoopEndState.epsilonOnlyTransitions && + maybeLoopEndState.transition(0).target is RuleStopState) { + state.isPrecedenceDecision = true; + } + } + } + } + } + } + + void verifyATN(ATN atn) { + // verify assumptions + for (ATNState state in atn.states) { + if (state == null) { + continue; + } + + checkCondition(state.onlyHasEpsilonTransitions() || + state.numberOfTransitions <= 1); + + if (state is PlusBlockStartState) { + checkCondition(state.loopBackState != null); + } + + if (state is StarLoopEntryState) { + StarLoopEntryState starLoopEntryState = state; + checkCondition(starLoopEntryState.loopBackState != null); + checkCondition(starLoopEntryState.numberOfTransitions == 2); + + if (starLoopEntryState.transition(0).target is StarBlockStartState) { + checkCondition( + starLoopEntryState.transition(1).target is LoopEndState); + checkCondition(!starLoopEntryState.nonGreedy); + } else if (starLoopEntryState.transition(0).target is LoopEndState) { + checkCondition( + starLoopEntryState.transition(1).target is StarBlockStartState); + checkCondition(starLoopEntryState.nonGreedy); + } else { + throw new StateError(""); + } + } + + if (state is StarLoopbackState) { + checkCondition(state.numberOfTransitions == 1); + checkCondition(state.transition(0).target is StarLoopEntryState); + } + + if (state is LoopEndState) { + checkCondition(state.loopBackState != null); + } + + if (state is RuleStartState) { + checkCondition(state.stopState != null); + } + + if (state is BlockStartState) { + checkCondition(state.endState != null); + } + + if (state is BlockEndState) { + checkCondition(state.startState != null); + } + + if (state is DecisionState) { + DecisionState decisionState = state; + checkCondition(decisionState.numberOfTransitions <= 1 || + decisionState.decision >= 0); + } else { + checkCondition( + state.numberOfTransitions <= 1 || state is RuleStopState); + } + } + } + + void checkCondition(bool condition, [String message = ""]) { + if (!condition) { + throw new StateError(message); + } + } + + int readInt() { + return this.data[this.pos++]; + } + + readInt32() { + var low = this.readInt(); + var high = this.readInt(); + return low | (high << 16); + } + + readLong() { + var low = this.readInt32(); + var high = this.readInt32(); + return (low & 0x00000000FFFFFFFF) | (high << 32); + } + + static final byteToHex = List.generate(256, (i) => i.toRadixString(16).padLeft(2, '0').toUpperCase()); + + readUUID() { + final bb = List(16); + for (var i = 7; i >= 0; i--) { + var int = this.readInt(); + /* jshint bitwise: false */ + bb[(2 * i) + 1] = int & 0xFF; + bb[2 * i] = (int >> 8) & 0xFF; + } + return byteToHex[bb[0]] + byteToHex[bb[1]] + + byteToHex[bb[2]] + byteToHex[bb[3]] + '-' + + byteToHex[bb[4]] + byteToHex[bb[5]] + '-' + + byteToHex[bb[6]] + byteToHex[bb[7]] + '-' + + byteToHex[bb[8]] + byteToHex[bb[9]] + '-' + + byteToHex[bb[10]] + byteToHex[bb[11]] + + byteToHex[bb[12]] + byteToHex[bb[13]] + + byteToHex[bb[14]] + byteToHex[bb[15]];; + } + + Transition edgeFactory(ATN atn, TransitionType type, int src, int trg, + int arg1, int arg2, int arg3, List sets) { + ATNState target = atn.states[trg]; + switch (type) { + case TransitionType.EPSILON: + return EpsilonTransition(target); + case TransitionType.RANGE: + return arg3 != 0 + ? RangeTransition(target, Token.EOF, arg2) + : RangeTransition(target, arg1, arg2); + case TransitionType.RULE: + RuleTransition rt = + new RuleTransition(atn.states[arg1], arg2, arg3, target); + return rt; + case TransitionType.PREDICATE: + PredicateTransition pt = + new PredicateTransition(target, arg1, arg2, arg3 != 0); + return pt; + case TransitionType.PRECEDENCE: + return new PrecedencePredicateTransition(target, arg1); + case TransitionType.ATOM: + return arg3 != 0 + ? AtomTransition(target, Token.EOF) + : new AtomTransition(target, arg1); + case TransitionType.ACTION: + ActionTransition a = + new ActionTransition(target, arg1, arg2, arg3 != 0); + return a; + case TransitionType.SET: + return new SetTransition(target, sets[arg1]); + case TransitionType.NOT_SET: + return new NotSetTransition(target, sets[arg1]); + case TransitionType.WILDCARD: + return new WildcardTransition(target); + case TransitionType.INVALID: + throw ArgumentError.value(type, "transition type", "not valid."); + } + } + + ATNState stateFactory(StateType type, int ruleIndex) { + ATNState s; + switch (type) { + case StateType.INVALID_TYPE: + return null; + case StateType.BASIC: + s = new BasicState(); + break; + case StateType.RULE_START: + s = new RuleStartState(); + break; + case StateType.BLOCK_START: + s = new BasicBlockStartState(); + break; + case StateType.PLUS_BLOCK_START: + s = new PlusBlockStartState(); + break; + case StateType.STAR_BLOCK_START: + s = new StarBlockStartState(); + break; + case StateType.TOKEN_START: + s = new TokensStartState(); + break; + case StateType.RULE_STOP: + s = new RuleStopState(); + break; + case StateType.BLOCK_END: + s = new BlockEndState(); + break; + case StateType.STAR_LOOP_BACK: + s = new StarLoopbackState(); + break; + case StateType.STAR_LOOP_ENTRY: + s = new StarLoopEntryState(); + break; + case StateType.PLUS_LOOP_BACK: + s = new PlusLoopbackState(); + break; + case StateType.LOOP_END: + s = new LoopEndState(); + break; + default: + throw ArgumentError.value(type, "state type", "not valid."); + } + + s.ruleIndex = ruleIndex; + return s; + } + + LexerAction lexerActionFactory(LexerActionType type, int data1, int data2) { + switch (type) { + case LexerActionType.CHANNEL: + return new LexerChannelAction(data1); + + case LexerActionType.CUSTOM: + return new LexerCustomAction(data1, data2); + + case LexerActionType.MODE: + return new LexerModeAction(data1); + + case LexerActionType.MORE: + return LexerMoreAction.INSTANCE; + + case LexerActionType.POP_MODE: + return LexerPopModeAction.INSTANCE; + + case LexerActionType.PUSH_MODE: + return new LexerPushModeAction(data1); + + case LexerActionType.SKIP: + return LexerSkipAction.INSTANCE; + + case LexerActionType.TYPE: + return new LexerTypeAction(data1); + default: + throw ArgumentError.value(type, "lexer action type", "not valid."); + } + } +} diff --git a/runtime/Dart/lib/src/atn/src/atn_simulator.dart b/runtime/Dart/lib/src/atn/src/atn_simulator.dart new file mode 100644 index 000000000..f24abf121 --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/atn_simulator.dart @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import '../../dfa/dfa.dart'; +import '../../prediction_context.dart'; +import 'atn.dart'; +import 'atn_config_set.dart'; + +abstract class ATNSimulator { + /** Must distinguish between missing edge and edge we know leads nowhere */ + + static final DFAState ERROR = + DFAState(stateNumber: 0x7FFFFFFF, configs: new ATNConfigSet()); + + final ATN atn; + + /** The context cache maps all PredictionContext objects that are equals() + * to a single cached copy. This cache is shared across all contexts + * in all ATNConfigs in all DFA states. We rebuild each ATNConfigSet + * to use only cached nodes/graphs in addDFAState(). We don't want to + * fill this during closure() since there are lots of contexts that + * pop up but are not used ever again. It also greatly slows down closure(). + * + *

This cache makes a huge difference in memory and a little bit in speed. + * For the Java grammar on java.*, it dropped the memory requirements + * at the end from 25M to 16M. We don't store any of the full context + * graphs in the DFA because they are limited to local context only, + * but apparently there's a lot of repetition there as well. We optimize + * the config contexts before storing the config set in the DFA states + * by literally rebuilding them with cached subgraphs only.

+ * + *

I tried a cache for use during closure operations, that was + * whacked after each adaptivePredict(). It cost a little bit + * more time I think and doesn't save on the overall footprint + * so it's not worth the complexity.

+ */ + final PredictionContextCache sharedContextCache; + + ATNSimulator(this.atn, this.sharedContextCache); + + void reset(); + + /** + * Clear the DFA cache used by the current instance. Since the DFA cache may + * be shared by multiple ATN simulators, this method may affect the + * performance (but not accuracy) of other parsers which are being used + * concurrently. + * + * @throws UnsupportedOperationException if the current instance does not + * support clearing the DFA. + * + * @since 4.3 + */ + void clearDFA() { + throw new UnsupportedError( + "This ATN simulator does not support clearing the DFA."); + } + + PredictionContext getCachedContext(PredictionContext context) { + if (sharedContextCache == null) return context; + + Map visited = + new Map(); + return PredictionContext.getCachedContext( + context, sharedContextCache, visited); + } +} + +/** Used to cache [PredictionContext] objects. Its used for the shared + * context cash associated with contexts in DFA states. This cache + * can be used for both lexers and parsers. + */ +class PredictionContextCache { + final cache = new Map(); + + /** Add a context to the cache and return it. If the context already exists, + * return that one instead and do not add a new context to the cache. + * Protect shared cache from unsafe thread access. + */ + PredictionContext add(PredictionContext ctx) { + if (ctx == PredictionContext.EMPTY) return PredictionContext.EMPTY; + PredictionContext existing = cache[ctx]; + if (existing != null) { +// System.out.println(name+" reuses "+existing); + return existing; + } + cache[ctx] = ctx; + return ctx; + } + + PredictionContext operator [](PredictionContext ctx) { + return cache[ctx]; + } + + int get length { + return cache.length; + } +} diff --git a/runtime/Dart/lib/src/atn/src/atn_state.dart b/runtime/Dart/lib/src/atn/src/atn_state.dart new file mode 100644 index 000000000..98f10f4ed --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/atn_state.dart @@ -0,0 +1,298 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:developer'; + +import 'package:logging/logging.dart'; + +import '../../interval_set.dart'; +import 'atn.dart'; +import 'transition.dart'; + +var INITIAL_NUM_TRANSITIONS = 4; + +enum StateType { + INVALID_TYPE, + BASIC, + RULE_START, + BLOCK_START, + PLUS_BLOCK_START, + STAR_BLOCK_START, + TOKEN_START, + RULE_STOP, + BLOCK_END, + STAR_LOOP_BACK, + STAR_LOOP_ENTRY, + PLUS_LOOP_BACK, + LOOP_END, +} + +/** + * The following images show the relation of states and + * {@link ATNState#transitions} for various grammar constructs. + * + *
    + * + *
  • Solid edges marked with an ε indicate a required + * [EpsilonTransition].
    • + * + *
  • Dashed edges indicate locations where any transition derived from + * [Transition] might appear.
    • + * + *
  • Dashed nodes are place holders for either a sequence of linked + * [BasicState] states or the inclusion of a block representing a nested + * construct in one of the forms below.
    • + * + *
  • Nodes showing multiple outgoing alternatives with a {@code ...} support + * any number of alternatives (one or more). Nodes without the {@code ...} only + * support the exact number of alternatives shown in the diagram.
    • + * + *
+ * + *

Basic Blocks

+ * + *

Rule

+ * + * + * + *

Block of 1 or more alternatives

+ * + * + * + *

Greedy Loops

+ * + *

Greedy Closure: {@code (...)*}

+ * + * + * + *

Greedy Positive Closure: {@code (...)+}

+ * + * + * + *

Greedy Optional: {@code (...)?}

+ * + * + * + *

Non-Greedy Loops

+ * + *

Non-Greedy Closure: {@code (...)*?}

+ * + * + * + *

Non-Greedy Positive Closure: {@code (...)+?}

+ * + * + * + *

Non-Greedy Optional: {@code (...)??}

+ * + * + */ +abstract class ATNState { + static final int INITIAL_NUM_TRANSITIONS = 4; + + static final int INVALID_STATE_NUMBER = -1; + + /** Which ATN are we in? */ + ATN atn = null; + + int stateNumber = INVALID_STATE_NUMBER; + + int ruleIndex; // at runtime, we don't have Rule objects + + bool epsilonOnlyTransitions = false; + + /** Track the transitions emanating from this ATN state. */ + List transitions = []; + + /** Used to cache lookahead during parsing, not used during construction */ + IntervalSet nextTokenWithinRule; + + int get hashCode { + return stateNumber; + } + + bool operator ==(Object o) { + // are these states same object? + if (o is ATNState) return stateNumber == o.stateNumber; + return false; + } + + bool isNonGreedyExitState() { + return false; + } + + String toString() { + return stateNumber.toString(); + } + + int get numberOfTransitions { + return transitions.length; + } + + void addTransition(Transition e) { + addTransitionAt(transitions.length, e); + } + + void addTransitionAt(int index, Transition e) { + if (transitions.isEmpty) { + epsilonOnlyTransitions = e.isEpsilon; + } else if (epsilonOnlyTransitions != e.isEpsilon) { + log("ATN state $stateNumber has both epsilon and non-epsilon transitions.\n", + level: Level.SEVERE.value); + epsilonOnlyTransitions = false; + } + + bool alreadyPresent = false; + for (Transition t in transitions) { + if (t.target.stateNumber == e.target.stateNumber) { + if (t.label != null && e.label != null && t.label == e.label) { +// System.err.println("Repeated transition upon "+e.label()+" from "+stateNumber+"->"+t.target.stateNumber); + alreadyPresent = true; + break; + } else if (t.isEpsilon && e.isEpsilon) { +// System.err.println("Repeated epsilon transition from "+stateNumber+"->"+t.target.stateNumber); + alreadyPresent = true; + break; + } + } + } + if (!alreadyPresent) { + transitions.insert(index, e); + } + } + + Transition transition(int i) { + return transitions[i]; + } + + void setTransition(int i, Transition e) { + transitions[i] = e; + } + + Transition removeTransition(int index) { + return transitions.removeAt(index); + } + + StateType get stateType; + + bool onlyHasEpsilonTransitions() => epsilonOnlyTransitions; + + void setRuleIndex(int ruleIndex) { + this.ruleIndex = ruleIndex; + } +} + +class BasicState extends ATNState { + @override + StateType get stateType => StateType.BASIC; +} + +class RuleStartState extends ATNState { + var stopState = null; + var isLeftRecursiveRule = false; + + @override + StateType get stateType => StateType.RULE_START; +} + +abstract class DecisionState extends ATNState { + int decision = 0; + bool nonGreedy = false; +} + +// The start of a regular {@code (...)} block. +abstract class BlockStartState extends DecisionState { + BlockEndState endState; +} + +class BasicBlockStartState extends BlockStartState { + @override + StateType get stateType => StateType.BLOCK_START; +} + +/** Start of {@code (A|B|...)+} loop. Technically a decision state, but + * we don't use for code generation; somebody might need it, so I'm defining + * it for completeness. In reality, the [PlusLoopbackState] node is the + * real decision-making note for {@code A+}. + */ +class PlusBlockStartState extends BlockStartState { + PlusLoopbackState loopBackState; + + @override + StateType get stateType => StateType.PLUS_BLOCK_START; +} + +/// The block that begins a closure loop. +class StarBlockStartState extends BlockStartState { + @override + StateType get stateType => StateType.STAR_BLOCK_START; +} + +/// The Tokens rule start state linking to each lexer rule start state */ +class TokensStartState extends DecisionState { + @override + StateType get stateType => StateType.TOKEN_START; +} + +/// The last node in the ATN for a rule, unless that rule is the start symbol. +/// In that case, there is one transition to EOF. Later, we might encode +/// references to all calls to this rule to compute FOLLOW sets for +/// error handling. +class RuleStopState extends ATNState { + @override + StateType get stateType => StateType.RULE_STOP; +} + +/// Terminal node of a simple {@code (a|b|c)} block. +class BlockEndState extends ATNState { + BlockStartState startState = null; + + @override + StateType get stateType => StateType.BLOCK_END; +} + +class StarLoopbackState extends ATNState { + StarLoopEntryState get loopEntryState { + return transition(0).target; + } + + @override + StateType get stateType => StateType.STAR_LOOP_BACK; +} + +class StarLoopEntryState extends DecisionState { + StarLoopbackState loopBackState; + + /** + * Indicates whether this state can benefit from a precedence DFA during SLL + * decision making. + * + *

This is a computed property that is calculated during ATN deserialization + * and stored for use in [ParserATNSimulator] and + * [ParserInterpreter].

+ * + * @see DFA#isPrecedenceDfa() + */ + bool isPrecedenceDecision = false; + + @override + StateType get stateType => StateType.STAR_LOOP_ENTRY; +} + +/// Decision state for {@code A+} and {@code (A|B)+}. It has two transitions: +/// one to the loop back to start of the block and one to exit. +class PlusLoopbackState extends DecisionState { + @override + StateType get stateType => StateType.PLUS_LOOP_BACK; +} + +/// Mark the end of a * or + loop. +class LoopEndState extends ATNState { + ATNState loopBackState; + + @override + StateType get stateType => StateType.LOOP_END; +} diff --git a/runtime/Dart/lib/src/atn/src/atn_type.dart b/runtime/Dart/lib/src/atn/src/atn_type.dart new file mode 100644 index 000000000..1b7eb8f2e --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/atn_type.dart @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +/// Represents the type of recognizer an ATN applies to. +enum ATNType { + /** + * A lexer grammar. + */ + LEXER, + + /** + * A parser grammar. + */ + PARSER +} diff --git a/runtime/Dart/lib/src/atn/src/info.dart b/runtime/Dart/lib/src/atn/src/info.dart new file mode 100644 index 000000000..50fba1e8e --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/info.dart @@ -0,0 +1,666 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import '../../dfa/dfa.dart'; +import '../../token_stream.dart'; +import '../../util/bit_set.dart'; +import 'atn_config_set.dart'; +import 'profiling_atn_simulator.dart'; +import 'semantic_context.dart'; + +/** + * This class represents profiling event information for a context sensitivity. + * Context sensitivities are decisions where a particular input resulted in an + * SLL conflict, but LL prediction produced a single unique alternative. + * + *

+ * In some cases, the unique alternative identified by LL prediction is not + * equal to the minimum represented alternative in the conflicting SLL + * configuration set. Grammars and inputs which result in this scenario are + * unable to use {@link PredictionMode#SLL}, which in turn means they cannot use + * the two-stage parsing strategy to improve parsing performance for that + * input.

+ * + * @see ParserATNSimulator#reportContextSensitivity + * @see ANTLRErrorListener#reportContextSensitivity + * + * @since 4.3 + */ +class ContextSensitivityInfo extends DecisionEventInfo { + /** + * Constructs a new instance of the [ContextSensitivityInfo] class + * with the specified detailed context sensitivity information. + * + * @param decision The decision number + * @param configs The final configuration set containing the unique + * alternative identified by full-context prediction + * @param input The input token stream + * @param startIndex The start index for the current prediction + * @param stopIndex The index at which the context sensitivity was + * identified during full-context prediction + */ + ContextSensitivityInfo(int decision, ATNConfigSet configs, TokenStream input, + int startIndex, int stopIndex) + : super(decision, configs, input, startIndex, stopIndex, true); +} + +/** + * This is the base class for gathering detailed information about prediction + * events which occur during parsing. + * + * Note that we could record the parser call stack at the time this event + * occurred but in the presence of left recursive rules, the stack is kind of + * meaningless. It's better to look at the individual configurations for their + * individual stacks. Of course that is a [PredictionContext] object + * not a parse tree node and so it does not have information about the extent + * (start...stop) of the various subtrees. Examining the stack tops of all + * configurations provide the return states for the rule invocations. + * From there you can get the enclosing rule. + * + * @since 4.3 + */ +class DecisionEventInfo { + /** + * The invoked decision number which this event is related to. + * + * @see ATN#decisionToState + */ + final int decision; + + /** + * The configuration set containing additional information relevant to the + * prediction state when the current event occurred, or null if no + * additional information is relevant or available. + */ + final ATNConfigSet configs; + + /** + * The input token stream which is being parsed. + */ + final TokenStream input; + + /** + * The token index in the input stream at which the current prediction was + * originally invoked. + */ + final int startIndex; + + /** + * The token index in the input stream at which the current event occurred. + */ + final int stopIndex; + + /** + * [true] if the current event occurred during LL prediction; + * otherwise, [false] if the input occurred during SLL prediction. + */ + final bool fullCtx; + + DecisionEventInfo(this.decision, this.configs, this.input, this.startIndex, + this.stopIndex, this.fullCtx); +} + +/** + * This class contains profiling gathered for a particular decision. + * + *

+ * Parsing performance in ANTLR 4 is heavily influenced by both static factors + * (e.g. the form of the rules in the grammar) and dynamic factors (e.g. the + * choice of input and the state of the DFA cache at the time profiling + * operations are started). For best results, gather and use aggregate + * statistics from a large sample of inputs representing the inputs expected in + * production before using the results to make changes in the grammar.

+ * + * @since 4.3 + */ +class DecisionInfo { + /** + * The decision number, which is an index into {@link ATN#decisionToState}. + */ + final int decision; + + /** + * The total number of times {@link ParserATNSimulator#adaptivePredict} was + * invoked for this decision. + */ + int invocations; + + /** + * The total time spent in {@link ParserATNSimulator#adaptivePredict} for + * this decision, in nanoseconds. + * + *

+ * The value of this field contains the sum of differential results obtained + * by {@link System#nanoTime()}, and is not adjusted to compensate for JIT + * and/or garbage collection overhead. For best accuracy, use a modern JVM + * implementation that provides precise results from + * {@link System#nanoTime()}, and perform profiling in a separate process + * which is warmed up by parsing the input prior to profiling. If desired, + * call {@link ATNSimulator#clearDFA} to reset the DFA cache to its initial + * state before starting the profiling measurement pass.

+ */ + int timeInPrediction; + + /** + * The sum of the lookahead required for SLL prediction for this decision. + * Note that SLL prediction is used before LL prediction for performance + * reasons even when {@link PredictionMode#LL} or + * {@link PredictionMode#LL_EXACT_AMBIG_DETECTION} is used. + */ + int SLL_TotalLook; + + /** + * Gets the minimum lookahead required for any single SLL prediction to + * complete for this decision, by reaching a unique prediction, reaching an + * SLL conflict state, or encountering a syntax error. + */ + int SLL_MinLook; + + /** + * Gets the maximum lookahead required for any single SLL prediction to + * complete for this decision, by reaching a unique prediction, reaching an + * SLL conflict state, or encountering a syntax error. + */ + int SLL_MaxLook; + + /** + * Gets the [LookaheadEventInfo] associated with the event where the + * {@link #SLL_MaxLook} value was set. + */ + LookaheadEventInfo SLL_MaxLookEvent; + + /** + * The sum of the lookahead required for LL prediction for this decision. + * Note that LL prediction is only used when SLL prediction reaches a + * conflict state. + */ + int LL_TotalLook; + + /** + * Gets the minimum lookahead required for any single LL prediction to + * complete for this decision. An LL prediction completes when the algorithm + * reaches a unique prediction, a conflict state (for + * {@link PredictionMode#LL}, an ambiguity state (for + * {@link PredictionMode#LL_EXACT_AMBIG_DETECTION}, or a syntax error. + */ + int LL_MinLook; + + /** + * Gets the maximum lookahead required for any single LL prediction to + * complete for this decision. An LL prediction completes when the algorithm + * reaches a unique prediction, a conflict state (for + * {@link PredictionMode#LL}, an ambiguity state (for + * {@link PredictionMode#LL_EXACT_AMBIG_DETECTION}, or a syntax error. + */ + int LL_MaxLook; + + /** + * Gets the [LookaheadEventInfo] associated with the event where the + * {@link #LL_MaxLook} value was set. + */ + LookaheadEventInfo LL_MaxLookEvent; + + /** + * A collection of [ContextSensitivityInfo] instances describing the + * context sensitivities encountered during LL prediction for this decision. + * + * @see ContextSensitivityInfo + */ + final List contextSensitivities = []; + + /** + * A collection of [ErrorInfo] instances describing the parse errors + * identified during calls to {@link ParserATNSimulator#adaptivePredict} for + * this decision. + * + * @see ErrorInfo + */ + final List errors = []; + + /** + * A collection of [AmbiguityInfo] instances describing the + * ambiguities encountered during LL prediction for this decision. + * + * @see AmbiguityInfo + */ + final List ambiguities = []; + + /** + * A collection of [PredicateEvalInfo] instances describing the + * results of evaluating individual predicates during prediction for this + * decision. + * + * @see PredicateEvalInfo + */ + final List predicateEvals = []; + + /** + * The total number of ATN transitions required during SLL prediction for + * this decision. An ATN transition is determined by the number of times the + * DFA does not contain an edge that is required for prediction, resulting + * in on-the-fly computation of that edge. + * + *

+ * If DFA caching of SLL transitions is employed by the implementation, ATN + * computation may cache the computed edge for efficient lookup during + * future parsing of this decision. Otherwise, the SLL parsing algorithm + * will use ATN transitions exclusively.

+ * + * @see #SLL_ATNTransitions + * @see ParserATNSimulator#computeTargetState + * @see LexerATNSimulator#computeTargetState + */ + int SLL_ATNTransitions; + + /** + * The total number of DFA transitions required during SLL prediction for + * this decision. + * + *

If the ATN simulator implementation does not use DFA caching for SLL + * transitions, this value will be 0.

+ * + * @see ParserATNSimulator#getExistingTargetState + * @see LexerATNSimulator#getExistingTargetState + */ + int SLL_DFATransitions; + + /** + * Gets the total number of times SLL prediction completed in a conflict + * state, resulting in fallback to LL prediction. + * + *

Note that this value is not related to whether or not + * {@link PredictionMode#SLL} may be used successfully with a particular + * grammar. If the ambiguity resolution algorithm applied to the SLL + * conflicts for this decision produce the same result as LL prediction for + * this decision, {@link PredictionMode#SLL} would produce the same overall + * parsing result as {@link PredictionMode#LL}.

+ */ + int LL_Fallback; + + /** + * The total number of ATN transitions required during LL prediction for + * this decision. An ATN transition is determined by the number of times the + * DFA does not contain an edge that is required for prediction, resulting + * in on-the-fly computation of that edge. + * + *

+ * If DFA caching of LL transitions is employed by the implementation, ATN + * computation may cache the computed edge for efficient lookup during + * future parsing of this decision. Otherwise, the LL parsing algorithm will + * use ATN transitions exclusively.

+ * + * @see #LL_DFATransitions + * @see ParserATNSimulator#computeTargetState + * @see LexerATNSimulator#computeTargetState + */ + int LL_ATNTransitions; + + /** + * The total number of DFA transitions required during LL prediction for + * this decision. + * + *

If the ATN simulator implementation does not use DFA caching for LL + * transitions, this value will be 0.

+ * + * @see ParserATNSimulator#getExistingTargetState + * @see LexerATNSimulator#getExistingTargetState + */ + int LL_DFATransitions; + + /** + * Constructs a new instance of the [DecisionInfo] class to contain + * statistics for a particular decision. + * + * @param decision The decision number + */ + DecisionInfo(this.decision); + + String toString() { + return "{" + + "decision=$decision" + + ", contextSensitivities=${contextSensitivities.length}" + + ", errors=${errors.length}" + + ", ambiguities=${ambiguities.length}" + + ", SLL_lookahead=$SLL_TotalLook" + + ", SLL_ATNTransitions=$SLL_ATNTransitions" + + ", SLL_DFATransitions=$SLL_DFATransitions" + + ", LL_Fallback=$LL_Fallback" + + ", LL_lookahead=$LL_TotalLook" + + ", LL_ATNTransitions=$LL_ATNTransitions" + + '}'; + } +} + +/** + * This class represents profiling event information for an ambiguity. + * Ambiguities are decisions where a particular input resulted in an SLL + * conflict, followed by LL prediction also reaching a conflict state + * (indicating a true ambiguity in the grammar). + * + *

+ * This event may be reported during SLL prediction in cases where the + * conflicting SLL configuration set provides sufficient information to + * determine that the SLL conflict is truly an ambiguity. For example, if none + * of the ATN configurations in the conflicting SLL configuration set have + * traversed a global follow transition (i.e. + * {@link ATNConfig#reachesIntoOuterContext} is 0 for all configurations), then + * the result of SLL prediction for that input is known to be equivalent to the + * result of LL prediction for that input.

+ * + *

+ * In some cases, the minimum represented alternative in the conflicting LL + * configuration set is not equal to the minimum represented alternative in the + * conflicting SLL configuration set. Grammars and inputs which result in this + * scenario are unable to use {@link PredictionMode#SLL}, which in turn means + * they cannot use the two-stage parsing strategy to improve parsing performance + * for that input.

+ * + * @see ParserATNSimulator#reportAmbiguity + * @see ANTLRErrorListener#reportAmbiguity + * + * @since 4.3 + */ +class AmbiguityInfo extends DecisionEventInfo { + /** The set of alternative numbers for this decision event that lead to a valid parse. */ + BitSet ambigAlts; + + /** + * Constructs a new instance of the [AmbiguityInfo] class with the + * specified detailed ambiguity information. + * + * @param decision The decision number + * @param configs The final configuration set identifying the ambiguous + * alternatives for the current input + * @param ambigAlts The set of alternatives in the decision that lead to a valid parse. + * The predicted alt is the min(ambigAlts) + * @param input The input token stream + * @param startIndex The start index for the current prediction + * @param stopIndex The index at which the ambiguity was identified during + * prediction + * @param fullCtx [true] if the ambiguity was identified during LL + * prediction; otherwise, [false] if the ambiguity was identified + * during SLL prediction + */ + AmbiguityInfo(int decision, ATNConfigSet configs, BitSet this.ambigAlts, + TokenStream input, int startIndex, int stopIndex, bool fullCtx) + : super(decision, configs, input, startIndex, stopIndex, fullCtx); +} + +/** + * This class represents profiling event information for a syntax error + * identified during prediction. Syntax errors occur when the prediction + * algorithm is unable to identify an alternative which would lead to a + * successful parse. + * + * @see Parser#notifyErrorListeners(Token, String, RecognitionException) + * @see ANTLRErrorListener#syntaxError + * + * @since 4.3 + */ +class ErrorInfo extends DecisionEventInfo { + /** + * Constructs a new instance of the [ErrorInfo] class with the + * specified detailed syntax error information. + * + * @param decision The decision number + * @param configs The final configuration set reached during prediction + * prior to reaching the {@link ATNSimulator#ERROR} state + * @param input The input token stream + * @param startIndex The start index for the current prediction + * @param stopIndex The index at which the syntax error was identified + * @param fullCtx [true] if the syntax error was identified during LL + * prediction; otherwise, [false] if the syntax error was identified + * during SLL prediction + */ + ErrorInfo(int decision, ATNConfigSet configs, TokenStream input, + int startIndex, int stopIndex, bool fullCtx) + : super(decision, configs, input, startIndex, stopIndex, fullCtx); +} + +/** + * This class represents profiling event information for tracking the lookahead + * depth required in order to make a prediction. + * + * @since 4.3 + */ +class LookaheadEventInfo extends DecisionEventInfo { + /** The alternative chosen by adaptivePredict(), not necessarily + * the outermost alt shown for a rule; left-recursive rules have + * user-level alts that differ from the rewritten rule with a (...) block + * and a (..)* loop. + */ + int predictedAlt; + + /** + * Constructs a new instance of the [LookaheadEventInfo] class with + * the specified detailed lookahead information. + * + * @param decision The decision number + * @param configs The final configuration set containing the necessary + * information to determine the result of a prediction, or null if + * the final configuration set is not available + * @param input The input token stream + * @param startIndex The start index for the current prediction + * @param stopIndex The index at which the prediction was finally made + * @param fullCtx [true] if the current lookahead is part of an LL + * prediction; otherwise, [false] if the current lookahead is part of + * an SLL prediction + */ + LookaheadEventInfo(int decision, ATNConfigSet configs, int this.predictedAlt, + TokenStream input, int startIndex, int stopIndex, bool fullCtx) + : super(decision, configs, input, startIndex, stopIndex, fullCtx); +} + +/** + * This class represents profiling event information for semantic predicate + * evaluations which occur during prediction. + * + * @see ParserATNSimulator#evalSemanticContext + * + * @since 4.3 + */ +class PredicateEvalInfo extends DecisionEventInfo { + /** + * The semantic context which was evaluated. + */ + final SemanticContext semctx; + + /** + * The alternative number for the decision which is guarded by the semantic + * context {@link #semctx}. Note that other ATN + * configurations may predict the same alternative which are guarded by + * other semantic contexts and/or {@link SemanticContext#NONE}. + */ + final int predictedAlt; + + /** + * The result of evaluating the semantic context {@link #semctx}. + */ + final bool evalResult; + + /** + * Constructs a new instance of the [PredicateEvalInfo] class with the + * specified detailed predicate evaluation information. + * + * @param decision The decision number + * @param input The input token stream + * @param startIndex The start index for the current prediction + * @param stopIndex The index at which the predicate evaluation was + * triggered. Note that the input stream may be reset to other positions for + * the actual evaluation of individual predicates. + * @param semctx The semantic context which was evaluated + * @param evalResult The results of evaluating the semantic context + * @param predictedAlt The alternative number for the decision which is + * guarded by the semantic context [semctx]. See {@link #predictedAlt} + * for more information. + * @param fullCtx [true] if the semantic context was + * evaluated during LL prediction; otherwise, [false] if the semantic + * context was evaluated during SLL prediction + * + * @see ParserATNSimulator#evalSemanticContext(SemanticContext, ParserRuleContext, int, boolean) + * @see SemanticContext#eval(Recognizer, RuleContext) + */ + PredicateEvalInfo( + int decision, + TokenStream input, + int startIndex, + int stopIndex, + this.semctx, + this.evalResult, + this.predictedAlt, + bool fullCtx) + : super(decision, new ATNConfigSet(), input, startIndex, stopIndex, + fullCtx); +} + +/** + * This class provides access to specific and aggregate statistics gathered + * during profiling of a parser. + * + * @since 4.3 + */ +class ParseInfo { + final ProfilingATNSimulator atnSimulator; + + ParseInfo(this.atnSimulator); + + /** + * Gets an array of [DecisionInfo] instances containing the profiling + * information gathered for each decision in the ATN. + * + * @return An array of [DecisionInfo] instances, indexed by decision + * number. + */ + List get decisionInfo { + return atnSimulator.decisionInfo; + } + + /** + * Gets the decision numbers for decisions that required one or more + * full-context predictions during parsing. These are decisions for which + * {@link DecisionInfo#LL_Fallback} is non-zero. + * + * @return A list of decision numbers which required one or more + * full-context predictions during parsing. + */ + List get llDecisions { + List decisions = atnSimulator.decisionInfo; + List LL = []; + for (int i = 0; i < decisions.length; i++) { + int fallBack = decisions[i].LL_Fallback; + if (fallBack > 0) LL.add(i); + } + return LL; + } + + /** + * Gets the total time spent during prediction across all decisions made + * during parsing. This value is the sum of + * {@link DecisionInfo#timeInPrediction} for all decisions. + */ + int get totalTimeInPrediction { + List decisions = atnSimulator.decisionInfo; + int t = 0; + for (int i = 0; i < decisions.length; i++) { + t += decisions[i].timeInPrediction; + } + return t; + } + + /** + * Gets the total number of SLL lookahead operations across all decisions + * made during parsing. This value is the sum of + * {@link DecisionInfo#SLL_TotalLook} for all decisions. + */ + int get totalSLLLookaheadOps { + List decisions = atnSimulator.decisionInfo; + int k = 0; + for (int i = 0; i < decisions.length; i++) { + k += decisions[i].SLL_TotalLook; + } + return k; + } + + /** + * Gets the total number of LL lookahead operations across all decisions + * made during parsing. This value is the sum of + * {@link DecisionInfo#LL_TotalLook} for all decisions. + */ + int get totalLLLookaheadOps { + List decisions = atnSimulator.decisionInfo; + int k = 0; + for (int i = 0; i < decisions.length; i++) { + k += decisions[i].LL_TotalLook; + } + return k; + } + + /** + * Gets the total number of ATN lookahead operations for SLL prediction + * across all decisions made during parsing. + */ + int get totalSLLATNLookaheadOps { + List decisions = atnSimulator.decisionInfo; + int k = 0; + for (int i = 0; i < decisions.length; i++) { + k += decisions[i].SLL_ATNTransitions; + } + return k; + } + + /** + * Gets the total number of ATN lookahead operations for LL prediction + * across all decisions made during parsing. + */ + int get totalLLATNLookaheadOps { + List decisions = atnSimulator.decisionInfo; + int k = 0; + for (int i = 0; i < decisions.length; i++) { + k += decisions[i].LL_ATNTransitions; + } + return k; + } + + /** + * Gets the total number of ATN lookahead operations for SLL and LL + * prediction across all decisions made during parsing. + * + *

+ * This value is the sum of {@link #getTotalSLLATNLookaheadOps} and + * {@link #getTotalLLATNLookaheadOps}.

+ */ + int get totalATNLookaheadOps { + List decisions = atnSimulator.decisionInfo; + int k = 0; + for (int i = 0; i < decisions.length; i++) { + k += decisions[i].SLL_ATNTransitions; + k += decisions[i].LL_ATNTransitions; + } + return k; + } + + /** + * Gets the total number of DFA states stored in the DFA cache for all + * decisions in the ATN. + */ + int get dfaSize { + int n = 0; + List decisionToDFA = atnSimulator.decisionToDFA; + for (int i = 0; i < decisionToDFA.length; i++) { + n += getDFASizeAt(i); + } + return n; + } + + /** + * Gets the total number of DFA states stored in the DFA cache for a + * particular decision. + */ + int getDFASizeAt(int decision) { + DFA decisionToDFA = atnSimulator.decisionToDFA[decision]; + return decisionToDFA.states.length; + } +} diff --git a/runtime/Dart/lib/src/atn/src/lexer_action.dart b/runtime/Dart/lib/src/atn/src/lexer_action.dart new file mode 100644 index 000000000..2c89e623b --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/lexer_action.dart @@ -0,0 +1,678 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import '../../lexer.dart'; +import '../../util/murmur_hash.dart'; + +/** + * Represents the serialization type of a [LexerAction]. + * + * @since 4.2 + */ +enum LexerActionType { + /** + * The type of a [LexerChannelAction] action. + */ + CHANNEL, + /** + * The type of a [LexerCustomAction] action. + */ + CUSTOM, + /** + * The type of a [LexerModeAction] action. + */ + MODE, + /** + * The type of a [LexerMoreAction] action. + */ + MORE, + /** + * The type of a [LexerPopModeAction] action. + */ + POP_MODE, + /** + * The type of a [LexerPushModeAction] action. + */ + PUSH_MODE, + /** + * The type of a [LexerSkipAction] action. + */ + SKIP, + /** + * The type of a [LexerTypeAction] action. + */ + TYPE, +} + +/** + * Represents a single action which can be executed following the successful + * match of a lexer rule. Lexer actions are used for both embedded action syntax + * and ANTLR 4's new lexer command syntax. + * + * @since 4.2 + */ +abstract class LexerAction { + /** + * Gets the serialization type of the lexer action. + * + * @return The serialization type of the lexer action. + */ + LexerActionType get actionType; + + /** + * Gets whether the lexer action is position-dependent. Position-dependent + * actions may have different semantics depending on the [CharStream] + * index at the time the action is executed. + * + *

Many lexer commands, including [type], [skip], and + * [more], do not check the input index during their execution. + * Actions like this are position-independent, and may be stored more + * efficiently as part of the {@link LexerATNConfig#lexerActionExecutor}.

+ * + * @return [true] if the lexer action semantics can be affected by the + * position of the input [CharStream] at the time it is executed; + * otherwise, [false]. + */ + bool get isPositionDependent; + + /** + * Execute the lexer action in the context of the specified [Lexer]. + * + *

For position-dependent actions, the input stream must already be + * positioned correctly prior to calling this method.

+ * + * @param lexer The lexer instance. + */ + void execute(Lexer lexer); +} + +/** + * Implements the [channel] lexer action by calling + * {@link Lexer#setChannel} with the assigned channel. + * + * @since 4.2 + */ +class LexerChannelAction implements LexerAction { + /** + * Gets the channel to use for the [Token] created by the lexer. + * + * @return The channel to use for the [Token] created by the lexer. + */ + final int channel; + + /** + * Constructs a new [channel] action with the specified channel value. + * @param channel The channel value to pass to {@link Lexer#setChannel}. + */ + LexerChannelAction(this.channel); + + @override + LexerActionType get actionType => LexerActionType.CHANNEL; + + bool get isPositionDependent => false; + + /** + * {@inheritDoc} + * + *

This action is implemented by calling {@link Lexer#setChannel} with the + * value provided by {@link #getChannel}.

+ */ + void execute(Lexer lexer) { + lexer.channel = channel; + } + + get hashCode { + int hash = MurmurHash.initialize(); + hash = MurmurHash.update(hash, actionType.index); + hash = MurmurHash.update(hash, channel); + return MurmurHash.finish(hash, 2); + } + + bool operator ==(Object obj) { + if (identical(obj, this)) { + return true; + } else if (obj is LexerChannelAction) { + return channel == obj.channel; + } + + return false; + } + + String toString() { + return "channel($channel)"; + } +} + +/** + * Executes a custom lexer action by calling {@link Recognizer#action} with the + * rule and action indexes assigned to the custom action. The implementation of + * a custom action is added to the generated code for the lexer in an override + * of {@link Recognizer#action} when the grammar is compiled. + * + *

This class may represent embedded actions created with the {...} + * syntax in ANTLR 4, as well as actions created for lexer commands where the + * command argument could not be evaluated when the grammar was compiled.

+ * + * @since 4.2 + */ +class LexerCustomAction implements LexerAction { + /** + * Gets the rule index to use for calls to {@link Recognizer#action}. + * + * @return The rule index for the custom action. + */ + final int ruleIndex; + + /** + * Gets the action index to use for calls to {@link Recognizer#action}. + * + * @return The action index for the custom action. + */ + final int actionIndex; + + /** + * Constructs a custom lexer action with the specified rule and action + * indexes. + * + * @param ruleIndex The rule index to use for calls to + * {@link Recognizer#action}. + * @param actionIndex The action index to use for calls to + * {@link Recognizer#action}. + */ + LexerCustomAction(this.ruleIndex, this.actionIndex); + + /** + * {@inheritDoc} + * + * @return This method returns {@link LexerActionType#CUSTOM}. + */ + + get actionType => LexerActionType.CUSTOM; + + /** + * Gets whether the lexer action is position-dependent. Position-dependent + * actions may have different semantics depending on the [CharStream] + * index at the time the action is executed. + * + *

Custom actions are position-dependent since they may represent a + * user-defined embedded action which makes calls to methods like + * {@link Lexer#getText}.

+ * + * @return This method returns [true]. + */ + + bool get isPositionDependent => true; + + /** + * {@inheritDoc} + * + *

Custom actions are implemented by calling {@link Lexer#action} with the + * appropriate rule and action indexes.

+ */ + + void execute(Lexer lexer) { + lexer.action(null, ruleIndex, actionIndex); + } + + get hashCode { + int hash = MurmurHash.initialize(); + hash = MurmurHash.update(hash, actionType.index); + hash = MurmurHash.update(hash, ruleIndex); + hash = MurmurHash.update(hash, actionIndex); + return MurmurHash.finish(hash, 3); + } + + bool operator ==(Object obj) { + if (identical(obj, this)) { + return true; + } else if (obj is LexerCustomAction) { + return ruleIndex == obj.ruleIndex && actionIndex == obj.actionIndex; + } + return false; + } +} + +/** + * Implements the [mode] lexer action by calling {@link Lexer#mode} with + * the assigned mode. + * + * @since 4.2 + */ +class LexerModeAction implements LexerAction { + /** + * Get the lexer mode this action should transition the lexer to. + * + * @return The lexer mode for this [mode] command. + */ + final int mode; + + /** + * Constructs a new [mode] action with the specified mode value. + * @param mode The mode value to pass to {@link Lexer#mode}. + */ + LexerModeAction(this.mode); + + /** + * {@inheritDoc} + * @return This method returns {@link LexerActionType#MODE}. + */ + + get actionType => LexerActionType.MODE; + + /** + * {@inheritDoc} + * @return This method returns [false]. + */ + + bool get isPositionDependent => false; + + /** + * {@inheritDoc} + * + *

This action is implemented by calling {@link Lexer#mode} with the + * value provided by {@link #getMode}.

+ */ + + void execute(Lexer lexer) { + lexer.mode(mode); + } + + get hashCode { + int hash = MurmurHash.initialize(); + hash = MurmurHash.update(hash, actionType.index); + hash = MurmurHash.update(hash, mode); + return MurmurHash.finish(hash, 2); + } + + bool operator ==(Object obj) { + if (identical(obj, this)) { + return true; + } else if (obj is LexerModeAction) { + return mode == obj.mode; + } + return false; + } + + String toString() { + return "mode($mode)"; + } +} + +/** + * Implements the [more] lexer action by calling {@link Lexer#more}. + * + *

The [more] command does not have any parameters, so this action is + * implemented as a singleton instance exposed by {@link #INSTANCE}.

+ * + * @since 4.2 + */ +class LexerMoreAction implements LexerAction { + /** + * Provides a singleton instance of this parameterless lexer action. + */ + static final LexerMoreAction INSTANCE = new LexerMoreAction(); + + /** + * {@inheritDoc} + * @return This method returns {@link LexerActionType#MORE}. + */ + get actionType => LexerActionType.MORE; + + /** + * {@inheritDoc} + * @return This method returns [false]. + */ + + bool get isPositionDependent => false; + + /** + * {@inheritDoc} + * + *

This action is implemented by calling {@link Lexer#more}.

+ */ + + void execute(Lexer lexer) { + lexer.more(); + } + + get hashCode { + int hash = MurmurHash.initialize(); + hash = MurmurHash.update(hash, actionType.index); + return MurmurHash.finish(hash, 1); + } + + bool operator ==(Object obj) { + return identical(obj, this); + } + + String toString() { + return "more"; + } +} + +/** + * Implements the [popMode] lexer action by calling {@link Lexer#popMode}. + * + *

The [popMode] command does not have any parameters, so this action is + * implemented as a singleton instance exposed by {@link #INSTANCE}.

+ * + * @since 4.2 + */ +class LexerPopModeAction implements LexerAction { + /** + * Provides a singleton instance of this parameterless lexer action. + */ + static final LexerPopModeAction INSTANCE = new LexerPopModeAction(); + + /** + * {@inheritDoc} + * @return This method returns {@link LexerActionType#POP_MODE}. + */ + + get actionType => LexerActionType.POP_MODE; + + /** + * {@inheritDoc} + * @return This method returns [false]. + */ + + bool get isPositionDependent => false; + + /** + * {@inheritDoc} + * + *

This action is implemented by calling {@link Lexer#popMode}.

+ */ + + void execute(Lexer lexer) { + lexer.popMode(); + } + + get hashCode { + int hash = MurmurHash.initialize(); + hash = MurmurHash.update(hash, actionType.index); + return MurmurHash.finish(hash, 1); + } + + bool operator ==(Object obj) { + return identical(obj, this); + } + + String toString() { + return "popMode"; + } +} + +/** + * Implements the [pushMode] lexer action by calling + * {@link Lexer#pushMode} with the assigned mode. + * + * @since 4.2 + */ +class LexerPushModeAction implements LexerAction { + /** + * Get the lexer mode this action should transition the lexer to. + * + * @return The lexer mode for this [pushMode] command. + */ + final int mode; + + /** + * Constructs a new [pushMode] action with the specified mode value. + * @param mode The mode value to pass to {@link Lexer#pushMode}. + */ + LexerPushModeAction(this.mode); + + /** + * {@inheritDoc} + * @return This method returns {@link LexerActionType#PUSH_MODE}. + */ + + get actionType => LexerActionType.PUSH_MODE; + + /** + * {@inheritDoc} + * @return This method returns [false]. + */ + + bool get isPositionDependent => false; + + /** + * {@inheritDoc} + * + *

This action is implemented by calling {@link Lexer#pushMode} with the + * value provided by {@link #getMode}.

+ */ + + void execute(Lexer lexer) { + lexer.pushMode(mode); + } + + get hashCode { + int hash = MurmurHash.initialize(); + hash = MurmurHash.update(hash, actionType.index); + hash = MurmurHash.update(hash, mode); + return MurmurHash.finish(hash, 2); + } + + bool operator ==(Object obj) { + if (identical(obj, this)) { + return true; + } else if (obj is LexerPushModeAction) { + return mode == obj.mode; + } + return false; + } + + String toString() { + return "pushMode($mode)"; + } +} + +/** + * Implements the [skip] lexer action by calling {@link Lexer#skip}. + * + *

The [skip] command does not have any parameters, so this action is + * implemented as a singleton instance exposed by {@link #INSTANCE}.

+ * + * @since 4.2 + */ +class LexerSkipAction implements LexerAction { + /** + * Provides a singleton instance of this parameterless lexer action. + */ + static final LexerSkipAction INSTANCE = new LexerSkipAction(); + + /** + * {@inheritDoc} + * @return This method returns {@link LexerActionType#SKIP}. + */ + + get actionType => LexerActionType.SKIP; + + /** + * {@inheritDoc} + * @return This method returns [false]. + */ + + bool get isPositionDependent => false; + + /** + * {@inheritDoc} + * + *

This action is implemented by calling {@link Lexer#skip}.

+ */ + void execute(Lexer lexer) { + lexer.skip(); + } + + get hashCode { + int hash = MurmurHash.initialize(); + hash = MurmurHash.update(hash, actionType.index); + return MurmurHash.finish(hash, 1); + } + + bool operator ==(Object obj) { + return identical(obj, this); + } + + String toString() { + return "skip"; + } +} + +/** + * Implements the [type] lexer action by calling {@link Lexer#setType} + * with the assigned type. + * + * @since 4.2 + */ +class LexerTypeAction implements LexerAction { + /** + * Gets the type to assign to a token created by the lexer. + * @return The type to assign to a token created by the lexer. + */ + final int type; + + /** + * Constructs a new [type] action with the specified token type value. + * @param type The type to assign to the token using {@link Lexer#setType}. + */ + LexerTypeAction(this.type); + + /** + * {@inheritDoc} + * @return This method returns {@link LexerActionType#TYPE}. + */ + get actionType => LexerActionType.TYPE; + + /** + * {@inheritDoc} + * @return This method returns [false]. + */ + + bool get isPositionDependent => false; + + /** + * {@inheritDoc} + * + *

This action is implemented by calling {@link Lexer#setType} with the + * value provided by {@link #getType}.

+ */ + + void execute(Lexer lexer) { + lexer.type = type; + } + + get hashCode { + int hash = MurmurHash.initialize(); + hash = MurmurHash.update(hash, actionType.index); + hash = MurmurHash.update(hash, type); + return MurmurHash.finish(hash, 2); + } + + bool operator ==(Object obj) { + if (identical(obj, this)) { + return true; + } else if (obj is LexerTypeAction) { + return type == obj.type; + } + return false; + } + + String toString() { + return "type($type)"; + } +} + +/** + * This implementation of [LexerAction] is used for tracking input offsets + * for position-dependent actions within a [LexerActionExecutor]. + * + *

This action is not serialized as part of the ATN, and is only required for + * position-dependent lexer actions which appear at a location other than the + * end of a rule. For more information about DFA optimizations employed for + * lexer actions, see {@link LexerActionExecutor#append} and + * {@link LexerActionExecutor#fixOffsetBeforeMatch}.

+ * + * @since 4.2 + */ +class LexerIndexedCustomAction implements LexerAction { + /** + * Gets the location in the input [CharStream] at which the lexer + * action should be executed. The value is interpreted as an offset relative + * to the token start index. + * + * @return The location in the input [CharStream] at which the lexer + * action should be executed. + */ + final int offset; + + /** + * Gets the lexer action to execute. + * + * @return A [LexerAction] object which executes the lexer action. + */ + final LexerAction action; + + /** + * Constructs a new indexed custom action by associating a character offset + * with a [LexerAction]. + * + *

Note: This class is only required for lexer actions for which + * {@link LexerAction#isPositionDependent} returns [true].

+ * + * @param offset The offset into the input [CharStream], relative to + * the token start index, at which the specified lexer action should be + * executed. + * @param action The lexer action to execute at a particular offset in the + * input [CharStream]. + */ + LexerIndexedCustomAction(this.offset, this.action); + + /** + * {@inheritDoc} + * + * @return This method returns the result of calling {@link #getActionType} + * on the [LexerAction] returned by {@link #getAction}. + */ + LexerActionType get actionType => action.actionType; + + /** + * {@inheritDoc} + * @return This method returns [true]. + */ + + bool get isPositionDependent => true; + + /** + * {@inheritDoc} + * + *

This method calls {@link #execute} on the result of {@link #getAction} + * using the provided [lexer].

+ */ + + void execute(Lexer lexer) { +// assume the input stream position was properly set by the calling code + action.execute(lexer); + } + + int get hashCode { + int hash = MurmurHash.initialize(); + hash = MurmurHash.update(hash, offset); + hash = MurmurHash.update(hash, action); + return MurmurHash.finish(hash, 2); + } + + bool operator ==(Object obj) { + if (obj == this) { + return true; + } else if (obj is LexerIndexedCustomAction) { + return offset == obj.offset && action == obj.action; + } + return false; + } +} diff --git a/runtime/Dart/lib/src/atn/src/lexer_action_executor.dart b/runtime/Dart/lib/src/atn/src/lexer_action_executor.dart new file mode 100644 index 000000000..08a0ac00d --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/lexer_action_executor.dart @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'package:collection/collection.dart'; + +import '../../input_stream.dart'; +import '../../lexer.dart'; +import '../../util/murmur_hash.dart'; +import 'lexer_action.dart'; + +/** + * Represents an executor for a sequence of lexer actions which traversed during + * the matching operation of a lexer rule (token). + * + *

The executor tracks position information for position-dependent lexer actions + * efficiently, ensuring that actions appearing only at the end of the rule do + * not cause bloating of the [DFA] created for the lexer.

+ * + * @since 4.2 + */ +class LexerActionExecutor { + /** + * Gets the lexer actions to be executed by this executor. + * @return The lexer actions to be executed by this executor. + */ + final List lexerActions; + + /** + * Caches the result of {@link #hashCode} since the hash code is an element + * of the performance-critical {@link LexerATNConfig#hashCode} operation. + */ + int get hashCode { + int hash = MurmurHash.initialize(); + for (LexerAction lexerAction in lexerActions) { + hash = MurmurHash.update(hash, lexerAction); + } + + return MurmurHash.finish(hash, lexerActions.length); + } + + /** + * Constructs an executor for a sequence of [LexerAction] actions. + * @param lexerActions The lexer actions to execute. + */ + LexerActionExecutor(List this.lexerActions) {} + + /** + * Creates a [LexerActionExecutor] which executes the actions for + * the input [lexerActionExecutor] followed by a specified + * [lexerAction]. + * + * @param lexerActionExecutor The executor for actions already traversed by + * the lexer while matching a token within a particular + * [LexerATNConfig]. If this is null, the method behaves as + * though it were an empty executor. + * @param lexerAction The lexer action to execute after the actions + * specified in [lexerActionExecutor]. + * + * @return A [LexerActionExecutor] for executing the combine actions + * of [lexerActionExecutor] and [lexerAction]. + */ + static LexerActionExecutor append( + LexerActionExecutor lexerActionExecutor, LexerAction lexerAction) { + if (lexerActionExecutor == null) { + return new LexerActionExecutor([lexerAction]); + } + + List lexerActions = + List.from(lexerActionExecutor.lexerActions); + lexerActions.add(lexerAction); + return new LexerActionExecutor(lexerActions); + } + + /** + * Creates a [LexerActionExecutor] which encodes the current offset + * for position-dependent lexer actions. + * + *

Normally, when the executor encounters lexer actions where + * {@link LexerAction#isPositionDependent} returns [true], it calls + * {@link IntStream#seek} on the input [CharStream] to set the input + * position to the end of the current token. This behavior provides + * for efficient DFA representation of lexer actions which appear at the end + * of a lexer rule, even when the lexer rule matches a variable number of + * characters.

+ * + *

Prior to traversing a match transition in the ATN, the current offset + * from the token start index is assigned to all position-dependent lexer + * actions which have not already been assigned a fixed offset. By storing + * the offsets relative to the token start index, the DFA representation of + * lexer actions which appear in the middle of tokens remains efficient due + * to sharing among tokens of the same length, regardless of their absolute + * position in the input stream.

+ * + *

If the current executor already has offsets assigned to all + * position-dependent lexer actions, the method returns [this].

+ * + * @param offset The current offset to assign to all position-dependent + * lexer actions which do not already have offsets assigned. + * + * @return A [LexerActionExecutor] which stores input stream offsets + * for all position-dependent lexer actions. + */ + LexerActionExecutor fixOffsetBeforeMatch(int offset) { + List updatedLexerActions = null; + for (int i = 0; i < lexerActions.length; i++) { + if (lexerActions[i].isPositionDependent && + !(lexerActions[i] is LexerIndexedCustomAction)) { + if (updatedLexerActions == null) { + updatedLexerActions = List.from(lexerActions); + } + + updatedLexerActions[i] = + new LexerIndexedCustomAction(offset, lexerActions[i]); + } + } + + if (updatedLexerActions == null) { + return this; + } + + return new LexerActionExecutor(updatedLexerActions); + } + + /** + * Execute the actions encapsulated by this executor within the context of a + * particular [Lexer]. + * + *

This method calls {@link IntStream#seek} to set the position of the + * [input] [CharStream] prior to calling + * {@link LexerAction#execute} on a position-dependent action. Before the + * method returns, the input position will be restored to the same position + * it was in when the method was invoked.

+ * + * @param lexer The lexer instance. + * @param input The input stream which is the source for the current token. + * When this method is called, the current {@link IntStream#index} for + * [input] should be the start of the following token, i.e. 1 + * character past the end of the current token. + * @param startIndex The token start index. This value may be passed to + * {@link IntStream#seek} to set the [input] position to the beginning + * of the token. + */ + void execute(Lexer lexer, CharStream input, int startIndex) { + bool requiresSeek = false; + int stopIndex = input.index; + try { + for (LexerAction lexerAction in lexerActions) { + if (lexerAction is LexerIndexedCustomAction) { + int offset = (lexerAction as LexerIndexedCustomAction).offset; + input.seek(startIndex + offset); + lexerAction = (lexerAction as LexerIndexedCustomAction).action; + requiresSeek = (startIndex + offset) != stopIndex; + } else if (lexerAction.isPositionDependent) { + input.seek(stopIndex); + requiresSeek = false; + } + + lexerAction.execute(lexer); + } + } finally { + if (requiresSeek) { + input.seek(stopIndex); + } + } + } + + bool operator ==(Object obj) { + if (identical(obj, this)) { + return true; + } else if (!(obj is LexerActionExecutor)) { + return false; + } + + LexerActionExecutor other = obj; + return hashCode == other.hashCode && + ListEquality().equals(lexerActions, other.lexerActions); + } +} diff --git a/runtime/Dart/lib/src/atn/src/lexer_atn_simulator.dart b/runtime/Dart/lib/src/atn/src/lexer_atn_simulator.dart new file mode 100644 index 000000000..9c407f84c --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/lexer_atn_simulator.dart @@ -0,0 +1,747 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:developer'; + +import 'package:logging/logging.dart'; + +import '../../dfa/dfa.dart'; +import '../../error/error.dart'; +import '../../input_stream.dart'; +import '../../interval_set.dart'; +import '../../lexer.dart'; +import '../../prediction_context.dart'; +import '../../token.dart'; +import 'atn.dart'; +import 'atn_config.dart'; +import 'atn_config_set.dart'; +import 'atn_simulator.dart'; +import 'atn_state.dart'; +import 'lexer_action_executor.dart'; +import 'transition.dart'; + +/// When we hit an accept state in either the DFA or the ATN, we +/// have to notify the character stream to start buffering characters +/// via {@link IntStream#mark} and record the current state. The current sim state +/// includes the current index into the input, the current line, +/// and current character position in that line. Note that the Lexer is +/// tracking the starting line and characterization of the token. These +/// variables track the "state" of the simulator when it hits an accept state. +/// +///

We track these variables separately for the DFA and ATN simulation +/// because the DFA simulation often has to fail over to the ATN +/// simulation. If the ATN simulation fails, we need the DFA to fall +/// back to its previously accepted state, if any. If the ATN succeeds, +/// then the ATN does the accept and the DFA simulator that invoked it +/// can simply return the predicted token type.

+class SimState { + int index = -1; + int line = 0; + int charPos = -1; + + DFAState dfaState = null; + + reset() { + this.index = -1; + this.line = 0; + this.charPos = -1; + this.dfaState = null; + } +} + +/** "dup" of ParserInterpreter */ +class LexerATNSimulator extends ATNSimulator { + static final bool debug = true; + static final bool dfa_debug = true; + + static final int MIN_DFA_EDGE = 0; + static final int MAX_DFA_EDGE = 127; // forces unicode to stay in ATN + + final Lexer recog; + + /** The current token's starting index into the character stream. + * Shared across DFA to ATN simulation in case the ATN fails and the + * DFA did not have a previous accept state. In this case, we use the + * ATN-generated exception object. + */ + int startIndex = -1; + + /** line number 1..n within the input */ + int line = 1; + + /** The index of the character relative to the beginning of the line 0..n-1 */ + int charPositionInLine = 0; + + List decisionToDFA; + int mode = Lexer.DEFAULT_MODE; + + /** Used during DFA/ATN exec to record the most recent accept configuration info */ + + final SimState prevAccept = new SimState(); + + LexerATNSimulator(ATN atn, List this.decisionToDFA, + PredictionContextCache sharedContextCache, + {Lexer this.recog = null}) + : super(atn, sharedContextCache); + + void copyState(LexerATNSimulator simulator) { + this.charPositionInLine = simulator.charPositionInLine; + this.line = simulator.line; + this.mode = simulator.mode; + this.startIndex = simulator.startIndex; + } + + int match(CharStream input, int mode) { + this.mode = mode; + int mark = input.mark(); + try { + this.startIndex = input.index; + this.prevAccept.reset(); + DFA dfa = decisionToDFA[mode]; + if (dfa.s0 == null) { + return matchATN(input); + } else { + return execATN(input, dfa.s0); + } + } finally { + input.release(mark); + } + } + + void reset() { + prevAccept.reset(); + startIndex = -1; + line = 1; + charPositionInLine = 0; + mode = Lexer.DEFAULT_MODE; + } + + void clearDFA() { + for (int d = 0; d < decisionToDFA.length; d++) { + decisionToDFA[d] = new DFA(atn.getDecisionState(d), d); + } + } + + int matchATN(CharStream input) { + ATNState startState = atn.modeToStartState[mode]; + + if (debug) { + log("matchATN mode $mode start: $startState\n", level: Level.FINE.value); + } + + int old_mode = mode; + + ATNConfigSet s0_closure = computeStartState(input, startState); + bool suppressEdge = s0_closure.hasSemanticContext; + s0_closure.hasSemanticContext = false; + + DFAState next = addDFAState(s0_closure); + if (!suppressEdge) { + decisionToDFA[mode].s0 = next; + } + + int predict = execATN(input, next); + + if (debug) { + log("DFA after matchATN: ${decisionToDFA[old_mode].toLexerString()}\n", + level: Level.FINE.value); + } + + return predict; + } + + int execATN(CharStream input, DFAState ds0) { + //log("enter exec index "+input.index()+" from "+ds0.configs, level: Level.FINE.value); + if (debug) { + log("start state closure=${ds0.configs}\n", level: Level.FINE.value); + } + + if (ds0.isAcceptState) { + // allow zero-length tokens + captureSimState(prevAccept, input, ds0); + } + + int t = input.LA(1); + + DFAState s = ds0; // s is current/from DFA state + + while (true) { + // while more work + if (debug) { + log("execATN loop starting closure: ${s.configs}\n", + level: Level.FINE.value); + } + + // As we move src->trg, src->trg, we keep track of the previous trg to + // avoid looking up the DFA state again, which is expensive. + // If the previous target was already part of the DFA, we might + // be able to avoid doing a reach operation upon t. If s!=null, + // it means that semantic predicates didn't prevent us from + // creating a DFA state. Once we know s!=null, we check to see if + // the DFA state has an edge already for t. If so, we can just reuse + // it's configuration set; there's no point in re-computing it. + // This is kind of like doing DFA simulation within the ATN + // simulation because DFA simulation is really just a way to avoid + // computing reach/closure sets. Technically, once we know that + // we have a previously added DFA state, we could jump over to + // the DFA simulator. But, that would mean popping back and forth + // a lot and making things more complicated algorithmically. + // This optimization makes a lot of sense for loops within DFA. + // A character will take us back to an existing DFA state + // that already has lots of edges out of it. e.g., .* in comments. + DFAState target = getExistingTargetState(s, t); + if (target == null) { + target = computeTargetState(input, s, t); + } + + if (target == ATNSimulator.ERROR) { + break; + } + + // If this is a consumable input element, make sure to consume before + // capturing the accept state so the input index, line, and char + // position accurately reflect the state of the interpreter at the + // end of the token. + if (t != IntStream.EOF) { + consume(input); + } + + if (target.isAcceptState) { + captureSimState(prevAccept, input, target); + if (t == IntStream.EOF) { + break; + } + } + + t = input.LA(1); + s = target; // flip; current DFA target becomes new src/from state + } + + return failOrAccept(prevAccept, input, s.configs, t); + } + + /** + * Get an existing target state for an edge in the DFA. If the target state + * for the edge has not yet been computed or is otherwise not available, + * this method returns null. + * + * @param s The current DFA state + * @param t The next input symbol + * @return The existing target DFA state for the given input symbol + * [t], or null if the target state for this edge is not + * already cached + */ + + DFAState getExistingTargetState(DFAState s, int t) { + if (s.edges == null || t < MIN_DFA_EDGE || t > MAX_DFA_EDGE) { + return null; + } + + DFAState target = s.edges[t - MIN_DFA_EDGE]; + if (debug && target != null) { + log("reuse state ${s.stateNumber} edge to ${target.stateNumber}", + level: Level.FINE.value); + } + + return target; + } + + /** + * Compute a target state for an edge in the DFA, and attempt to add the + * computed state and corresponding edge to the DFA. + * + * @param input The input stream + * @param s The current DFA state + * @param t The next input symbol + * + * @return The computed target DFA state for the given input symbol + * [t]. If [t] does not lead to a valid DFA state, this method + * returns {@link #ERROR}. + */ + + DFAState computeTargetState(CharStream input, DFAState s, int t) { + ATNConfigSet reach = new OrderedATNConfigSet(); + + // if we don't find an existing DFA state + // Fill reach starting from closure, following t transitions + getReachableConfigSet(input, s.configs, reach, t); + + if (reach.isEmpty) { + // we got nowhere on t from s + if (!reach.hasSemanticContext) { + // we got nowhere on t, don't throw out this knowledge; it'd + // cause a failover from DFA later. + addDFAEdge(s, t, ATNSimulator.ERROR); + } + + // stop when we can't match any more char + return ATNSimulator.ERROR; + } + + // Add an edge from s to target DFA found/created for reach + return addDFAEdgeByConfig(s, t, reach); + } + + int failOrAccept( + SimState prevAccept, CharStream input, ATNConfigSet reach, int t) { + if (prevAccept.dfaState != null) { + LexerActionExecutor lexerActionExecutor = + prevAccept.dfaState.lexerActionExecutor; + accept(input, lexerActionExecutor, startIndex, prevAccept.index, + prevAccept.line, prevAccept.charPos); + return prevAccept.dfaState.prediction; + } else { + // if no accept and EOF is first char, return EOF + if (t == IntStream.EOF && input.index == startIndex) { + return Token.EOF; + } + + throw new LexerNoViableAltException(recog, input, startIndex, reach); + } + } + + /** Given a starting configuration set, figure out all ATN configurations + * we can reach upon input [t]. Parameter [reach] is a return + * parameter. + */ + void getReachableConfigSet( + CharStream input, ATNConfigSet configs, ATNConfigSet reach, int t) { + // this is used to skip processing for configs which have a lower priority + // than a config that already reached an accept state for the same rule + int skipAlt = ATN.INVALID_ALT_NUMBER; + for (ATNConfig c in configs) { + bool currentAltReachedAcceptState = c.alt == skipAlt; + if (currentAltReachedAcceptState && + (c as LexerATNConfig).hasPassedThroughNonGreedyDecision()) { + continue; + } + + if (debug) { + log("testing ${getTokenName(t)} at ${c.toString(recog, true)}\n", + level: Level.FINE.value); + } + + int n = c.state.numberOfTransitions; + for (int ti = 0; ti < n; ti++) { + // for each transition + Transition trans = c.state.transition(ti); + ATNState target = getReachableTarget(trans, t); + if (target != null) { + LexerActionExecutor lexerActionExecutor = + (c as LexerATNConfig).lexerActionExecutor; + if (lexerActionExecutor != null) { + lexerActionExecutor = lexerActionExecutor + .fixOffsetBeforeMatch(input.index - startIndex); + } + + bool treatEofAsEpsilon = t == IntStream.EOF; + if (closure( + input, + new LexerATNConfig.dup(c, target, + lexerActionExecutor: lexerActionExecutor), + reach, + currentAltReachedAcceptState, + true, + treatEofAsEpsilon)) { + // any remaining configs for this alt have a lower priority than + // the one that just reached an accept state. + skipAlt = c.alt; + break; + } + } + } + } + } + + void accept(CharStream input, LexerActionExecutor lexerActionExecutor, + int startIndex, int index, int line, int charPos) { + if (debug) { + log("ACTION $lexerActionExecutor\n", level: Level.FINE.value); + } + + // seek to after last char in token + input.seek(index); + this.line = line; + this.charPositionInLine = charPos; + + if (lexerActionExecutor != null && recog != null) { + lexerActionExecutor.execute(recog, input, startIndex); + } + } + + ATNState getReachableTarget(Transition trans, int t) { + if (trans.matches(t, Lexer.MIN_CHAR_VALUE, Lexer.MAX_CHAR_VALUE)) { + return trans.target; + } + + return null; + } + + ATNConfigSet computeStartState(CharStream input, ATNState p) { + PredictionContext initialContext = PredictionContext.EMPTY; + ATNConfigSet configs = new OrderedATNConfigSet(); + for (int i = 0; i < p.numberOfTransitions; i++) { + ATNState target = p.transition(i).target; + LexerATNConfig c = new LexerATNConfig(target, i + 1, initialContext); + closure(input, c, configs, false, false, false); + } + return configs; + } + + /** + * Since the alternatives within any lexer decision are ordered by + * preference, this method stops pursuing the closure as soon as an accept + * state is reached. After the first accept state is reached by depth-first + * search from [config], all other (potentially reachable) states for + * this rule would have a lower priority. + * + * @return [true] if an accept state is reached, otherwise + * [false]. + */ + bool closure( + CharStream input, + LexerATNConfig config, + ATNConfigSet configs, + bool currentAltReachedAcceptState, + bool speculative, + bool treatEofAsEpsilon) { + if (debug) { + log("closure(" + config.toString(recog, true) + ")", + level: Level.FINE.value); + } + + if (config.state is RuleStopState) { + if (debug) { + if (recog != null) { + log("closure at ${recog.ruleNames[config.state.ruleIndex]} rule stop $config\n", + level: Level.FINE.value); + } else { + log("closure at rule stop $config\n", level: Level.FINE.value); + } + } + + if (config.context == null || config.context.hasEmptyPath()) { + if (config.context == null || config.context.isEmpty) { + configs.add(config); + return true; + } else { + configs.add(new LexerATNConfig.dup(config, config.state, + context: PredictionContext.EMPTY)); + currentAltReachedAcceptState = true; + } + } + + if (config.context != null && !config.context.isEmpty) { + for (int i = 0; i < config.context.length; i++) { + if (config.context.getReturnState(i) != + PredictionContext.EMPTY_RETURN_STATE) { + PredictionContext newContext = + config.context.getParent(i); // "pop" return state + ATNState returnState = atn.states[config.context.getReturnState(i)]; + LexerATNConfig c = new LexerATNConfig.dup(config, returnState, + context: newContext); + currentAltReachedAcceptState = closure(input, c, configs, + currentAltReachedAcceptState, speculative, treatEofAsEpsilon); + } + } + } + + return currentAltReachedAcceptState; + } + + // optimization + if (!config.state.onlyHasEpsilonTransitions()) { + if (!currentAltReachedAcceptState || + !config.hasPassedThroughNonGreedyDecision()) { + configs.add(config); + } + } + + ATNState p = config.state; + for (int i = 0; i < p.numberOfTransitions; i++) { + Transition t = p.transition(i); + LexerATNConfig c = getEpsilonTarget( + input, config, t, configs, speculative, treatEofAsEpsilon); + if (c != null) { + currentAltReachedAcceptState = closure(input, c, configs, + currentAltReachedAcceptState, speculative, treatEofAsEpsilon); + } + } + + return currentAltReachedAcceptState; + } + + // side-effect: can alter configs.hasSemanticContext + + LexerATNConfig getEpsilonTarget( + CharStream input, + LexerATNConfig config, + Transition t, + ATNConfigSet configs, + bool speculative, + bool treatEofAsEpsilon) { + LexerATNConfig c = null; + switch (t.type) { + case TransitionType.RULE: + RuleTransition ruleTransition = t; + PredictionContext newContext = SingletonPredictionContext.create( + config.context, ruleTransition.followState.stateNumber); + c = new LexerATNConfig.dup(config, t.target, context: newContext); + break; + + case TransitionType.PRECEDENCE: + throw new UnsupportedError( + "Precedence predicates are not supported in lexers."); + case TransitionType.PREDICATE: + /* Track traversing semantic predicates. If we traverse, + we cannot add a DFA state for this "reach" computation + because the DFA would not test the predicate again in the + future. Rather than creating collections of semantic predicates + like v3 and testing them on prediction, v4 will test them on the + fly all the time using the ATN not the DFA. This is slower but + semantically it's not used that often. One of the key elements to + this predicate mechanism is not adding DFA states that see + predicates immediately afterwards in the ATN. For example, + + a : ID {p1}? | ID {p2}? ; + + should create the start state for rule 'a' (to save start state + competition), but should not create target of ID state. The + collection of ATN states the following ID references includes + states reached by traversing predicates. Since this is when we + test them, we cannot cash the DFA state target of ID. + */ + PredicateTransition pt = t; + if (debug) { + log("EVAL rule ${pt.ruleIndex}:${pt.predIndex}", + level: Level.FINE.value); + } + configs.hasSemanticContext = true; + if (evaluatePredicate(input, pt.ruleIndex, pt.predIndex, speculative)) { + c = new LexerATNConfig.dup(config, t.target); + } + break; + case TransitionType.ACTION: + if (config.context == null || config.context.hasEmptyPath()) { + // execute actions anywhere in the start rule for a token. + // + // TODO: if the entry rule is invoked recursively, some + // actions may be executed during the recursive call. The + // problem can appear when hasEmptyPath() is true but + // isEmpty is false. In this case, the config needs to be + // split into two contexts - one with just the empty path + // and another with everything but the empty path. + // Unfortunately, the current algorithm does not allow + // getEpsilonTarget to return two configurations, so + // additional modifications are needed before we can support + // the split operation. + LexerActionExecutor lexerActionExecutor = LexerActionExecutor.append( + config.lexerActionExecutor, + atn.lexerActions[(t as ActionTransition).actionIndex]); + c = new LexerATNConfig.dup(config, t.target, + lexerActionExecutor: lexerActionExecutor); + } else { + // ignore actions in referenced rules + c = new LexerATNConfig.dup(config, t.target); + } + break; + + case TransitionType.EPSILON: + c = new LexerATNConfig.dup(config, t.target); + break; + + case TransitionType.ATOM: + case TransitionType.RANGE: + case TransitionType.SET: + if (treatEofAsEpsilon) { + if (t.matches( + IntStream.EOF, Lexer.MIN_CHAR_VALUE, Lexer.MAX_CHAR_VALUE)) { + c = new LexerATNConfig.dup(config, t.target); + break; + } + } + break; + case TransitionType.NOT_SET: + break; + case TransitionType.WILDCARD: + break; + case TransitionType.INVALID: + throw ArgumentError.value(t.type, "TransitionType"); + break; + } + + return c; + } + + /** + * Evaluate a predicate specified in the lexer. + * + *

If [speculative] is [true], this method was called before + * {@link #consume} for the matched character. This method should call + * {@link #consume} before evaluating the predicate to ensure position + * sensitive values, including {@link Lexer#getText}, {@link Lexer#getLine}, + * and {@link Lexer#getCharPositionInLine}, properly reflect the current + * lexer state. This method should restore [input] and the simulator + * to the original state before returning (i.e. undo the actions made by the + * call to {@link #consume}.

+ * + * @param input The input stream. + * @param ruleIndex The rule containing the predicate. + * @param predIndex The index of the predicate within the rule. + * @param speculative [true] if the current index in [input] is + * one character before the predicate's location. + * + * @return [true] if the specified predicate evaluates to + * [true]. + */ + bool evaluatePredicate( + CharStream input, int ruleIndex, int predIndex, bool speculative) { + // assume true if no recognizer was provided + if (recog == null) { + return true; + } + + if (!speculative) { + return recog.sempred(null, ruleIndex, predIndex); + } + + int savedCharPositionInLine = charPositionInLine; + int savedLine = line; + int index = input.index; + int marker = input.mark(); + try { + consume(input); + return recog.sempred(null, ruleIndex, predIndex); + } finally { + charPositionInLine = savedCharPositionInLine; + line = savedLine; + input.seek(index); + input.release(marker); + } + } + + void captureSimState(SimState settings, CharStream input, DFAState dfaState) { + settings.index = input.index; + settings.line = line; + settings.charPos = charPositionInLine; + settings.dfaState = dfaState; + } + + DFAState addDFAEdgeByConfig(DFAState from, int t, ATNConfigSet q) { + /* leading to this call, ATNConfigSet.hasSemanticContext is used as a + * marker indicating dynamic predicate evaluation makes this edge + * dependent on the specific input sequence, so the static edge in the + * DFA should be omitted. The target DFAState is still created since + * execATN has the ability to resynchronize with the DFA state cache + * following the predicate evaluation step. + * + * TJP notes: next time through the DFA, we see a pred again and eval. + * If that gets us to a previously created (but dangling) DFA + * state, we can continue in pure DFA mode from there. + */ + bool suppressEdge = q.hasSemanticContext; + q.hasSemanticContext = false; + + DFAState to = addDFAState(q); + + if (suppressEdge) { + return to; + } + + addDFAEdge(from, t, to); + return to; + } + + void addDFAEdge(DFAState p, int t, DFAState q) { + if (t < MIN_DFA_EDGE || t > MAX_DFA_EDGE) { + // Only track edges within the DFA bounds + return; + } + + if (debug) { + log("EDGE $p -> $q upon ${String.fromCharCode(t)}", + level: Level.FINE.value); + } + + if (p.edges == null) { + // make room for tokens 1..n and -1 masquerading as index 0 + p.edges = List(MAX_DFA_EDGE - MIN_DFA_EDGE + 1); + } + p.edges[t - MIN_DFA_EDGE] = q; // connect + } + + /** Add a new DFA state if there isn't one with this set of + configurations already. This method also detects the first + configuration containing an ATN rule stop state. Later, when + traversing the DFA, we will know which rule to accept. + */ + + DFAState addDFAState(ATNConfigSet configs) { + /* the lexer evaluates predicates on-the-fly; by this point configs + * should not contain any configurations with unevaluated predicates. + */ + assert(!configs.hasSemanticContext); + + DFAState proposed = new DFAState(configs: configs); + ATNConfig firstConfigWithRuleStopState = null; + for (ATNConfig c in configs) { + if (c.state is RuleStopState) { + firstConfigWithRuleStopState = c; + break; + } + } + + if (firstConfigWithRuleStopState != null) { + proposed.isAcceptState = true; + proposed.lexerActionExecutor = + (firstConfigWithRuleStopState as LexerATNConfig) + .lexerActionExecutor; + proposed.prediction = + atn.ruleToTokenType[firstConfigWithRuleStopState.state.ruleIndex]; + } + + DFA dfa = decisionToDFA[mode]; + DFAState existing = dfa.states[proposed]; + if (existing != null) return existing; + + DFAState newState = proposed; + + newState.stateNumber = dfa.states.length; + configs.readOnly = true; + newState.configs = configs; + dfa.states[newState] = newState; + return newState; + } + + DFA getDFA(int mode) { + return decisionToDFA[mode]; + } + + /** Get the text matched so far for the current token. + */ + + String getText(CharStream input) { + // index is first lookahead char, don't include. + return input.getText(Interval.of(startIndex, input.index - 1)); + } + + void consume(CharStream input) { + int curChar = input.LA(1); + if (curChar == 10) { // Is new line + line++; + charPositionInLine = 0; + } else { + charPositionInLine++; + } + input.consume(); + } + + String getTokenName(int t) { + if (t == -1) return "EOF"; + //if ( atn.g!=null ) return atn.g.getTokenDisplayName(t); + return "'${String.fromCharCode(t)}'"; + } +} diff --git a/runtime/Dart/lib/src/atn/src/parser_atn_simulator.dart b/runtime/Dart/lib/src/atn/src/parser_atn_simulator.dart new file mode 100644 index 000000000..01be3ba57 --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/parser_atn_simulator.dart @@ -0,0 +1,2701 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:collection'; +import 'dart:developer'; + +import 'package:logging/logging.dart'; + +import '../../vocabulary.dart'; +import '../../dfa/dfa.dart'; +import '../../error/error.dart'; +import '../../input_stream.dart'; +import '../../interval_set.dart'; +import '../../misc/pair.dart'; +import '../../parser.dart'; +import '../../parser_rule_context.dart'; +import '../../prediction_context.dart'; +import '../../rule_context.dart'; +import '../../token.dart'; +import '../../token_stream.dart'; +import '../../util/bit_set.dart'; +import '../../util/murmur_hash.dart'; +import 'atn.dart'; +import 'atn_config.dart'; +import 'atn_config_set.dart'; +import 'atn_simulator.dart'; +import 'atn_state.dart'; +import 'semantic_context.dart'; +import 'transition.dart'; + +/** + * The embodiment of the adaptive LL(*), ALL(*), parsing strategy. + * + *

+ * The basic complexity of the adaptive strategy makes it harder to understand. + * We begin with ATN simulation to build paths in a DFA. Subsequent prediction + * requests go through the DFA first. If they reach a state without an edge for + * the current symbol, the algorithm fails over to the ATN simulation to + * complete the DFA path for the current input (until it finds a conflict state + * or uniquely predicting state).

+ * + *

+ * All of that is done without using the outer context because we want to create + * a DFA that is not dependent upon the rule invocation stack when we do a + * prediction. One DFA works in all contexts. We avoid using context not + * necessarily because it's slower, although it can be, but because of the DFA + * caching problem. The closure routine only considers the rule invocation stack + * created during prediction beginning in the decision rule. For example, if + * prediction occurs without invoking another rule's ATN, there are no context + * stacks in the configurations. When lack of context leads to a conflict, we + * don't know if it's an ambiguity or a weakness in the strong LL(*) parsing + * strategy (versus full LL(*)).

+ * + *

+ * When SLL yields a configuration set with conflict, we rewind the input and + * retry the ATN simulation, this time using full outer context without adding + * to the DFA. Configuration context stacks will be the full invocation stacks + * from the start rule. If we get a conflict using full context, then we can + * definitively say we have a true ambiguity for that input sequence. If we + * don't get a conflict, it implies that the decision is sensitive to the outer + * context. (It is not context-sensitive in the sense of context-sensitive + * grammars.)

+ * + *

+ * The next time we reach this DFA state with an SLL conflict, through DFA + * simulation, we will again retry the ATN simulation using full context mode. + * This is slow because we can't save the results and have to "interpret" the + * ATN each time we get that input.

+ * + *

+ * CACHING FULL CONTEXT PREDICTIONS

+ * + *

+ * We could cache results from full context to predicted alternative easily and + * that saves a lot of time but doesn't work in presence of predicates. The set + * of visible predicates from the ATN start state changes depending on the + * context, because closure can fall off the end of a rule. I tried to cache + * tuples (stack context, semantic context, predicted alt) but it was slower + * than interpreting and much more complicated. Also required a huge amount of + * memory. The goal is not to create the world's fastest parser anyway. I'd like + * to keep this algorithm simple. By launching multiple threads, we can improve + * the speed of parsing across a large number of files.

+ * + *

+ * There is no strict ordering between the amount of input used by SLL vs LL, + * which makes it really hard to build a cache for full context. Let's say that + * we have input A B C that leads to an SLL conflict with full context X. That + * implies that using X we might only use A B but we could also use A B C D to + * resolve conflict. Input A B C D could predict alternative 1 in one position + * in the input and A B C E could predict alternative 2 in another position in + * input. The conflicting SLL configurations could still be non-unique in the + * full context prediction, which would lead us to requiring more input than the + * original A B C. To make a prediction cache work, we have to track the exact + * input used during the previous prediction. That amounts to a cache that maps + * X to a specific DFA for that context.

+ * + *

+ * Something should be done for left-recursive expression predictions. They are + * likely LL(1) + pred eval. Easier to do the whole SLL unless error and retry + * with full LL thing Sam does.

+ * + *

+ * AVOIDING FULL CONTEXT PREDICTION

+ * + *

+ * We avoid doing full context retry when the outer context is empty, we did not + * dip into the outer context by falling off the end of the decision state rule, + * or when we force SLL mode.

+ * + *

+ * As an example of the not dip into outer context case, consider as super + * constructor calls versus function calls. One grammar might look like + * this:

+ * + * 
+ * ctorBody
+ *   : '{' superCall? stat* '}'
+ *   ;
+ *
+ * + *

+ * Or, you might see something like

+ * + * 
+ * stat
+ *   : superCall ';'
+ *   | expression ';'
+ *   | ...
+ *   ;
+ *
+ * + *

+ * In both cases I believe that no closure operations will dip into the outer + * context. In the first case ctorBody in the worst case will stop at the '}'. + * In the 2nd case it should stop at the ';'. Both cases should stay within the + * entry rule and not dip into the outer context.

+ * + *

+ * PREDICATES

+ * + *

+ * Predicates are always evaluated if present in either SLL or LL both. SLL and + * LL simulation deals with predicates differently. SLL collects predicates as + * it performs closure operations like ANTLR v3 did. It delays predicate + * evaluation until it reaches and accept state. This allows us to cache the SLL + * ATN simulation whereas, if we had evaluated predicates on-the-fly during + * closure, the DFA state configuration sets would be different and we couldn't + * build up a suitable DFA.

+ * + *

+ * When building a DFA accept state during ATN simulation, we evaluate any + * predicates and return the sole semantically valid alternative. If there is + * more than 1 alternative, we report an ambiguity. If there are 0 alternatives, + * we throw an exception. Alternatives without predicates act like they have + * true predicates. The simple way to think about it is to strip away all + * alternatives with false predicates and choose the minimum alternative that + * remains.

+ * + *

+ * When we start in the DFA and reach an accept state that's predicated, we test + * those and return the minimum semantically viable alternative. If no + * alternatives are viable, we throw an exception.

+ * + *

+ * During full LL ATN simulation, closure always evaluates predicates and + * on-the-fly. This is crucial to reducing the configuration set size during + * closure. It hits a landmine when parsing with the Java grammar, for example, + * without this on-the-fly evaluation.

+ * + *

+ * SHARING DFA

+ * + *

+ * All instances of the same parser share the same decision DFAs through a + * static field. Each instance gets its own ATN simulator but they share the + * same {@link #decisionToDFA} field. They also share a + * [PredictionContextCache] object that makes sure that all + * [PredictionContext] objects are shared among the DFA states. This makes + * a big size difference.

+ * + *

+ * THREAD SAFETY

+ * + *

+ * The [ParserATNSimulator] locks on the {@link #decisionToDFA} field when + * it adds a new DFA object to that array. {@link #addDFAEdge} + * locks on the DFA for the current decision when setting the + * {@link DFAState#edges} field. {@link #addDFAState} locks on + * the DFA for the current decision when looking up a DFA state to see if it + * already exists. We must make sure that all requests to add DFA states that + * are equivalent result in the same shared DFA object. This is because lots of + * threads will be trying to update the DFA at once. The + * {@link #addDFAState} method also locks inside the DFA lock + * but this time on the shared context cache when it rebuilds the + * configurations' [PredictionContext] objects using cached + * subgraphs/nodes. No other locking occurs, even during DFA simulation. This is + * safe as long as we can guarantee that all threads referencing + * {@code s.edge[t]} get the same physical target [DFAState], or + * null. Once into the DFA, the DFA simulation does not reference the + * {@link DFA#states} map. It follows the {@link DFAState#edges} field to new + * targets. The DFA simulator will either find {@link DFAState#edges} to be + * null, to be non-null and {@code dfa.edges[t]} null, or + * {@code dfa.edges[t]} to be non-null. The + * {@link #addDFAEdge} method could be racing to set the field + * but in either case the DFA simulator works; if null, and requests ATN + * simulation. It could also race trying to get {@code dfa.edges[t]}, but either + * way it will work because it's not doing a test and set operation.

+ * + *

+ * Starting with SLL then failing to combined SLL/LL (Two-Stage + * Parsing)

+ * + *

+ * Sam pointed out that if SLL does not give a syntax error, then there is no + * point in doing full LL, which is slower. We only have to try LL if we get a + * syntax error. For maximum speed, Sam starts the parser set to pure SLL + * mode with the [BailErrorStrategy]:

+ * + * 
+ * parser.{@link Parser#interpreter interpreter}.{@link #setPredictionMode setPredictionMode}{@code (}{@link PredictionMode#SLL}{@code )};
+ * parser.{@link Parser#setErrorHandler setErrorHandler}(new [BailErrorStrategy]());
+ *
+ * + *

+ * If it does not get a syntax error, then we're done. If it does get a syntax + * error, we need to retry with the combined SLL/LL strategy.

+ * + *

+ * The reason this works is as follows. If there are no SLL conflicts, then the + * grammar is SLL (at least for that input set). If there is an SLL conflict, + * the full LL analysis must yield a set of viable alternatives which is a + * subset of the alternatives reported by SLL. If the LL set is a singleton, + * then the grammar is LL but not SLL. If the LL set is the same size as the SLL + * set, the decision is SLL. If the LL set has size > 1, then that decision + * is truly ambiguous on the current input. If the LL set is smaller, then the + * SLL conflict resolution might choose an alternative that the full LL would + * rule out as a possibility based upon better context information. If that's + * the case, then the SLL parse will definitely get an error because the full LL + * analysis says it's not viable. If SLL conflict resolution chooses an + * alternative within the LL set, them both SLL and LL would choose the same + * alternative because they both choose the minimum of multiple conflicting + * alternatives.

+ * + *

+ * Let's say we have a set of SLL conflicting alternatives {@code {1, 2, 3}} and + * a smaller LL set called s. If s is {@code {2, 3}}, then SLL + * parsing will get an error because SLL will pursue alternative 1. If + * s is {@code {1, 2}} or {@code {1, 3}} then both SLL and LL will + * choose the same alternative because alternative one is the minimum of either + * set. If s is {@code {2}} or {@code {3}} then SLL will get a syntax + * error. If s is {@code {1}} then SLL will succeed.

+ * + *

+ * Of course, if the input is invalid, then we will get an error for sure in + * both SLL and LL parsing. Erroneous input will therefore require 2 passes over + * the input.

+ */ +class ParserATNSimulator extends ATNSimulator { + static final bool debug = false; + static final bool debug_list_atn_decisions = false; + static final bool dfa_debug = false; + static final bool retry_debug = false; + + /** Just in case this optimization is bad, add an ENV variable to turn it off */ + static final bool TURN_OFF_LR_LOOP_ENTRY_BRANCH_OPT = + bool.fromEnvironment("TURN_OFF_LR_LOOP_ENTRY_BRANCH_OPT"); + + final Parser parser; + + final List decisionToDFA; + + /** SLL, LL, or LL + exact ambig detection? */ + + PredictionMode predictionMode = PredictionMode.LL; + + /** Each prediction operation uses a cache for merge of prediction contexts. + * Don't keep around as it wastes huge amounts of memory. DoubleKeyMap + * isn't synchronized but we're ok since two threads shouldn't reuse same + * parser/atnsim object because it can only handle one input at a time. + * This maps graphs a and b to merged result c. (a,b)→c. We can avoid + * the merge if we ever see a and b again. Note that (b,a)→c should + * also be examined during cache lookup. + */ + Map, PredictionContext> mergeCache; + + // LAME globals to avoid parameters!!!!! I need these down deep in predTransition + TokenStream input; + int startIndex = 0; + ParserRuleContext _outerContext; + DFA _dfa; + + ParserATNSimulator(Parser this.parser, ATN atn, this.decisionToDFA, + PredictionContextCache sharedContextCache) + : super(atn, sharedContextCache) { + // DOTGenerator dot = new DOTGenerator(null); + // log(dot.getDOT(atn.rules.get(0), parser.getRuleNames())); + // log(dot.getDOT(atn.rules.get(1), parser.getRuleNames())); + } + + void reset() {} + + void clearDFA() { + for (int d = 0; d < decisionToDFA.length; d++) { + decisionToDFA[d] = new DFA(atn.getDecisionState(d), d); + } + } + + int adaptivePredict( + TokenStream input_, int decision, ParserRuleContext outerContext) { + if (debug || debug_list_atn_decisions) { + log("adaptivePredict decision $decision" + + " exec LA(1)==" + + getLookaheadName(input_) + + " line ${input_.LT(1).line}:${input_.LT(1).charPositionInLine}"); + } + + input = input_; + startIndex = input_.index; + _outerContext = outerContext; + DFA dfa = decisionToDFA[decision]; + _dfa = dfa; + + int m = input_.mark(); + int index = startIndex; + + // Now we are certain to have a specific decision's DFA + // But, do we still need an initial state? + try { + DFAState s0; + if (dfa.isPrecedenceDfa()) { + // the start state for a precedence DFA depends on the current + // parser precedence, and is provided by a DFA method. + s0 = dfa.getPrecedenceStartState(parser.precedence); + } else { + // the start state for a "regular" DFA is just s0 + s0 = dfa.s0; + } + + if (s0 == null) { + if (outerContext == null) outerContext = ParserRuleContext.EMPTY; + if (debug || debug_list_atn_decisions) { + log("predictATN decision ${dfa.decision}" + + " exec LA(1)==" + + getLookaheadName(input_) + + ", outerContext=" + + outerContext.toString(recog: parser)); + } + + bool fullCtx = false; + ATNConfigSet s0_closure = computeStartState( + dfa.atnStartState, ParserRuleContext.EMPTY, fullCtx); + + if (dfa.isPrecedenceDfa()) { + /* If this is a precedence DFA, we use applyPrecedenceFilter + * to convert the computed start state to a precedence start + * state. We then use DFA.setPrecedenceStartState to set the + * appropriate start state for the precedence level rather + * than simply setting DFA.s0. + */ + dfa.s0.configs = + s0_closure; // not used for prediction but useful to know start configs anyway + s0_closure = applyPrecedenceFilter(s0_closure); + s0 = addDFAState(dfa, new DFAState(configs: s0_closure)); + dfa.setPrecedenceStartState(parser.precedence, s0); + } else { + s0 = addDFAState(dfa, new DFAState(configs: s0_closure)); + dfa.s0 = s0; + } + } + + int alt = execATN(dfa, s0, input_, index, outerContext); + if (debug) + log("DFA after predictATN: " + dfa.toString(parser.vocabulary)); + return alt; + } finally { + mergeCache = null; // wack cache after each prediction + _dfa = null; + input_.seek(index); + input_.release(m); + } + } + + /** Performs ATN simulation to compute a predicted alternative based + * upon the remaining input, but also updates the DFA cache to avoid + * having to traverse the ATN again for the same input sequence. + + There are some key conditions we're looking for after computing a new + set of ATN configs (proposed DFA state): + * if the set is empty, there is no viable alternative for current symbol + * does the state uniquely predict an alternative? + * does the state have a conflict that would prevent us from + putting it on the work list? + + We also have some key operations to do: + * add an edge from previous DFA state to potentially new DFA state, D, + upon current symbol but only if adding to work list, which means in all + cases except no viable alternative (and possibly non-greedy decisions?) + * collecting predicates and adding semantic context to DFA accept states + * adding rule context to context-sensitive DFA accept states + * consuming an input symbol + * reporting a conflict + * reporting an ambiguity + * reporting a context sensitivity + * reporting insufficient predicates + + cover these cases: + dead end + single alt + single alt + preds + conflict + conflict + preds + */ + int execATN(DFA dfa, DFAState s0, TokenStream input, int startIndex, + ParserRuleContext outerContext) { + if (debug || debug_list_atn_decisions) { + log("execATN decision ${dfa.decision}" + + " exec LA(1)==" + + getLookaheadName(input) + + " line ${input.LT(1).line}" + + ":${input.LT(1).charPositionInLine}"); + } + + DFAState previousD = s0; + + if (debug) log("s0 = $s0"); + + int t = input.LA(1); + + while (true) { + // while more work + DFAState D = getExistingTargetState(previousD, t); + if (D == null) { + D = computeTargetState(dfa, previousD, t); + } + + if (D == ATNSimulator.ERROR) { + // if any configs in previous dipped into outer context, that + // means that input up to t actually finished entry rule + // at least for SLL decision. Full LL doesn't dip into outer + // so don't need special case. + // We will get an error no matter what so delay until after + // decision; better error message. Also, no reachable target + // ATN states in SLL implies LL will also get nowhere. + // If conflict in states that dip out, choose min since we + // will get error no matter what. + NoViableAltException e = + noViableAlt(input, outerContext, previousD.configs, startIndex); + input.seek(startIndex); + int alt = getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule( + previousD.configs, outerContext); + if (alt != ATN.INVALID_ALT_NUMBER) { + return alt; + } + throw e; + } + + if (D.requiresFullContext && predictionMode != PredictionMode.SLL) { + // IF PREDS, MIGHT RESOLVE TO SINGLE ALT => SLL (or syntax error) + BitSet conflictingAlts = D.configs.conflictingAlts; + if (D.predicates != null) { + if (debug) log("DFA state has preds in DFA sim LL failover"); + int conflictIndex = input.index; + if (conflictIndex != startIndex) { + input.seek(startIndex); + } + + conflictingAlts = + evalSemanticContext(D.predicates, outerContext, true); + if (conflictingAlts.cardinality == 1) { + if (debug) log("Full LL avoided"); + return conflictingAlts.nextset(0); + } + + if (conflictIndex != startIndex) { + // restore the index so reporting the fallback to full + // context occurs with the index at the correct spot + input.seek(conflictIndex); + } + } + + if (dfa_debug) log("ctx sensitive state ${outerContext} in $D"); + bool fullCtx = true; + ATNConfigSet s0_closure = + computeStartState(dfa.atnStartState, outerContext, fullCtx); + reportAttemptingFullContext( + dfa, conflictingAlts, D.configs, startIndex, input.index); + int alt = execATNWithFullContext( + dfa, D, s0_closure, input, startIndex, outerContext); + return alt; + } + + if (D.isAcceptState) { + if (D.predicates == null) { + return D.prediction; + } + + int stopIndex = input.index; + input.seek(startIndex); + BitSet alts = evalSemanticContext(D.predicates, outerContext, true); + switch (alts.cardinality) { + case 0: + throw noViableAlt(input, outerContext, D.configs, startIndex); + + case 1: + return alts.nextset(0); + + default: + // report ambiguity after predicate evaluation to make sure the correct + // set of ambig alts is reported. + reportAmbiguity( + dfa, D, startIndex, stopIndex, false, alts, D.configs); + return alts.nextset(0); + } + } + + previousD = D; + + if (t != IntStream.EOF) { + input.consume(); + t = input.LA(1); + } + } + } + + /** + * Get an existing target state for an edge in the DFA. If the target state + * for the edge has not yet been computed or is otherwise not available, + * this method returns null. + * + * @param previousD The current DFA state + * @param t The next input symbol + * @return The existing target DFA state for the given input symbol + * [t], or null if the target state for this edge is not + * already cached + */ + DFAState getExistingTargetState(DFAState previousD, int t) { + List edges = previousD.edges; + if (edges == null || t + 1 < 0 || t + 1 >= edges.length) { + return null; + } + + return edges[t + 1]; + } + + /** + * Compute a target state for an edge in the DFA, and attempt to add the + * computed state and corresponding edge to the DFA. + * + * @param dfa The DFA + * @param previousD The current DFA state + * @param t The next input symbol + * + * @return The computed target DFA state for the given input symbol + * [t]. If [t] does not lead to a valid DFA state, this method + * returns {@link #ERROR}. + */ + DFAState computeTargetState(DFA dfa, DFAState previousD, int t) { + ATNConfigSet reach = computeReachSet(previousD.configs, t, false); + if (reach == null) { + addDFAEdge(dfa, previousD, t, ATNSimulator.ERROR); + return ATNSimulator.ERROR; + } + + // create new target state; we'll add to DFA after it's complete + DFAState D = new DFAState(configs: reach); + + int predictedAlt = getUniqueAlt(reach); + + if (debug) { + List altSubSets = + PredictionModeExtension.getConflictingAltSubsets(reach); + log("SLL altSubSets=$altSubSets" + + ", configs=$reach" + + ", predict=$predictedAlt, allSubsetsConflict=${PredictionModeExtension.allSubsetsConflict(altSubSets)}" + + ", conflictingAlts=${getConflictingAlts(reach)}"); + } + + if (predictedAlt != ATN.INVALID_ALT_NUMBER) { + // NO CONFLICT, UNIQUELY PREDICTED ALT + D.isAcceptState = true; + D.configs.uniqueAlt = predictedAlt; + D.prediction = predictedAlt; + } else if (PredictionModeExtension.hasSLLConflictTerminatingPrediction( + predictionMode, reach)) { + // MORE THAN ONE VIABLE ALTERNATIVE + D.configs.conflictingAlts = getConflictingAlts(reach); + D.requiresFullContext = true; + // in SLL-only mode, we will stop at this state and return the minimum alt + D.isAcceptState = true; + D.prediction = D.configs.conflictingAlts.nextset(0); + } + + if (D.isAcceptState && D.configs.hasSemanticContext) { + predicateDFAState(D, atn.getDecisionState(dfa.decision)); + if (D.predicates != null) { + D.prediction = ATN.INVALID_ALT_NUMBER; + } + } + + // all adds to dfa are done after we've created full D state + D = addDFAEdge(dfa, previousD, t, D); + return D; + } + + void predicateDFAState(DFAState dfaState, DecisionState decisionState) { + // We need to test all predicates, even in DFA states that + // uniquely predict alternative. + int nalts = decisionState.numberOfTransitions; + // Update DFA so reach becomes accept state with (predicate,alt) + // pairs if preds found for conflicting alts + BitSet altsToCollectPredsFrom = + getConflictingAltsOrUniqueAlt(dfaState.configs); + List altToPred = + getPredsForAmbigAlts(altsToCollectPredsFrom, dfaState.configs, nalts); + if (altToPred != null) { + dfaState.predicates = + getPredicatePredictions(altsToCollectPredsFrom, altToPred); + dfaState.prediction = ATN.INVALID_ALT_NUMBER; // make sure we use preds + } else { + // There are preds in configs but they might go away + // when OR'd together like {p}? || NONE == NONE. If neither + // alt has preds, resolve to min alt + dfaState.prediction = altsToCollectPredsFrom.nextset(0); + } + } + + // comes back with reach.uniqueAlt set to a valid alt + int execATNWithFullContext( + DFA dfa, + DFAState D, // how far we got in SLL DFA before failing over + ATNConfigSet s0, + TokenStream input, + int startIndex, + ParserRuleContext outerContext) { + if (debug || debug_list_atn_decisions) { + log("execATNWithFullContext $s0"); + } + bool fullCtx = true; + bool foundExactAmbig = false; + ATNConfigSet reach = null; + ATNConfigSet previous = s0; + input.seek(startIndex); + int t = input.LA(1); + int predictedAlt; + while (true) { + // while more work +// log("LL REACH "+getLookaheadName(input)+ +// " from configs.size="+previous.length+ +// " line "+input.LT(1).getLine()+":"+input.LT(1).getCharPositionInLine()); + reach = computeReachSet(previous, t, fullCtx); + if (reach == null) { + // if any configs in previous dipped into outer context, that + // means that input up to t actually finished entry rule + // at least for LL decision. Full LL doesn't dip into outer + // so don't need special case. + // We will get an error no matter what so delay until after + // decision; better error message. Also, no reachable target + // ATN states in SLL implies LL will also get nowhere. + // If conflict in states that dip out, choose min since we + // will get error no matter what. + NoViableAltException e = + noViableAlt(input, outerContext, previous, startIndex); + input.seek(startIndex); + int alt = getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule( + previous, outerContext); + if (alt != ATN.INVALID_ALT_NUMBER) { + return alt; + } + throw e; + } + + List altSubSets = + PredictionModeExtension.getConflictingAltSubsets(reach); + if (debug) { + log("LL altSubSets=$altSubSets" + + ", predict=${PredictionModeExtension.getUniqueAlt(altSubSets)}" + + ", resolvesToJustOneViableAlt=${PredictionModeExtension.resolvesToJustOneViableAlt(altSubSets)}"); + } + +// log("altSubSets: "+altSubSets); +// log("reach="+reach+", "+reach.conflictingAlts, level: Level.SEVERE.value); + reach.uniqueAlt = getUniqueAlt(reach); + // unique prediction? + if (reach.uniqueAlt != ATN.INVALID_ALT_NUMBER) { + predictedAlt = reach.uniqueAlt; + break; + } + if (predictionMode != PredictionMode.LL_EXACT_AMBIG_DETECTION) { + predictedAlt = + PredictionModeExtension.resolvesToJustOneViableAlt(altSubSets); + if (predictedAlt != ATN.INVALID_ALT_NUMBER) { + break; + } + } else { + // In exact ambiguity mode, we never try to terminate early. + // Just keeps scarfing until we know what the conflict is + if (PredictionModeExtension.allSubsetsConflict(altSubSets) && + PredictionModeExtension.allSubsetsEqual(altSubSets)) { + foundExactAmbig = true; + predictedAlt = PredictionModeExtension.getSingleViableAlt(altSubSets); + break; + } + // else there are multiple non-conflicting subsets or + // we're not sure what the ambiguity is yet. + // So, keep going. + } + + previous = reach; + if (t != IntStream.EOF) { + input.consume(); + t = input.LA(1); + } + } + + // If the configuration set uniquely predicts an alternative, + // without conflict, then we know that it's a full LL decision + // not SLL. + if (reach.uniqueAlt != ATN.INVALID_ALT_NUMBER) { + reportContextSensitivity( + dfa, predictedAlt, reach, startIndex, input.index); + return predictedAlt; + } + + // We do not check predicates here because we have checked them + // on-the-fly when doing full context prediction. + + /* + In non-exact ambiguity detection mode, we might actually be able to + detect an exact ambiguity, but I'm not going to spend the cycles + needed to check. We only emit ambiguity warnings in exact ambiguity + mode. + + For example, we might know that we have conflicting configurations. + But, that does not mean that there is no way forward without a + conflict. It's possible to have nonconflicting alt subsets as in: + + LL altSubSets=[{1, 2}, {1, 2}, {1}, {1, 2}] + + from + + [(17,1,[5 $]), (13,1,[5 10 $]), (21,1,[5 10 $]), (11,1,[$]), + (13,2,[5 10 $]), (21,2,[5 10 $]), (11,2,[$])] + + In this case, (17,1,[5 $]) indicates there is some next sequence that + would resolve this without conflict to alternative 1. Any other viable + next sequence, however, is associated with a conflict. We stop + looking for input because no amount of further lookahead will alter + the fact that we should predict alternative 1. We just can't say for + sure that there is an ambiguity without looking further. + */ + reportAmbiguity(dfa, D, startIndex, input.index, foundExactAmbig, + reach.alts, reach); + + return predictedAlt; + } + + ATNConfigSet computeReachSet(ATNConfigSet config, int t, bool fullCtx) { + if (debug) log("in computeReachSet, starting closure: $config"); + + if (mergeCache == null) { + mergeCache = {}; + } + + ATNConfigSet intermediate = new ATNConfigSet(fullCtx); + + /* Configurations already in a rule stop state indicate reaching the end + * of the decision rule (local context) or end of the start rule (full + * context). Once reached, these configurations are never updated by a + * closure operation, so they are handled separately for the performance + * advantage of having a smaller intermediate set when calling closure. + * + * For full-context reach operations, separate handling is required to + * ensure that the alternative matching the longest overall sequence is + * chosen when multiple such configurations can match the input. + */ + List skippedStopStates = null; + + // First figure out where we can reach on input t + for (ATNConfig c in config) { + if (debug) log("testing " + getTokenName(t) + " at " + c.toString()); + + if (c.state is RuleStopState) { + assert(c.context.isEmpty); + if (fullCtx || t == IntStream.EOF) { + if (skippedStopStates == null) { + skippedStopStates = []; + } + + skippedStopStates.add(c); + } + + continue; + } + + int n = c.state.numberOfTransitions; + for (int ti = 0; ti < n; ti++) { + // for each transition + Transition trans = c.state.transition(ti); + ATNState target = getReachableTarget(trans, t); + if (target != null) { + intermediate.add(new ATNConfig.dup(c, state: target), mergeCache); + } + } + } + + // Now figure out where the reach operation can take us... + + ATNConfigSet reach = null; + + /* This block optimizes the reach operation for intermediate sets which + * trivially indicate a termination state for the overall + * adaptivePredict operation. + * + * The conditions assume that intermediate + * contains all configurations relevant to the reach set, but this + * condition is not true when one or more configurations have been + * withheld in skippedStopStates, or when the current symbol is EOF. + */ + if (skippedStopStates == null && t != Token.EOF) { + if (intermediate.length == 1) { + // Don't pursue the closure if there is just one state. + // It can only have one alternative; just add to result + // Also don't pursue the closure if there is unique alternative + // among the configurations. + reach = intermediate; + } else if (getUniqueAlt(intermediate) != ATN.INVALID_ALT_NUMBER) { + // Also don't pursue the closure if there is unique alternative + // among the configurations. + reach = intermediate; + } + } + + /* If the reach set could not be trivially determined, perform a closure + * operation on the intermediate set to compute its initial value. + */ + if (reach == null) { + reach = new ATNConfigSet(fullCtx); + Set closureBusy = Set(); + bool treatEofAsEpsilon = t == Token.EOF; + for (ATNConfig c in intermediate) { + closure(c, reach, closureBusy, false, fullCtx, treatEofAsEpsilon); + } + } + + if (t == IntStream.EOF) { + /* After consuming EOF no additional input is possible, so we are + * only interested in configurations which reached the end of the + * decision rule (local context) or end of the start rule (full + * context). Update reach to contain only these configurations. This + * handles both explicit EOF transitions in the grammar and implicit + * EOF transitions following the end of the decision or start rule. + * + * When reach==intermediate, no closure operation was performed. In + * this case, removeAllConfigsNotInRuleStopState needs to check for + * reachable rule stop states as well as configurations already in + * a rule stop state. + * + * This is handled before the configurations in skippedStopStates, + * because any configurations potentially added from that list are + * already guaranteed to meet this condition whether or not it's + * required. + */ + reach = removeAllConfigsNotInRuleStopState(reach, reach == intermediate); + } + + /* If skippedStopStates is not null, then it contains at least one + * configuration. For full-context reach operations, these + * configurations reached the end of the start rule, in which case we + * only add them back to reach if no configuration during the current + * closure operation reached such a state. This ensures adaptivePredict + * chooses an alternative matching the longest overall sequence when + * multiple alternatives are viable. + */ + if (skippedStopStates != null && + (!fullCtx || + !PredictionModeExtension.hasConfigInRuleStopState(reach))) { + assert(!skippedStopStates.isEmpty); + for (ATNConfig c in skippedStopStates) { + reach.add(c, mergeCache); + } + } + + if (reach.isEmpty) return null; + return reach; + } + + /** + * Return a configuration set containing only the configurations from + * [configs] which are in a [RuleStopState]. If all + * configurations in [configs] are already in a rule stop state, this + * method simply returns [configs]. + * + *

When [lookToEndOfRule] is true, this method uses + * {@link ATN#nextTokens} for each configuration in [configs] which is + * not already in a rule stop state to see if a rule stop state is reachable + * from the configuration via epsilon-only transitions.

+ * + * @param configs the configuration set to update + * @param lookToEndOfRule when true, this method checks for rule stop states + * reachable by epsilon-only transitions from each configuration in + * [configs]. + * + * @return [configs] if all configurations in [configs] are in a + * rule stop state, otherwise return a new configuration set containing only + * the configurations from [configs] which are in a rule stop state + */ + ATNConfigSet removeAllConfigsNotInRuleStopState( + ATNConfigSet configs, bool lookToEndOfRule) { + if (PredictionModeExtension.allConfigsInRuleStopStates(configs)) { + return configs; + } + + ATNConfigSet result = new ATNConfigSet(configs.fullCtx); + for (ATNConfig config in configs) { + if (config.state is RuleStopState) { + result.add(config, mergeCache); + continue; + } + + if (lookToEndOfRule && config.state.onlyHasEpsilonTransitions()) { + IntervalSet nextTokens = atn.nextTokens(config.state); + if (nextTokens.contains(Token.EPSILON)) { + ATNState endOfRuleState = atn.ruleToStopState[config.state.ruleIndex]; + result.add( + new ATNConfig.dup(config, state: endOfRuleState), mergeCache); + } + } + } + + return result; + } + + ATNConfigSet computeStartState(ATNState p, RuleContext ctx, bool fullCtx) { + // always at least the implicit call to start rule + PredictionContext initialContext = + PredictionContext.fromRuleContext(atn, ctx); + ATNConfigSet configs = new ATNConfigSet(fullCtx); + + for (int i = 0; i < p.numberOfTransitions; i++) { + ATNState target = p.transition(i).target; + ATNConfig c = new ATNConfig(target, i + 1, initialContext); + Set closureBusy = Set(); + closure(c, configs, closureBusy, true, fullCtx, false); + } + + return configs; + } + + /* parrt internal source braindump that doesn't mess up + * external API spec. + context-sensitive in that they can only be properly evaluated + in the context of the proper prec argument. Without pruning, + these predicates are normal predicates evaluated when we reach + conflict state (or unique prediction). As we cannot evaluate + these predicates out of context, the resulting conflict leads + to full LL evaluation and nonlinear prediction which shows up + very clearly with fairly large expressions. + + Example grammar: + + e : e '*' e + | e '+' e + | INT + ; + + We convert that to the following: + + e[int prec] + : INT + ( {3>=prec}? '*' e[4] + | {2>=prec}? '+' e[3] + )* + ; + + The (..)* loop has a decision for the inner block as well as + an enter or exit decision, which is what concerns us here. At + the 1st + of input 1+2+3, the loop entry sees both predicates + and the loop exit also sees both predicates by falling off the + edge of e. This is because we have no stack information with + SLL and find the follow of e, which will hit the return states + inside the loop after e[4] and e[3], which brings it back to + the enter or exit decision. In this case, we know that we + cannot evaluate those predicates because we have fallen off + the edge of the stack and will in general not know which prec + parameter is the right one to use in the predicate. + + Because we have special information, that these are precedence + predicates, we can resolve them without failing over to full + LL despite their context sensitive nature. We make an + assumption that prec[-1] <= prec[0], meaning that the current + precedence level is greater than or equal to the precedence + level of recursive invocations above us in the stack. For + example, if predicate {3>=prec}? is true of the current prec, + then one option is to enter the loop to match it now. The + other option is to exit the loop and the left recursive rule + to match the current operator in rule invocation further up + the stack. But, we know that all of those prec are lower or + the same value and so we can decide to enter the loop instead + of matching it later. That means we can strip out the other + configuration for the exit branch. + + So imagine we have (14,1,$,{2>=prec}?) and then + (14,2,$-dipsIntoOuterContext,{2>=prec}?). The optimization + allows us to collapse these two configurations. We know that + if {2>=prec}? is true for the current prec parameter, it will + also be true for any prec from an invoking e call, indicated + by dipsIntoOuterContext. As the predicates are both true, we + have the option to evaluate them early in the decision start + state. We do this by stripping both predicates and choosing to + enter the loop as it is consistent with the notion of operator + precedence. It's also how the full LL conflict resolution + would work. + + The solution requires a different DFA start state for each + precedence level. + + The basic filter mechanism is to remove configurations of the + form (p, 2, pi) if (p, 1, pi) exists for the same p and pi. In + other words, for the same ATN state and predicate context, + remove any configuration associated with an exit branch if + there is a configuration associated with the enter branch. + + It's also the case that the filter evaluates precedence + predicates and resolves conflicts according to precedence + levels. For example, for input 1+2+3 at the first +, we see + prediction filtering + + [(11,1,[$],{3>=prec}?), (14,1,[$],{2>=prec}?), (5,2,[$],up=1), + (11,2,[$],up=1), (14,2,[$],up=1)],hasSemanticContext=true,dipsIntoOuterContext + + to + + [(11,1,[$]), (14,1,[$]), (5,2,[$],up=1)],dipsIntoOuterContext + + This filters because {3>=prec}? evals to true and collapses + (11,1,[$],{3>=prec}?) and (11,2,[$],up=1) since early conflict + resolution based upon rules of operator precedence fits with + our usual match first alt upon conflict. + + We noticed a problem where a recursive call resets precedence + to 0. Sam's fix: each config has flag indicating if it has + returned from an expr[0] call. then just don't filter any + config with that flag set. flag is carried along in + closure(). so to avoid adding field, set bit just under sign + bit of dipsIntoOuterContext (SUPPRESS_PRECEDENCE_FILTER). + With the change you filter "unless (p, 2, pi) was reached + after leaving the rule stop state of the LR rule containing + state p, corresponding to a rule invocation with precedence + level 0" + */ + + /** + * This method transforms the start state computed by + * {@link #computeStartState} to the special start state used by a + * precedence DFA for a particular precedence value. The transformation + * process applies the following changes to the start state's configuration + * set. + * + *
    + *
  1. Evaluate the precedence predicates for each configuration using + * {@link SemanticContext#evalPrecedence}.
    2. + *
  2. When {@link ATNConfig#isPrecedenceFilterSuppressed} is [false], + * remove all configurations which predict an alternative greater than 1, + * for which another configuration that predicts alternative 1 is in the + * same ATN state with the same prediction context. This transformation is + * valid for the following reasons: + *
      + *
    • The closure block cannot contain any epsilon transitions which bypass + * the body of the closure, so all states reachable via alternative 1 are + * part of the precedence alternatives of the transformed left-recursive + * rule.
      • + *
    • The "primary" portion of a left recursive rule cannot contain an + * epsilon transition, so the only way an alternative other than 1 can exist + * in a state that is also reachable via alternative 1 is by nesting calls + * to the left-recursive rule, with the outer calls not being at the + * preferred precedence level. The + * {@link ATNConfig#isPrecedenceFilterSuppressed} property marks ATN + * configurations which do not meet this condition, and therefore are not + * eligible for elimination during the filtering process.
      • + *
    + *
    3. + *
+ * + *

+ * The prediction context must be considered by this filter to address + * situations like the following. + *

+ * + * 
+   * grammar TA;
+   * prog: statement* EOF;
+   * statement: letterA | statement letterA 'b' ;
+   * letterA: 'a';
+   *
+ * + *

+ * If the above grammar, the ATN state immediately before the token + * reference {@code 'a'} in [letterA] is reachable from the left edge + * of both the primary and closure blocks of the left-recursive rule + * [statement]. The prediction context associated with each of these + * configurations distinguishes between them, and prevents the alternative + * which stepped out to [prog] (and then back in to [statement] + * from being eliminated by the filter. + *

+ * + * @param configs The configuration set computed by + * {@link #computeStartState} as the start state for the DFA. + * @return The transformed configuration set representing the start state + * for a precedence DFA at a particular precedence level (determined by + * calling {@link Parser#getPrecedence}). + */ + ATNConfigSet applyPrecedenceFilter(ATNConfigSet configs) { + Map statesFromAlt1 = {}; + ATNConfigSet configSet = new ATNConfigSet(configs.fullCtx); + for (ATNConfig config in configs) { + // handle alt 1 first + if (config.alt != 1) { + continue; + } + + SemanticContext updatedContext = + config.semanticContext.evalPrecedence(parser, _outerContext); + if (updatedContext == null) { + // the configuration was eliminated + continue; + } + + statesFromAlt1[config.state.stateNumber] = config.context; + if (updatedContext != config.semanticContext) { + configSet.add( + new ATNConfig.dup(config, semanticContext: updatedContext), + mergeCache); + } else { + configSet.add(config, mergeCache); + } + } + + for (ATNConfig config in configs) { + if (config.alt == 1) { + // already handled + continue; + } + + if (!config.isPrecedenceFilterSuppressed()) { + /* In the future, this elimination step could be updated to also + * filter the prediction context for alternatives predicting alt>1 + * (basically a graph subtraction algorithm). + */ + PredictionContext context = statesFromAlt1[config.state.stateNumber]; + if (context != null && context == config.context) { + // eliminated + continue; + } + } + + configSet.add(config, mergeCache); + } + + return configSet; + } + + ATNState getReachableTarget(Transition trans, int ttype) { + if (trans.matches(ttype, 0, atn.maxTokenType)) { + return trans.target; + } + + return null; + } + + List getPredsForAmbigAlts( + BitSet ambigAlts, ATNConfigSet configs, int nalts) { + // REACH=[1|1|[]|0:0, 1|2|[]|0:1] + /* altToPred starts as an array of all null contexts. The entry at index i + * corresponds to alternative i. altToPred[i] may have one of three values: + * 1. null: no ATNConfig c is found such that c.alt==i + * 2. SemanticContext.NONE: At least one ATNConfig c exists such that + * c.alt==i and c.semanticContext==SemanticContext.NONE. In other words, + * alt i has at least one unpredicated config. + * 3. Non-NONE Semantic Context: There exists at least one, and for all + * ATNConfig c such that c.alt==i, c.semanticContext!=SemanticContext.NONE. + * + * From this, it is clear that NONE||anything==NONE. + */ + List altToPred = List(nalts + 1); + for (ATNConfig c in configs) { + if (ambigAlts[c.alt]) { + altToPred[c.alt] = + SemanticContext.or(altToPred[c.alt], c.semanticContext); + } + } + + int nPredAlts = 0; + for (int i = 1; i <= nalts; i++) { + if (altToPred[i] == null) { + altToPred[i] = SemanticContext.NONE; + } else if (altToPred[i] != SemanticContext.NONE) { + nPredAlts++; + } + } + +// // Optimize away p||p and p&&p TODO: optimize() was a no-op +// for (int i = 0; i < altToPred.length; i++) { +// altToPred[i] = altToPred[i].optimize(); +// } + + // nonambig alts are null in altToPred + if (nPredAlts == 0) altToPred = null; + if (debug) log("getPredsForAmbigAlts result $altToPred"); + return altToPred; + } + + List getPredicatePredictions( + BitSet ambigAlts, List altToPred) { + List pairs = []; + bool containsPredicate = false; + for (int i = 1; i < altToPred.length; i++) { + SemanticContext pred = altToPred[i]; + + // unpredicated is indicated by SemanticContext.NONE + assert(pred != null); + + if (ambigAlts != null && ambigAlts[i]) { + pairs.add(new PredPrediction(pred, i)); + } + if (pred != SemanticContext.NONE) containsPredicate = true; + } + + if (!containsPredicate) { + return null; + } + +// log(Arrays.toString(altToPred)+"->"+pairs); + return pairs; + } + + /** + * This method is used to improve the localization of error messages by + * choosing an alternative rather than throwing a + * [NoViableAltException] in particular prediction scenarios where the + * {@link #ERROR} state was reached during ATN simulation. + * + *

+ * The default implementation of this method uses the following + * algorithm to identify an ATN configuration which successfully parsed the + * decision entry rule. Choosing such an alternative ensures that the + * [ParserRuleContext] returned by the calling rule will be complete + * and valid, and the syntax error will be reported later at a more + * localized location.

+ * + *
    + *
  • If a syntactically valid path or paths reach the end of the decision rule and + * they are semantically valid if predicated, return the min associated alt.
    • + *
  • Else, if a semantically invalid but syntactically valid path exist + * or paths exist, return the minimum associated alt. + *
    • + *
  • Otherwise, return {@link ATN#INVALID_ALT_NUMBER}.
    • + *
+ * + *

+ * In some scenarios, the algorithm described above could predict an + * alternative which will result in a [FailedPredicateException] in + * the parser. Specifically, this could occur if the only configuration + * capable of successfully parsing to the end of the decision rule is + * blocked by a semantic predicate. By choosing this alternative within + * {@link #adaptivePredict} instead of throwing a + * [NoViableAltException], the resulting + * [FailedPredicateException] in the parser will identify the specific + * predicate which is preventing the parser from successfully parsing the + * decision rule, which helps developers identify and correct logic errors + * in semantic predicates. + *

+ * + * @param configs The ATN configurations which were valid immediately before + * the {@link #ERROR} state was reached + * @param outerContext The is the \gamma_0 initial parser context from the paper + * or the parser stack at the instant before prediction commences. + * + * @return The value to return from {@link #adaptivePredict}, or + * {@link ATN#INVALID_ALT_NUMBER} if a suitable alternative was not + * identified and {@link #adaptivePredict} should report an error instead. + */ + int getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule( + ATNConfigSet configs, ParserRuleContext outerContext) { + Pair sets = + splitAccordingToSemanticValidity(configs, outerContext); + ATNConfigSet semValidConfigs = sets.a; + ATNConfigSet semInvalidConfigs = sets.b; + int alt = getAltThatFinishedDecisionEntryRule(semValidConfigs); + if (alt != ATN.INVALID_ALT_NUMBER) { + // semantically/syntactically viable path exists + return alt; + } + // Is there a syntactically valid path with a failed pred? + if (semInvalidConfigs.length > 0) { + alt = getAltThatFinishedDecisionEntryRule(semInvalidConfigs); + if (alt != ATN.INVALID_ALT_NUMBER) { + // syntactically viable path exists + return alt; + } + } + return ATN.INVALID_ALT_NUMBER; + } + + int getAltThatFinishedDecisionEntryRule(ATNConfigSet configs) { + IntervalSet alts = new IntervalSet(); + for (ATNConfig c in configs) { + if (c.outerContextDepth > 0 || + (c.state is RuleStopState && c.context.hasEmptyPath())) { + alts.addOne(c.alt); + } + } + if (alts.length == 0) return ATN.INVALID_ALT_NUMBER; + return alts.minElement; + } + + /** Walk the list of configurations and split them according to + * those that have preds evaluating to true/false. If no pred, assume + * true pred and include in succeeded set. Returns Pair of sets. + * + * Create a new set so as not to alter the incoming parameter. + * + * Assumption: the input stream has been restored to the starting point + * prediction, which is where predicates need to evaluate. + */ + Pair splitAccordingToSemanticValidity( + ATNConfigSet configs, ParserRuleContext outerContext) { + ATNConfigSet succeeded = new ATNConfigSet(configs.fullCtx); + ATNConfigSet failed = new ATNConfigSet(configs.fullCtx); + for (ATNConfig c in configs) { + if (c.semanticContext != SemanticContext.NONE) { + bool predicateEvaluationResult = evalSemanticContextOne( + c.semanticContext, outerContext, c.alt, configs.fullCtx); + if (predicateEvaluationResult) { + succeeded.add(c); + } else { + failed.add(c); + } + } else { + succeeded.add(c); + } + } + return new Pair(succeeded, failed); + } + + /** Look through a list of predicate/alt pairs, returning alts for the + * pairs that win. A [NONE] predicate indicates an alt containing an + * unpredicated config which behaves as "always true." If !complete + * then we stop at the first predicate that evaluates to true. This + * includes pairs with null predicates. + */ + BitSet evalSemanticContext(List predPredictions, + ParserRuleContext outerContext, bool complete) { + BitSet predictions = new BitSet(); + for (PredPrediction pair in predPredictions) { + if (pair.pred == SemanticContext.NONE) { + predictions.set(pair.alt); + if (!complete) { + break; + } + continue; + } + + bool fullCtx = false; // in dfa + bool predicateEvaluationResult = + evalSemanticContextOne(pair.pred, outerContext, pair.alt, fullCtx); + if (debug || dfa_debug) { + log("eval pred $pair=$predicateEvaluationResult"); + } + + if (predicateEvaluationResult) { + if (debug || dfa_debug) log("PREDICT " + pair.alt); + predictions.set(pair.alt); + if (!complete) { + break; + } + } + } + + return predictions; + } + + /** + * Evaluate a semantic context within a specific parser context. + * + *

+ * This method might not be called for every semantic context evaluated + * during the prediction process. In particular, we currently do not + * evaluate the following but it may change in the future:

+ * + *
    + *
  • Precedence predicates (represented by + * {@link SemanticContext.PrecedencePredicate}) are not currently evaluated + * through this method.
    • + *
  • Operator predicates (represented by {@link SemanticContext.AND} and + * {@link SemanticContext.OR}) are evaluated as a single semantic + * context, rather than evaluating the operands individually. + * Implementations which require evaluation results from individual + * predicates should override this method to explicitly handle evaluation of + * the operands within operator predicates.
    • + *
+ * + * @param pred The semantic context to evaluate + * @param parserCallStack The parser context in which to evaluate the + * semantic context + * @param alt The alternative which is guarded by [pred] + * @param fullCtx [true] if the evaluation is occurring during LL + * prediction; otherwise, [false] if the evaluation is occurring + * during SLL prediction + * + * @since 4.3 + */ + bool evalSemanticContextOne(SemanticContext pred, + ParserRuleContext parserCallStack, int alt, bool fullCtx) { + return pred.eval(parser, parserCallStack); + } + + /* TODO: If we are doing predicates, there is no point in pursuing + closure operations if we reach a DFA state that uniquely predicts + alternative. We will not be caching that DFA state and it is a + waste to pursue the closure. Might have to advance when we do + ambig detection thought :( + */ + + void closure( + ATNConfig config, + ATNConfigSet configs, + Set closureBusy, + bool collectPredicates, + bool fullCtx, + bool treatEofAsEpsilon) { + final int initialDepth = 0; + closureCheckingStopState(config, configs, closureBusy, collectPredicates, + fullCtx, initialDepth, treatEofAsEpsilon); + assert(!fullCtx || !configs.dipsIntoOuterContext); + } + + void closureCheckingStopState( + ATNConfig config, + ATNConfigSet configs, + Set closureBusy, + bool collectPredicates, + bool fullCtx, + int depth, + bool treatEofAsEpsilon) { + if (debug) log("closure(" + config.toString(parser, true) + ")"); + + if (config.state is RuleStopState) { + // We hit rule end. If we have context info, use it + // run thru all possible stack tops in ctx + if (!config.context.isEmpty) { + for (int i = 0; i < config.context.length; i++) { + if (config.context.getReturnState(i) == + PredictionContext.EMPTY_RETURN_STATE) { + if (fullCtx) { + configs.add( + new ATNConfig.dup(config, + state: config.state, context: PredictionContext.EMPTY), + mergeCache); + continue; + } else { + // we have no context info, just chase follow links (if greedy) + if (debug) + log("FALLING off rule " + getRuleName(config.state.ruleIndex)); + closure_(config, configs, closureBusy, collectPredicates, fullCtx, + depth, treatEofAsEpsilon); + } + continue; + } + ATNState returnState = atn.states[config.context.getReturnState(i)]; + PredictionContext newContext = + config.context.getParent(i); // "pop" return state + ATNConfig c = new ATNConfig( + returnState, config.alt, newContext, config.semanticContext); + // While we have context to pop back from, we may have + // gotten that context AFTER having falling off a rule. + // Make sure we track that we are now out of context. + // + // This assignment also propagates the + // isPrecedenceFilterSuppressed() value to the new + // configuration. + c.reachesIntoOuterContext = config.reachesIntoOuterContext; +// assert(depth > int.MIN_VALUE); + closureCheckingStopState(c, configs, closureBusy, collectPredicates, + fullCtx, depth - 1, treatEofAsEpsilon); + } + return; + } else if (fullCtx) { + // reached end of start rule + configs.add(config, mergeCache); + return; + } else { + // else if we have no context info, just chase follow links (if greedy) + if (debug) + log("FALLING off rule " + getRuleName(config.state.ruleIndex)); + } + } + + closure_(config, configs, closureBusy, collectPredicates, fullCtx, depth, + treatEofAsEpsilon); + } + + /** Do the actual work of walking epsilon edges */ + void closure_( + ATNConfig config, + ATNConfigSet configs, + Set closureBusy, + bool collectPredicates, + bool fullCtx, + int depth, + bool treatEofAsEpsilon) { + ATNState p = config.state; + // optimization + if (!p.onlyHasEpsilonTransitions()) { + configs.add(config, mergeCache); + // make sure to not return here, because EOF transitions can act as + // both epsilon transitions and non-epsilon transitions. +// if ( debug ) log("added config "+configs); + } + + for (int i = 0; i < p.numberOfTransitions; i++) { + if (i == 0 && canDropLoopEntryEdgeInLeftRecursiveRule(config)) continue; + + Transition t = p.transition(i); + bool continueCollecting = !(t is ActionTransition) && collectPredicates; + ATNConfig c = getEpsilonTarget(config, t, continueCollecting, depth == 0, + fullCtx, treatEofAsEpsilon); + if (c != null) { + int newDepth = depth; + if (config.state is RuleStopState) { + assert(!fullCtx); + // target fell off end of rule; mark resulting c as having dipped into outer context + // We can't get here if incoming config was rule stop and we had context + // track how far we dip into outer context. Might + // come in handy and we avoid evaluating context dependent + // preds if this is > 0. + + if (_dfa != null && _dfa.isPrecedenceDfa()) { + int outermostPrecedenceReturn = + (t as EpsilonTransition).outermostPrecedenceReturn; + if (outermostPrecedenceReturn == _dfa.atnStartState.ruleIndex) { + c.setPrecedenceFilterSuppressed(true); + } + } + + c.reachesIntoOuterContext++; + + if (!closureBusy.add(c)) { + // avoid infinite recursion for right-recursive rules + continue; + } + + // TODO: can remove? only care when we add to set per middle of this method + configs.dipsIntoOuterContext = true; +// assert(newDepth > int.MIN_VALUE); + newDepth--; + if (debug) log("dips into outer ctx: $c"); + } else { + if (!t.isEpsilon && !closureBusy.add(c)) { + // avoid infinite recursion for EOF* and EOF+ + continue; + } + + if (t is RuleTransition) { + // latch when newDepth goes negative - once we step out of the entry context we can't return + if (newDepth >= 0) { + newDepth++; + } + } + } + + closureCheckingStopState(c, configs, closureBusy, continueCollecting, + fullCtx, newDepth, treatEofAsEpsilon); + } + } + } + + /** Implements first-edge (loop entry) elimination as an optimization + * during closure operations. See antlr/antlr4#1398. + * + * The optimization is to avoid adding the loop entry config when + * the exit path can only lead back to the same + * StarLoopEntryState after popping context at the rule end state + * (traversing only epsilon edges, so we're still in closure, in + * this same rule). + * + * We need to detect any state that can reach loop entry on + * epsilon w/o exiting rule. We don't have to look at FOLLOW + * links, just ensure that all stack tops for config refer to key + * states in LR rule. + * + * To verify we are in the right situation we must first check + * closure is at a StarLoopEntryState generated during LR removal. + * Then we check that each stack top of context is a return state + * from one of these cases: + * + * 1. 'not' expr, '(' type ')' expr. The return state points at loop entry state + * 2. expr op expr. The return state is the block end of internal block of (...)* + * 3. 'between' expr 'and' expr. The return state of 2nd expr reference. + * That state points at block end of internal block of (...)*. + * 4. expr '?' expr ':' expr. The return state points at block end, + * which points at loop entry state. + * + * If any is true for each stack top, then closure does not add a + * config to the current config set for edge[0], the loop entry branch. + * + * Conditions fail if any context for the current config is: + * + * a. empty (we'd fall out of expr to do a global FOLLOW which could + * even be to some weird spot in expr) or, + * b. lies outside of expr or, + * c. lies within expr but at a state not the BlockEndState + * generated during LR removal + * + * Do we need to evaluate predicates ever in closure for this case? + * + * No. Predicates, including precedence predicates, are only + * evaluated when computing a DFA start state. I.e., only before + * the lookahead (but not parser) consumes a token. + * + * There are no epsilon edges allowed in LR rule alt blocks or in + * the "primary" part (ID here). If closure is in + * StarLoopEntryState any lookahead operation will have consumed a + * token as there are no epsilon-paths that lead to + * StarLoopEntryState. We do not have to evaluate predicates + * therefore if we are in the generated StarLoopEntryState of a LR + * rule. Note that when making a prediction starting at that + * decision point, decision d=2, compute-start-state performs + * closure starting at edges[0], edges[1] emanating from + * StarLoopEntryState. That means it is not performing closure on + * StarLoopEntryState during compute-start-state. + * + * How do we know this always gives same prediction answer? + * + * Without predicates, loop entry and exit paths are ambiguous + * upon remaining input +b (in, say, a+b). Either paths lead to + * valid parses. Closure can lead to consuming + immediately or by + * falling out of this call to expr back into expr and loop back + * again to StarLoopEntryState to match +b. In this special case, + * we choose the more efficient path, which is to take the bypass + * path. + * + * The lookahead language has not changed because closure chooses + * one path over the other. Both paths lead to consuming the same + * remaining input during a lookahead operation. If the next token + * is an operator, lookahead will enter the choice block with + * operators. If it is not, lookahead will exit expr. Same as if + * closure had chosen to enter the choice block immediately. + * + * Closure is examining one config (some loopentrystate, some alt, + * context) which means it is considering exactly one alt. Closure + * always copies the same alt to any derived configs. + * + * How do we know this optimization doesn't mess up precedence in + * our parse trees? + * + * Looking through expr from left edge of stat only has to confirm + * that an input, say, a+b+c; begins with any valid interpretation + * of an expression. The precedence actually doesn't matter when + * making a decision in stat seeing through expr. It is only when + * parsing rule expr that we must use the precedence to get the + * right interpretation and, hence, parse tree. + * + * @since 4.6 + */ + bool canDropLoopEntryEdgeInLeftRecursiveRule(ATNConfig config) { + if (TURN_OFF_LR_LOOP_ENTRY_BRANCH_OPT) return false; + ATNState p = config.state; + // First check to see if we are in StarLoopEntryState generated during + // left-recursion elimination. For efficiency, also check if + // the context has an empty stack case. If so, it would mean + // global FOLLOW so we can't perform optimization + if (p.stateType != StateType.STAR_LOOP_ENTRY || + !(p as StarLoopEntryState) + .isPrecedenceDecision || // Are we the special loop entry/exit state? + config.context.isEmpty || // If SLL wildcard + config.context.hasEmptyPath()) { + return false; + } + + // Require all return states to return back to the same rule + // that p is in. + int numCtxs = config.context.length; + for (int i = 0; i < numCtxs; i++) { + // for each stack context + ATNState returnState = atn.states[config.context.getReturnState(i)]; + if (returnState.ruleIndex != p.ruleIndex) return false; + } + + BlockStartState decisionStartState = p.transition(0).target; + int blockEndStateNum = decisionStartState.endState.stateNumber; + BlockEndState blockEndState = atn.states[blockEndStateNum]; + + // Verify that the top of each stack context leads to loop entry/exit + // state through epsilon edges and w/o leaving rule. + for (int i = 0; i < numCtxs; i++) { + // for each stack context + int returnStateNumber = config.context.getReturnState(i); + ATNState returnState = atn.states[returnStateNumber]; + // all states must have single outgoing epsilon edge + if (returnState.numberOfTransitions != 1 || + !returnState.transition(0).isEpsilon) { + return false; + } + // Look for prefix op case like 'not expr', (' type ')' expr + ATNState returnStateTarget = returnState.transition(0).target; + if (returnState.stateType == StateType.BLOCK_END && + returnStateTarget == p) { + continue; + } + // Look for 'expr op expr' or case where expr's return state is block end + // of (...)* internal block; the block end points to loop back + // which points to p but we don't need to check that + if (returnState == blockEndState) { + continue; + } + // Look for ternary expr ? expr : expr. The return state points at block end, + // which points at loop entry state + if (returnStateTarget == blockEndState) { + continue; + } + // Look for complex prefix 'between expr and expr' case where 2nd expr's + // return state points at block end state of (...)* internal block + if (returnStateTarget.stateType == StateType.BLOCK_END && + returnStateTarget.numberOfTransitions == 1 && + returnStateTarget.transition(0).isEpsilon && + returnStateTarget.transition(0).target == p) { + continue; + } + + // anything else ain't conforming + return false; + } + + return true; + } + + String getRuleName(int index) { + if (parser != null && index >= 0) return parser.ruleNames[index]; + return ""; + } + + ATNConfig getEpsilonTarget( + ATNConfig config, + Transition t, + bool collectPredicates, + bool inContext, + bool fullCtx, + bool treatEofAsEpsilon) { + switch (t.type) { + case TransitionType.RULE: + return ruleTransition(config, t); + + case TransitionType.PRECEDENCE: + return precedenceTransition( + config, t, collectPredicates, inContext, fullCtx); + + case TransitionType.PREDICATE: + return predTransition(config, t, collectPredicates, inContext, fullCtx); + case TransitionType.ACTION: + return actionTransition(config, t); + + case TransitionType.EPSILON: + return new ATNConfig.dup(config, state: t.target); + + case TransitionType.ATOM: + case TransitionType.RANGE: + case TransitionType.SET: + // EOF transitions act like epsilon transitions after the first EOF + // transition is traversed + if (treatEofAsEpsilon) { + if (t.matches(Token.EOF, 0, 1)) { + return new ATNConfig.dup(config, state: t.target); + } + } + + return null; + + default: + return null; + } + } + + ATNConfig actionTransition(ATNConfig config, ActionTransition t) { + if (debug) log("ACTION edge ${t.ruleIndex}:${t.actionIndex}"); + return new ATNConfig.dup(config, state: t.target); + } + + ATNConfig precedenceTransition( + ATNConfig config, + PrecedencePredicateTransition pt, + bool collectPredicates, + bool inContext, + bool fullCtx) { + if (debug) { + log("PRED (collectPredicates=$collectPredicates) ${pt.precedence}>=_p, ctx dependent=true"); + if (parser != null) { + log("context surrounding pred is ${parser.getRuleInvocationStack()}"); + } + } + + ATNConfig c = null; + if (collectPredicates && inContext) { + if (fullCtx) { + // In full context mode, we can evaluate predicates on-the-fly + // during closure, which dramatically reduces the size of + // the config sets. It also obviates the need to test predicates + // later during conflict resolution. + int currentPosition = input.index; + input.seek(startIndex); + bool predSucceeds = evalSemanticContextOne( + pt.predicate, _outerContext, config.alt, fullCtx); + input.seek(currentPosition); + if (predSucceeds) { + c = new ATNConfig.dup(config, state: pt.target); // no pred context + } + } else { + SemanticContext newSemCtx = + SemanticContext.and(config.semanticContext, pt.predicate); + c = new ATNConfig.dup(config, + state: pt.target, semanticContext: newSemCtx); + } + } else { + c = new ATNConfig.dup(config, state: pt.target); + } + + if (debug) log("config from pred transition=$c"); + return c; + } + + ATNConfig predTransition(ATNConfig config, PredicateTransition pt, + bool collectPredicates, bool inContext, bool fullCtx) { + if (debug) { + log("PRED (collectPredicates=$collectPredicates) " + + "${pt.ruleIndex}:${pt.predIndex}" + + ", ctx dependent=${pt.isCtxDependent}"); + if (parser != null) { + log("context surrounding pred is ${parser.getRuleInvocationStack()}"); + } + } + + ATNConfig c = null; + if (collectPredicates && + (!pt.isCtxDependent || (pt.isCtxDependent && inContext))) { + if (fullCtx) { + // In full context mode, we can evaluate predicates on-the-fly + // during closure, which dramatically reduces the size of + // the config sets. It also obviates the need to test predicates + // later during conflict resolution. + int currentPosition = input.index; + input.seek(startIndex); + bool predSucceeds = evalSemanticContextOne( + pt.predicate, _outerContext, config.alt, fullCtx); + input.seek(currentPosition); + if (predSucceeds) { + c = new ATNConfig.dup(config, state: pt.target); // no pred context + } + } else { + SemanticContext newSemCtx = + SemanticContext.and(config.semanticContext, pt.predicate); + c = new ATNConfig.dup(config, + state: pt.target, semanticContext: newSemCtx); + } + } else { + c = new ATNConfig.dup(config, state: pt.target); + } + + if (debug) log("config from pred transition=$c"); + return c; + } + + ATNConfig ruleTransition(ATNConfig config, RuleTransition t) { + if (debug) { + log("CALL rule " + + getRuleName(t.target.ruleIndex) + + ", ctx=${config.context}"); + } + + ATNState returnState = t.followState; + PredictionContext newContext = SingletonPredictionContext.create( + config.context, returnState.stateNumber); + return new ATNConfig.dup(config, state: t.target, context: newContext); + } + + /** + * Gets a [BitSet] containing the alternatives in [configs] + * which are part of one or more conflicting alternative subsets. + * + * @param configs The [ATNConfigSet] to analyze. + * @return The alternatives in [configs] which are part of one or more + * conflicting alternative subsets. If [configs] does not contain any + * conflicting subsets, this method returns an empty [BitSet]. + */ + BitSet getConflictingAlts(ATNConfigSet configs) { + List altsets = + PredictionModeExtension.getConflictingAltSubsets(configs); + return PredictionModeExtension.getAlts(altsets); + } + + /** + Sam pointed out a problem with the previous definition, v3, of + ambiguous states. If we have another state associated with conflicting + alternatives, we should keep going. For example, the following grammar + + s : (ID | ID ID?) ';' ; + + When the ATN simulation reaches the state before ';', it has a DFA + state that looks like: [12|1|[], 6|2|[], 12|2|[]]. Naturally + 12|1|[] and 12|2|[] conflict, but we cannot stop processing this node + because alternative to has another way to continue, via [6|2|[]]. + The key is that we have a single state that has config's only associated + with a single alternative, 2, and crucially the state transitions + among the configurations are all non-epsilon transitions. That means + we don't consider any conflicts that include alternative 2. So, we + ignore the conflict between alts 1 and 2. We ignore a set of + conflicting alts when there is an intersection with an alternative + associated with a single alt state in the state→config-list map. + + It's also the case that we might have two conflicting configurations but + also a 3rd nonconflicting configuration for a different alternative: + [1|1|[], 1|2|[], 8|3|[]]. This can come about from grammar: + + a : A | A | A B ; + + After matching input A, we reach the stop state for rule A, state 1. + State 8 is the state right before B. Clearly alternatives 1 and 2 + conflict and no amount of further lookahead will separate the two. + However, alternative 3 will be able to continue and so we do not + stop working on this state. In the previous example, we're concerned + with states associated with the conflicting alternatives. Here alt + 3 is not associated with the conflicting configs, but since we can continue + looking for input reasonably, I don't declare the state done. We + ignore a set of conflicting alts when we have an alternative + that we still need to pursue. + */ + BitSet getConflictingAltsOrUniqueAlt(ATNConfigSet configs) { + BitSet conflictingAlts; + if (configs.uniqueAlt != ATN.INVALID_ALT_NUMBER) { + conflictingAlts = new BitSet(); + conflictingAlts.set(configs.uniqueAlt); + } else { + conflictingAlts = configs.conflictingAlts; + } + return conflictingAlts; + } + + String getTokenName(int t) { + if (t == Token.EOF) { + return "EOF"; + } + + Vocabulary vocabulary = parser != null + ? parser.vocabulary + : VocabularyImpl.EMPTY_VOCABULARY; + String displayName = vocabulary.getDisplayName(t); + if (displayName == t.toString()) { + return displayName; + } + + return displayName + "<$t>"; + } + + String getLookaheadName(TokenStream input) { + return getTokenName(input.LA(1)); + } + + /** Used for debugging in adaptivePredict around execATN but I cut + * it out for clarity now that alg. works well. We can leave this + * "dead" code for a bit. + */ + void dumpDeadEndConfigs(NoViableAltException nvae) { + log("dead end configs: ", level: Level.SEVERE.value); + for (ATNConfig c in nvae.deadEndConfigs) { + String trans = "no edges"; + if (c.state.numberOfTransitions > 0) { + Transition t = c.state.transition(0); + if (t is AtomTransition) { + AtomTransition at = t; + trans = "Atom " + getTokenName(at.atomLabel); + } else if (t is SetTransition) { + SetTransition st = t; + bool not = st is NotSetTransition; + trans = (not ? "~" : "") + "Set " + st.label.toString(); + } + } + log(c.toString(parser, true) + ":" + trans, level: Level.SEVERE.value); + } + } + + NoViableAltException noViableAlt(TokenStream input, + ParserRuleContext outerContext, ATNConfigSet configs, int startIndex) { + return new NoViableAltException(parser, input, input.get(startIndex), + input.LT(1), configs, outerContext); + } + + static int getUniqueAlt(ATNConfigSet configs) { + int alt = ATN.INVALID_ALT_NUMBER; + for (ATNConfig c in configs) { + if (alt == ATN.INVALID_ALT_NUMBER) { + alt = c.alt; // found first alt + } else if (c.alt != alt) { + return ATN.INVALID_ALT_NUMBER; + } + } + return alt; + } + + /** + * Add an edge to the DFA, if possible. This method calls + * {@link #addDFAState} to ensure the [to] state is present in the + * DFA. If [from] is null, or if [t] is outside the + * range of edges that can be represented in the DFA tables, this method + * returns without adding the edge to the DFA. + * + *

If [to] is null, this method returns null. + * Otherwise, this method returns the [DFAState] returned by calling + * {@link #addDFAState} for the [to] state.

+ * + * @param dfa The DFA + * @param from The source state for the edge + * @param t The input symbol + * @param to The target state for the edge + * + * @return If [to] is null, this method returns null; + * otherwise this method returns the result of calling {@link #addDFAState} + * on [to] + */ + DFAState addDFAEdge(DFA dfa, DFAState from, int t, DFAState to) { + if (debug) { + log("EDGE $from -> $to upon " + getTokenName(t)); + } + + if (to == null) { + return null; + } + + to = addDFAState(dfa, to); // used existing if possible not incoming + if (from == null || t < -1 || t > atn.maxTokenType) { + return to; + } + + if (from.edges == null) { + from.edges = List(atn.maxTokenType + 1 + 1); + } + + from.edges[t + 1] = to; // connect + + if (debug) { + log("DFA=\n" + + dfa.toString(parser != null + ? parser.vocabulary + : VocabularyImpl.EMPTY_VOCABULARY)); + } + + return to; + } + + /** + * Add state [D] to the DFA if it is not already present, and return + * the actual instance stored in the DFA. If a state equivalent to [D] + * is already in the DFA, the existing state is returned. Otherwise this + * method returns [D] after adding it to the DFA. + * + *

If [D] is {@link #ERROR}, this method returns {@link #ERROR} and + * does not change the DFA.

+ * + * @param dfa The dfa + * @param D The DFA state to add + * @return The state stored in the DFA. This will be either the existing + * state if [D] is already in the DFA, or [D] itself if the + * state was not already present. + */ + DFAState addDFAState(DFA dfa, DFAState D) { + if (D == ATNSimulator.ERROR) { + return D; + } + + DFAState existing = dfa.states[D]; + if (existing != null) return existing; + + D.stateNumber = dfa.states.length; + if (!D.configs.readOnly) { + D.configs.optimizeConfigs(this); + D.configs.readOnly = true; + } + dfa.states[D] = D; + if (debug) log("adding new DFA state: $D"); + return D; + } + + void reportAttemptingFullContext(DFA dfa, BitSet conflictingAlts, + ATNConfigSet configs, int startIndex, int stopIndex) { + if (debug || retry_debug) { + Interval interval = Interval.of(startIndex, stopIndex); + log("reportAttemptingFullContext decision=${dfa.decision}:$configs" + + ", input=" + + parser.tokenStream.getText(interval)); + } + if (parser != null) + parser.errorListenerDispatch.reportAttemptingFullContext( + parser, dfa, startIndex, stopIndex, conflictingAlts, configs); + } + + void reportContextSensitivity(DFA dfa, int prediction, ATNConfigSet configs, + int startIndex, int stopIndex) { + if (debug || retry_debug) { + Interval interval = Interval.of(startIndex, stopIndex); + log("reportContextSensitivity decision=${dfa.decision}:$configs" + + ", input=" + + parser.tokenStream.getText(interval)); + } + if (parser != null) + parser.errorListenerDispatch.reportContextSensitivity( + parser, dfa, startIndex, stopIndex, prediction, configs); + } + + /** If context sensitive parsing, we know it's ambiguity not conflict */ + void reportAmbiguity( + DFA dfa, + DFAState D, // the DFA state from execATN() that had SLL conflicts + int startIndex, + int stopIndex, + bool exact, + BitSet ambigAlts, + ATNConfigSet configs) // configs that LL not SLL considered conflicting + { + if (debug || retry_debug) { + Interval interval = Interval.of(startIndex, stopIndex); + log("reportAmbiguity $ambigAlts:$configs" + + ", input=" + + parser.tokenStream.getText(interval)); + } + if (parser != null) + parser.errorListenerDispatch.reportAmbiguity( + parser, dfa, startIndex, stopIndex, exact, ambigAlts, configs); + } +} + +/** + * This enumeration defines the prediction modes available in ANTLR 4 along with + * utility methods for analyzing configuration sets for conflicts and/or + * ambiguities. + */ +enum PredictionMode { + /** + * The SLL(*) prediction mode. This prediction mode ignores the current + * parser context when making predictions. This is the fastest prediction + * mode, and provides correct results for many grammars. This prediction + * mode is more powerful than the prediction mode provided by ANTLR 3, but + * may result in syntax errors for grammar and input combinations which are + * not SLL. + * + *

+ * When using this prediction mode, the parser will either return a correct + * parse tree (i.e. the same parse tree that would be returned with the + * {@link #LL} prediction mode), or it will report a syntax error. If a + * syntax error is encountered when using the {@link #SLL} prediction mode, + * it may be due to either an actual syntax error in the input or indicate + * that the particular combination of grammar and input requires the more + * powerful {@link #LL} prediction abilities to complete successfully.

+ * + *

+ * This prediction mode does not provide any guarantees for prediction + * behavior for syntactically-incorrect inputs.

+ */ + SLL, + /** + * The LL(*) prediction mode. This prediction mode allows the current parser + * context to be used for resolving SLL conflicts that occur during + * prediction. This is the fastest prediction mode that guarantees correct + * parse results for all combinations of grammars with syntactically correct + * inputs. + * + *

+ * When using this prediction mode, the parser will make correct decisions + * for all syntactically-correct grammar and input combinations. However, in + * cases where the grammar is truly ambiguous this prediction mode might not + * report a precise answer for exactly which alternatives are + * ambiguous.

+ * + *

+ * This prediction mode does not provide any guarantees for prediction + * behavior for syntactically-incorrect inputs.

+ */ + LL, + /** + * The LL(*) prediction mode with exact ambiguity detection. In addition to + * the correctness guarantees provided by the {@link #LL} prediction mode, + * this prediction mode instructs the prediction algorithm to determine the + * complete and exact set of ambiguous alternatives for every ambiguous + * decision encountered while parsing. + * + *

+ * This prediction mode may be used for diagnosing ambiguities during + * grammar development. Due to the performance overhead of calculating sets + * of ambiguous alternatives, this prediction mode should be avoided when + * the exact results are not necessary.

+ * + *

+ * This prediction mode does not provide any guarantees for prediction + * behavior for syntactically-incorrect inputs.

+ */ + LL_EXACT_AMBIG_DETECTION, +} + +extension PredictionModeExtension on PredictionMode { + /** + * Computes the SLL prediction termination condition. + * + *

+ * This method computes the SLL prediction termination condition for both of + * the following cases.

+ * + *
    + *
  • The usual SLL+LL fallback upon SLL conflict
    • + *
  • Pure SLL without LL fallback
    • + *
+ * + *

COMBINED SLL+LL PARSING

+ * + *

When LL-fallback is enabled upon SLL conflict, correct predictions are + * ensured regardless of how the termination condition is computed by this + * method. Due to the substantially higher cost of LL prediction, the + * prediction should only fall back to LL when the additional lookahead + * cannot lead to a unique SLL prediction.

+ * + *

Assuming combined SLL+LL parsing, an SLL configuration set with only + * conflicting subsets should fall back to full LL, even if the + * configuration sets don't resolve to the same alternative (e.g. + * {@code {1,2}} and {@code {3,4}}. If there is at least one non-conflicting + * configuration, SLL could continue with the hopes that more lookahead will + * resolve via one of those non-conflicting configurations.

+ * + *

Here's the prediction termination rule them: SLL (for SLL+LL parsing) + * stops when it sees only conflicting configuration subsets. In contrast, + * full LL keeps going when there is uncertainty.

+ * + *

HEURISTIC

+ * + *

As a heuristic, we stop prediction when we see any conflicting subset + * unless we see a state that only has one alternative associated with it. + * The single-alt-state thing lets prediction continue upon rules like + * (otherwise, it would admit defeat too soon):

+ * + *

{@code [12|1|[], 6|2|[], 12|2|[]]. s : (ID | ID ID?) ';' ;}

+ * + *

When the ATN simulation reaches the state before {@code ';'}, it has a + * DFA state that looks like: {@code [12|1|[], 6|2|[], 12|2|[]]}. Naturally + * {@code 12|1|[]} and {@code 12|2|[]} conflict, but we cannot stop + * processing this node because alternative to has another way to continue, + * via {@code [6|2|[]]}.

+ * + *

It also let's us continue for this rule:

+ * + *

{@code [1|1|[], 1|2|[], 8|3|[]] a : A | A | A B ;}

+ * + *

After matching input A, we reach the stop state for rule A, state 1. + * State 8 is the state right before B. Clearly alternatives 1 and 2 + * conflict and no amount of further lookahead will separate the two. + * However, alternative 3 will be able to continue and so we do not stop + * working on this state. In the previous example, we're concerned with + * states associated with the conflicting alternatives. Here alt 3 is not + * associated with the conflicting configs, but since we can continue + * looking for input reasonably, don't declare the state done.

+ * + *

PURE SLL PARSING

+ * + *

To handle pure SLL parsing, all we have to do is make sure that we + * combine stack contexts for configurations that differ only by semantic + * predicate. From there, we can do the usual SLL termination heuristic.

+ * + *

PREDICATES IN SLL+LL PARSING

+ * + *

SLL decisions don't evaluate predicates until after they reach DFA stop + * states because they need to create the DFA cache that works in all + * semantic situations. In contrast, full LL evaluates predicates collected + * during start state computation so it can ignore predicates thereafter. + * This means that SLL termination detection can totally ignore semantic + * predicates.

+ * + *

Implementation-wise, [ATNConfigSet] combines stack contexts but not + * semantic predicate contexts so we might see two configurations like the + * following.

+ * + *

{@code (s, 1, x, {}), (s, 1, x', {p})}

+ * + *

Before testing these configurations against others, we have to merge + * [x] and {@code x'} (without modifying the existing configurations). + * For example, we test {@code (x+x')==x''} when looking for conflicts in + * the following configurations.

+ * + *

{@code (s, 1, x, {}), (s, 1, x', {p}), (s, 2, x'', {})}

+ * + *

If the configuration set has predicates (as indicated by + * {@link ATNConfigSet#hasSemanticContext}), this algorithm makes a copy of + * the configurations to strip out all of the predicates so that a standard + * [ATNConfigSet] will merge everything ignoring predicates.

+ */ + static bool hasSLLConflictTerminatingPrediction( + PredictionMode mode, ATNConfigSet configs) { +/* Configs in rule stop states indicate reaching the end of the decision + * rule (local context) or end of start rule (full context). If all + * configs meet this condition, then none of the configurations is able + * to match additional input so we terminate prediction. + */ + if (allConfigsInRuleStopStates(configs)) { + return true; + } + +// pure SLL mode parsing + if (mode == PredictionMode.SLL) { +// Don't bother with combining configs from different semantic +// contexts if we can fail over to full LL; costs more time +// since we'll often fail over anyway. + if (configs.hasSemanticContext) { +// dup configs, tossing out semantic predicates + ATNConfigSet dup = new ATNConfigSet(); + for (ATNConfig c in configs) { + c = new ATNConfig.dup(c, semanticContext: SemanticContext.NONE); + dup.add(c); + } + configs = dup; + } +// now we have combined contexts for configs with dissimilar preds + } + +// pure SLL or combined SLL+LL mode parsing + + List altsets = getConflictingAltSubsets(configs); + bool heuristic = + hasConflictingAltSet(altsets) && !hasStateAssociatedWithOneAlt(configs); + return heuristic; + } + + /** + * Checks if any configuration in [configs] is in a + * [RuleStopState]. Configurations meeting this condition have reached + * the end of the decision rule (local context) or end of start rule (full + * context). + * + * @param configs the configuration set to test + * @return [true] if any configuration in [configs] is in a + * [RuleStopState], otherwise [false] + */ + static bool hasConfigInRuleStopState(ATNConfigSet configs) { + for (ATNConfig c in configs) { + if (c.state is RuleStopState) { + return true; + } + } + + return false; + } + + /** + * Checks if all configurations in [configs] are in a + * [RuleStopState]. Configurations meeting this condition have reached + * the end of the decision rule (local context) or end of start rule (full + * context). + * + * @param configs the configuration set to test + * @return [true] if all configurations in [configs] are in a + * [RuleStopState], otherwise [false] + */ + static bool allConfigsInRuleStopStates(ATNConfigSet configs) { + for (ATNConfig config in configs) { + if (!(config.state is RuleStopState)) { + return false; + } + } + + return true; + } + + /** + * Full LL prediction termination. + * + *

Can we stop looking ahead during ATN simulation or is there some + * uncertainty as to which alternative we will ultimately pick, after + * consuming more input? Even if there are partial conflicts, we might know + * that everything is going to resolve to the same minimum alternative. That + * means we can stop since no more lookahead will change that fact. On the + * other hand, there might be multiple conflicts that resolve to different + * minimums. That means we need more look ahead to decide which of those + * alternatives we should predict.

+ * + *

The basic idea is to split the set of configurations [C], into + * conflicting subsets {@code (s, _, ctx, _)} and singleton subsets with + * non-conflicting configurations. Two configurations conflict if they have + * identical {@link ATNConfig#state} and {@link ATNConfig#context} values + * but different {@link ATNConfig#alt} value, e.g. {@code (s, i, ctx, _)} + * and {@code (s, j, ctx, _)} for {@code i!=j}.

+ * + *

Reduce these configuration subsets to the set of possible alternatives. + * You can compute the alternative subsets in one pass as follows:

+ * + *

{@code A_s,ctx = {i | (s, i, ctx, _)}} for each configuration in + * [C] holding [s] and [ctx] fixed.

+ * + *

Or in pseudo-code, for each configuration [c] in [C]:

+ * + * 
+   * map[c] U= c.{@link ATNConfig#alt alt} # map hash/equals uses s and x, not
+   * alt and not pred
+   *
+ * + *

The values in [map] are the set of {@code A_s,ctx} sets.

+ * + *

If {@code |A_s,ctx|=1} then there is no conflict associated with + * [s] and [ctx].

+ * + *

Reduce the subsets to singletons by choosing a minimum of each subset. If + * the union of these alternative subsets is a singleton, then no amount of + * more lookahead will help us. We will always pick that alternative. If, + * however, there is more than one alternative, then we are uncertain which + * alternative to predict and must continue looking for resolution. We may + * or may not discover an ambiguity in the future, even if there are no + * conflicting subsets this round.

+ * + *

The biggest sin is to terminate early because it means we've made a + * decision but were uncertain as to the eventual outcome. We haven't used + * enough lookahead. On the other hand, announcing a conflict too late is no + * big deal; you will still have the conflict. It's just inefficient. It + * might even look until the end of file.

+ * + *

No special consideration for semantic predicates is required because + * predicates are evaluated on-the-fly for full LL prediction, ensuring that + * no configuration contains a semantic context during the termination + * check.

+ * + *

CONFLICTING CONFIGS

+ * + *

Two configurations {@code (s, i, x)} and {@code (s, j, x')}, conflict + * when {@code i!=j} but {@code x=x'}. Because we merge all + * {@code (s, i, _)} configurations together, that means that there are at + * most [n] configurations associated with state [s] for + * [n] possible alternatives in the decision. The merged stacks + * complicate the comparison of configuration contexts [x] and + * {@code x'}. Sam checks to see if one is a subset of the other by calling + * merge and checking to see if the merged result is either [x] or + * {@code x'}. If the [x] associated with lowest alternative [i] + * is the superset, then [i] is the only possible prediction since the + * others resolve to {@code min(i)} as well. However, if [x] is + * associated with {@code j>i} then at least one stack configuration for + * [j] is not in conflict with alternative [i]. The algorithm + * should keep going, looking for more lookahead due to the uncertainty.

+ * + *

For simplicity, I'm doing a equality check between [x] and + * {@code x'} that lets the algorithm continue to consume lookahead longer + * than necessary. The reason I like the equality is of course the + * simplicity but also because that is the test you need to detect the + * alternatives that are actually in conflict.

+ * + *

CONTINUE/STOP RULE

+ * + *

Continue if union of resolved alternative sets from non-conflicting and + * conflicting alternative subsets has more than one alternative. We are + * uncertain about which alternative to predict.

+ * + *

The complete set of alternatives, {@code [i for (_,i,_)]}, tells us which + * alternatives are still in the running for the amount of input we've + * consumed at this point. The conflicting sets let us to strip away + * configurations that won't lead to more states because we resolve + * conflicts to the configuration with a minimum alternate for the + * conflicting set.

+ * + *

CASES

+ * + *
    + * + *
  • no conflicts and more than 1 alternative in set => continue
    • + * + *
  • {@code (s, 1, x)}, {@code (s, 2, x)}, {@code (s, 3, z)}, + * {@code (s', 1, y)}, {@code (s', 2, y)} yields non-conflicting set + * {@code {3}} U conflicting sets {@code min({1,2})} U {@code min({1,2})} = + * {@code {1,3}} => continue + *
    • + * + *
  • {@code (s, 1, x)}, {@code (s, 2, x)}, {@code (s', 1, y)}, + * {@code (s', 2, y)}, {@code (s'', 1, z)} yields non-conflicting set + * {@code {1}} U conflicting sets {@code min({1,2})} U {@code min({1,2})} = + * {@code {1}} => stop and predict 1
    • + * + *
  • {@code (s, 1, x)}, {@code (s, 2, x)}, {@code (s', 1, y)}, + * {@code (s', 2, y)} yields conflicting, reduced sets {@code {1}} U + * {@code {1}} = {@code {1}} => stop and predict 1, can announce + * ambiguity {@code {1,2}}
    • + * + *
  • {@code (s, 1, x)}, {@code (s, 2, x)}, {@code (s', 2, y)}, + * {@code (s', 3, y)} yields conflicting, reduced sets {@code {1}} U + * {@code {2}} = {@code {1,2}} => continue
    • + * + *
  • {@code (s, 1, x)}, {@code (s, 2, x)}, {@code (s', 3, y)}, + * {@code (s', 4, y)} yields conflicting, reduced sets {@code {1}} U + * {@code {3}} = {@code {1,3}} => continue
    • + * + *
+ * + *

EXACT AMBIGUITY DETECTION

+ * + *

If all states report the same conflicting set of alternatives, then we + * know we have the exact ambiguity set.

+ * + *

|A_i|>1 and + * A_i = A_j for all i, j.

+ * + *

In other words, we continue examining lookahead until all {@code A_i} + * have more than one alternative and all {@code A_i} are the same. If + * {@code A={{1,2}, {1,3}}}, then regular LL prediction would terminate + * because the resolved set is {@code {1}}. To determine what the real + * ambiguity is, we have to know whether the ambiguity is between one and + * two or one and three so we keep going. We can only stop prediction when + * we need exact ambiguity detection when the sets look like + * {@code A={{1,2}}} or {@code {{1,2},{1,2}}}, etc...

+ */ + static int resolvesToJustOneViableAlt(List altsets) { + return getSingleViableAlt(altsets); + } + + /** + * Determines if every alternative subset in [altsets] contains more + * than one alternative. + * + * @param altsets a collection of alternative subsets + * @return [true] if every [BitSet] in [altsets] has + * {@link BitSet#cardinality cardinality} > 1, otherwise [false] + */ + static bool allSubsetsConflict(List altsets) { + return !hasNonConflictingAltSet(altsets); + } + + /** + * Determines if any single alternative subset in [altsets] contains + * exactly one alternative. + * + * @param altsets a collection of alternative subsets + * @return [true] if [altsets] contains a [BitSet] with + * {@link BitSet#cardinality cardinality} 1, otherwise [false] + */ + static bool hasNonConflictingAltSet(List altsets) { + for (BitSet alts in altsets) { + if (alts.cardinality == 1) { + return true; + } + } + return false; + } + + /** + * Determines if any single alternative subset in [altsets] contains + * more than one alternative. + * + * @param altsets a collection of alternative subsets + * @return [true] if [altsets] contains a [BitSet] with + * {@link BitSet#cardinality cardinality} > 1, otherwise [false] + */ + static bool hasConflictingAltSet(List altsets) { + for (BitSet alts in altsets) { + if (alts.cardinality > 1) { + return true; + } + } + return false; + } + + /** + * Determines if every alternative subset in [altsets] is equivalent. + * + * @param altsets a collection of alternative subsets + * @return [true] if every member of [altsets] is equal to the + * others, otherwise [false] + */ + static bool allSubsetsEqual(List altsets) { + final first = altsets.first; + return altsets.every((e) => e == first); + } + + /** + * Returns the unique alternative predicted by all alternative subsets in + * [altsets]. If no such alternative exists, this method returns + * {@link ATN#INVALID_ALT_NUMBER}. + * + * @param altsets a collection of alternative subsets + */ + static int getUniqueAlt(List altsets) { + BitSet all = getAlts(altsets); + if (all.cardinality == 1) return all.nextset(0); + return ATN.INVALID_ALT_NUMBER; + } + + /** + * Gets the complete set of represented alternatives for a collection of + * alternative subsets. This method returns the union of each [BitSet] + * in [altsets]. + * + * @param altsets a collection of alternative subsets + * @return the set of represented alternatives in [altsets] + */ + static BitSet getAlts(List altsets) { + BitSet all = new BitSet(); + for (BitSet alts in altsets) { + all.or(alts); + } + return all; + } + + /** + * Get union of all alts from configs. + * + * @since 4.5.1 + */ + static BitSet getAltsFromConfigs(ATNConfigSet configs) { + BitSet alts = new BitSet(); + for (ATNConfig config in configs) { + alts.set(config.alt); + } + return alts; + } + + /** + * This function gets the conflicting alt subsets from a configuration set. + * For each configuration [c] in [configs]: + * + * 
+   * map[c] U= c.{@link ATNConfig#alt alt} # map hash/equals uses s and x, not
+   * alt and not pred
+   *
+ */ + static List getConflictingAltSubsets(ATNConfigSet configs) { + final configToAlts = + new HashMap(equals: (ATNConfig a, ATNConfig b) { + if (identical(a, b)) return true; + if (a == null || b == null) return false; + return a.state.stateNumber == b.state.stateNumber && + a.context == b.context; + }, hashCode: (ATNConfig o) { + /** + * The hash code is only a function of the {@link ATNState#stateNumber} + * and {@link ATNConfig#context}. + */ + int hashCode = MurmurHash.initialize(7); + hashCode = MurmurHash.update(hashCode, o.state.stateNumber); + hashCode = MurmurHash.update(hashCode, o.context); + hashCode = MurmurHash.finish(hashCode, 2); + return hashCode; + }); + for (ATNConfig c in configs) { + BitSet alts = configToAlts[c]; + if (alts == null) { + alts = new BitSet(); + configToAlts[c] = alts; + } + alts.set(c.alt); + } + return configToAlts.values.toList(); + } + + /** + * Get a map from state to alt subset from a configuration set. For each + * configuration [c] in [configs]: + * + * 
+   * map[c.{@link ATNConfig#state state}] U= c.{@link ATNConfig#alt alt}
+   *
+ */ + static Map getStateToAltMap(ATNConfigSet configs) { + Map m = {}; + for (ATNConfig c in configs) { + BitSet alts = m[c.state]; + if (alts == null) { + alts = new BitSet(); + m[c.state] = alts; + } + alts.set(c.alt); + } + return m; + } + + static bool hasStateAssociatedWithOneAlt(ATNConfigSet configs) { + Map x = getStateToAltMap(configs); + for (BitSet alts in x.values) { + if (alts.cardinality == 1) return true; + } + return false; + } + + static int getSingleViableAlt(List altsets) { + BitSet viableAlts = new BitSet(); + for (BitSet alts in altsets) { + int minAlt = alts.nextset(0); + viableAlts.set(minAlt); + if (viableAlts.cardinality > 1) { + // more than 1 viable alt + return ATN.INVALID_ALT_NUMBER; + } + } + return viableAlts.nextset(0); + } +} diff --git a/runtime/Dart/lib/src/atn/src/profiling_atn_simulator.dart b/runtime/Dart/lib/src/atn/src/profiling_atn_simulator.dart new file mode 100644 index 000000000..06984d47f --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/profiling_atn_simulator.dart @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:math'; + +import '../../dfa/dfa.dart'; +import '../../parser.dart'; +import '../../parser_rule_context.dart'; +import '../../token_stream.dart'; +import '../../util/bit_set.dart'; +import 'atn_config_set.dart'; +import 'atn_simulator.dart'; +import 'info.dart'; +import 'parser_atn_simulator.dart'; +import 'semantic_context.dart'; + +class ProfilingATNSimulator extends ParserATNSimulator { + List decisions; + int numDecisions; + + int _sllStopIndex; + int _llStopIndex; + + int currentDecision; + DFAState currentState; + + /** At the point of LL failover, we record how SLL would resolve the conflict so that + * we can determine whether or not a decision / input pair is context-sensitive. + * If LL gives a different result than SLL's predicted alternative, we have a + * context sensitivity for sure. The converse is not necessarily true, however. + * It's possible that after conflict resolution chooses minimum alternatives, + * SLL could get the same answer as LL. Regardless of whether or not the result indicates + * an ambiguity, it is not treated as a context sensitivity because LL prediction + * was not required in order to produce a correct prediction for this decision and input sequence. + * It may in fact still be a context sensitivity but we don't know by looking at the + * minimum alternatives for the current input. + */ + int conflictingAltResolvedBySLL; + + ProfilingATNSimulator(Parser parser) + : super(parser, parser.interpreter.atn, parser.interpreter.decisionToDFA, + parser.interpreter.sharedContextCache) { + numDecisions = atn.decisionToState.length; + decisions = List(numDecisions); + for (int i = 0; i < numDecisions; i++) { + decisions[i] = new DecisionInfo(i); + } + } + + int adaptivePredict( + TokenStream input, int decision, ParserRuleContext outerContext) { + try { + this._sllStopIndex = -1; + this._llStopIndex = -1; + this.currentDecision = decision; + + final start = + DateTime.now(); // TODO get nano seconds expensive but useful info + int alt = super.adaptivePredict(input, decision, outerContext); + final stop = DateTime.now(); + decisions[decision].timeInPrediction += + (stop.difference(start)).inMicroseconds; + decisions[decision].invocations++; + + int SLL_k = _sllStopIndex - startIndex + 1; + decisions[decision].SLL_TotalLook += SLL_k; + decisions[decision].SLL_MinLook = decisions[decision].SLL_MinLook == 0 + ? SLL_k + : min(decisions[decision].SLL_MinLook, SLL_k); + if (SLL_k > decisions[decision].SLL_MaxLook) { + decisions[decision].SLL_MaxLook = SLL_k; + decisions[decision].SLL_MaxLookEvent = new LookaheadEventInfo( + decision, null, alt, input, startIndex, _sllStopIndex, false); + } + + if (_llStopIndex >= 0) { + int LL_k = _llStopIndex - startIndex + 1; + decisions[decision].LL_TotalLook += LL_k; + decisions[decision].LL_MinLook = decisions[decision].LL_MinLook == 0 + ? LL_k + : min(decisions[decision].LL_MinLook, LL_k); + if (LL_k > decisions[decision].LL_MaxLook) { + decisions[decision].LL_MaxLook = LL_k; + decisions[decision].LL_MaxLookEvent = new LookaheadEventInfo( + decision, null, alt, input, startIndex, _llStopIndex, true); + } + } + + return alt; + } finally { + this.currentDecision = -1; + } + } + + DFAState getExistingTargetState(DFAState previousD, int t) { + // this method is called after each time the input position advances + // during SLL prediction + _sllStopIndex = input.index; + + DFAState existingTargetState = super.getExistingTargetState(previousD, t); + if (existingTargetState != null) { + decisions[currentDecision] + .SLL_DFATransitions++; // count only if we transition over a DFA state + if (existingTargetState == ATNSimulator.ERROR) { + decisions[currentDecision].errors.add(new ErrorInfo(currentDecision, + previousD.configs, input, startIndex, _sllStopIndex, false)); + } + } + + currentState = existingTargetState; + return existingTargetState; + } + + DFAState computeTargetState(DFA dfa, DFAState previousD, int t) { + DFAState state = super.computeTargetState(dfa, previousD, t); + currentState = state; + return state; + } + + ATNConfigSet computeReachSet(ATNConfigSet closure, int t, bool fullCtx) { + if (fullCtx) { + // this method is called after each time the input position advances + // during full context prediction + _llStopIndex = input.index; + } + + ATNConfigSet reachConfigs = super.computeReachSet(closure, t, fullCtx); + if (fullCtx) { + decisions[currentDecision] + .LL_ATNTransitions++; // count computation even if error + if (reachConfigs != null) { + } else { + // no reach on current lookahead symbol. ERROR. + // TODO: does not handle delayed errors per getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule() + decisions[currentDecision].errors.add(new ErrorInfo( + currentDecision, closure, input, startIndex, _llStopIndex, true)); + } + } else { + decisions[currentDecision].SLL_ATNTransitions++; + if (reachConfigs != null) { + } else { + // no reach on current lookahead symbol. ERROR. + decisions[currentDecision].errors.add(new ErrorInfo( + currentDecision, closure, input, startIndex, _sllStopIndex, false)); + } + } + return reachConfigs; + } + + bool evalSemanticContextOne(SemanticContext pred, + ParserRuleContext parserCallStack, int alt, bool fullCtx) { + bool result = + super.evalSemanticContextOne(pred, parserCallStack, alt, fullCtx); + if (!(pred is PrecedencePredicate)) { + bool fullContext = _llStopIndex >= 0; + int stopIndex = fullContext ? _llStopIndex : _sllStopIndex; + decisions[currentDecision].predicateEvals.add(new PredicateEvalInfo( + currentDecision, + input, + startIndex, + stopIndex, + pred, + result, + alt, + fullCtx)); + } + + return result; + } + + void reportAttemptingFullContext(DFA dfa, BitSet conflictingAlts, + ATNConfigSet configs, int startIndex, int stopIndex) { + if (conflictingAlts != null) { + conflictingAltResolvedBySLL = conflictingAlts.nextset(0); + } else { + conflictingAltResolvedBySLL = configs.alts.nextset(0); + } + decisions[currentDecision].LL_Fallback++; + super.reportAttemptingFullContext( + dfa, conflictingAlts, configs, startIndex, stopIndex); + } + + void reportContextSensitivity(DFA dfa, int prediction, ATNConfigSet configs, + int startIndex, int stopIndex) { + if (prediction != conflictingAltResolvedBySLL) { + decisions[currentDecision].contextSensitivities.add( + new ContextSensitivityInfo( + currentDecision, configs, input, startIndex, stopIndex)); + } + super.reportContextSensitivity( + dfa, prediction, configs, startIndex, stopIndex); + } + + void reportAmbiguity(DFA dfa, DFAState D, int startIndex, int stopIndex, + bool exact, BitSet ambigAlts, ATNConfigSet configs) { + final int prediction = + ambigAlts != null ? ambigAlts.nextset(0) : configs.alts.nextset(0); + if (configs.fullCtx && prediction != conflictingAltResolvedBySLL) { + // Even though this is an ambiguity we are reporting, we can + // still detect some context sensitivities. Both SLL and LL + // are showing a conflict, hence an ambiguity, but if they resolve + // to different minimum alternatives we have also identified a + // context sensitivity. + decisions[currentDecision].contextSensitivities.add( + new ContextSensitivityInfo( + currentDecision, configs, input, startIndex, stopIndex)); + } + decisions[currentDecision].ambiguities.add(new AmbiguityInfo( + currentDecision, + configs, + ambigAlts, + input, + startIndex, + stopIndex, + configs.fullCtx)); + super.reportAmbiguity( + dfa, D, startIndex, stopIndex, exact, ambigAlts, configs); + } + + // --------------------------------------------------------------------- + + List get decisionInfo { + return decisions; + } +} diff --git a/runtime/Dart/lib/src/atn/src/semantic_context.dart b/runtime/Dart/lib/src/atn/src/semantic_context.dart new file mode 100644 index 000000000..cc117f829 --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/semantic_context.dart @@ -0,0 +1,399 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'package:collection/collection.dart'; + +import '../../recognizer.dart'; +import '../../rule_context.dart'; +import '../../util/murmur_hash.dart'; + +/** A tree structure used to record the semantic context in which + * an ATN configuration is valid. It's either a single predicate, + * a conjunction {@code p1&&p2}, or a sum of products {@code p1||p2}. + * + *

I have scoped the [AND], [OR], and [Predicate] subclasses of + * [SemanticContext] within the scope of this outer class.

+ */ +abstract class SemanticContext { + /** + * The default [SemanticContext], which is semantically equivalent to + * a predicate of the form {@code {true}?}. + */ + static const SemanticContext NONE = const Predicate(); + + const SemanticContext(); + + /** + * For context independent predicates, we evaluate them without a local + * context (i.e., null context). That way, we can evaluate them without + * having to create proper rule-specific context during prediction (as + * opposed to the parser, which creates them naturally). In a practical + * sense, this avoids a cast exception from RuleContext to myruleContext. + * + *

For context dependent predicates, we must pass in a local context so that + * references such as $arg evaluate properly as _localctx.arg. We only + * capture context dependent predicates in the context in which we begin + * prediction, so we passed in the outer context here in case of context + * dependent predicate evaluation.

+ */ + bool eval(Recognizer parser, RuleContext parserCallStack); + + /** + * Evaluate the precedence predicates for the context and reduce the result. + * + * @param parser The parser instance. + * @param parserCallStack + * @return The simplified semantic context after precedence predicates are + * evaluated, which will be one of the following values. + *
    + *
  • {@link #NONE}in if the predicate simplifies to [true] after + * precedence predicates are evaluated.
    • + *
  • nullin if the predicate simplifies to [false] after + * precedence predicates are evaluated.
    • + *
  • [this]in if the semantic context is not changed as a result of + * precedence predicate evaluation.
    • + *
  • A non-null [SemanticContext]in the new simplified + * semantic context after precedence predicates are evaluated.
    • + *
+ */ + SemanticContext evalPrecedence(Recognizer parser, + RuleContext parserCallStack) { + return this; + } + + static SemanticContext and(SemanticContext a, SemanticContext b) { + if (a == null || a == NONE) return b; + if (b == null || b == NONE) return a; + AND result = new AND(a, b); + if (result.opnds.length == 1) { + return result.opnds[0]; + } + + return result; + } + + /** + * + * @see ParserATNSimulator#getPredsForAmbigAlts + */ + static SemanticContext or(SemanticContext a, SemanticContext b) { + if (a == null) return b; + if (b == null) return a; + if (a == NONE || b == NONE) return NONE; + OR result = new OR(a, b); + if (result.opnds.length == 1) { + return result.opnds[0]; + } + + return result; + } + + static Iterable filterPrecedencePredicates( + Iterable collection) { + return collection.where((e) => e is PrecedencePredicate).map((e)=> e as PrecedencePredicate); + } + + static Iterable filterNonPrecedencePredicates( + Iterable collection) { + return collection.where((e) => !(e is PrecedencePredicate)); + } +} + +class Predicate extends SemanticContext { + final int ruleIndex; + final int predIndex; + final bool isCtxDependent; // e.g., $i ref in pred + + const Predicate( + [this.ruleIndex = -1, this.predIndex = -1, this.isCtxDependent = false]); + + bool eval(Recognizer parser, RuleContext parserCallStack) { + RuleContext localctx = isCtxDependent ? parserCallStack : null; + return parser.sempred(localctx, ruleIndex, predIndex); + } + + int get hashCode { + int hashCode = MurmurHash.initialize(); + hashCode = MurmurHash.update(hashCode, ruleIndex); + hashCode = MurmurHash.update(hashCode, predIndex); + hashCode = MurmurHash.update(hashCode, isCtxDependent ? 1 : 0); + hashCode = MurmurHash.finish(hashCode, 3); + return hashCode; + } + + operator ==(Object obj) { + return obj is Predicate && + this.ruleIndex == obj.ruleIndex && + this.predIndex == obj.predIndex && + this.isCtxDependent == obj.isCtxDependent; + } + + String toString() { + return "{$ruleIndex:$predIndex}?"; + } +} + +class PrecedencePredicate extends SemanticContext + implements Comparable { + final int precedence; + + PrecedencePredicate([this.precedence = 0]); + + bool eval(Recognizer parser, RuleContext parserCallStack) { + return parser.precpred(parserCallStack, precedence); + } + + SemanticContext evalPrecedence(Recognizer parser, + RuleContext parserCallStack) { + if (parser.precpred(parserCallStack, precedence)) { + return SemanticContext.NONE; + } else { + return null; + } + } + + int compareTo(PrecedencePredicate o) { + return precedence - o.precedence; + } + + get hashCode { + int hashCode = 1; + hashCode = 31 * hashCode + precedence; + return hashCode; + } + + operator ==(Object obj) { + if (!(obj is PrecedencePredicate)) { + return false; + } + PrecedencePredicate other = obj; + return this.precedence == other.precedence; + } + +// precedence >= _precedenceStack.peek() + String toString() { + return "{$precedence>=prec}?"; + } +} + +/** + * This is the base class for semantic context "operators", which operate on + * a collection of semantic context "operands". + * + * @since 4.3 + */ +abstract class Operator extends SemanticContext { + /** + * Gets the operands for the semantic context operator. + * + * @return a collection of [SemanticContext] operands for the + * operator. + * + * @since 4.3 + */ + List get operands; +} + +/** + * A semantic context which is true whenever none of the contained contexts + * is false. + */ + +class AND extends Operator { + List opnds; + + AND(SemanticContext a, SemanticContext b) { + Set operands = Set(); + if (a is AND) + operands.addAll(a.opnds); + else + operands.add(a); + if (b is AND) + operands.addAll(b.opnds); + else + operands.add(b); + + Iterable precedencePredicates = + SemanticContext.filterPrecedencePredicates(operands); + + operands = SemanticContext.filterNonPrecedencePredicates(operands).toSet(); + if (!precedencePredicates.isEmpty) { + // interested in the transition with the lowest precedence + PrecedencePredicate reduced = + precedencePredicates.reduce((a, b) => a.compareTo(b) <= 0 ? a : b); + operands.add(reduced); + } + + opnds = operands.toList(); + } + + List get operands { + return opnds; + } + + operator ==(Object obj) { + if (!(obj is AND)) return false; + AND other = obj; + return ListEquality().equals(this.opnds, other.opnds); + } + + get hashCode { + return MurmurHash.getHashCode(opnds, this.runtimeType.hashCode); + } + + /** + * {@inheritDoc} + * + *

+ * The evaluation of predicates by this context is short-circuiting, but + * unordered.

+ */ + + bool eval(Recognizer parser, RuleContext parserCallStack) { + for (SemanticContext opnd in opnds) { + if (!opnd.eval(parser, parserCallStack)) return false; + } + return true; + } + + SemanticContext evalPrecedence(Recognizer parser, + RuleContext parserCallStack) { + bool differs = false; + List operands = []; + for (SemanticContext context in opnds) { + SemanticContext evaluated = + context.evalPrecedence(parser, parserCallStack); + differs |= (evaluated != context); + if (evaluated == null) { + // The AND context is false if any element is false + return null; + } else if (evaluated != SemanticContext.NONE) { + // Reduce the result by skipping true elements + operands.add(evaluated); + } + } + + if (!differs) { + return this; + } + + if (operands.isEmpty) { + // all elements were true, so the AND context is true + return SemanticContext.NONE; + } + + SemanticContext result = operands[0]; + for (int i = 1; i < operands.length; i++) { + result = SemanticContext.and(result, operands[i]); + } + + return result; + } + + String toString() { + return opnds.join("&&"); + } +} + +/** + * A semantic context which is true whenever at least one of the contained + * contexts is true. + */ +class OR extends Operator { + List opnds; + + OR(SemanticContext a, SemanticContext b) { + Set operands = Set(); + if (a is OR) + operands.addAll(a.opnds); + else + operands.add(a); + if (b is OR) + operands.addAll(b.opnds); + else + operands.add(b); + + Iterable precedencePredicates = + SemanticContext.filterPrecedencePredicates(operands); + + operands = SemanticContext.filterNonPrecedencePredicates(operands).toSet(); + if (!precedencePredicates.isEmpty) { + // interested in the transition with the highest precedence + PrecedencePredicate reduced = + precedencePredicates.reduce((a, b) => a.compareTo(b) >= 0 ? a : b); + operands.add(reduced); + } + + this.opnds = operands.toList(); + } + + List get operands { + return opnds; + } + + operator ==(Object obj) { + if (!(obj is OR)) return false; + OR other = obj; + return ListEquality().equals(this.opnds, other.opnds); + } + + get hashCode { + return MurmurHash.getHashCode(opnds, this.runtimeType.hashCode); + } + + /** + * {@inheritDoc} + * + *

+ * The evaluation of predicates by this context is short-circuiting, but + * unordered.

+ */ + + bool eval(Recognizer parser, RuleContext parserCallStack) { + for (SemanticContext opnd in opnds) { + if (opnd.eval(parser, parserCallStack)) return true; + } + return false; + } + + SemanticContext evalPrecedence(Recognizer parser, + RuleContext parserCallStack) { + bool differs = false; + List operands = []; + for (SemanticContext context in opnds) { + SemanticContext evaluated = + context.evalPrecedence(parser, parserCallStack); + differs |= (evaluated != context); + if (evaluated == SemanticContext.NONE) { + // The OR context is true if any element is true + return SemanticContext.NONE; + } else if (evaluated != null) { + // Reduce the result by skipping false elements + operands.add(evaluated); + } + } + + if (!differs) { + return this; + } + + if (operands.isEmpty) { + // all elements were false, so the OR context is false + return null; + } + + SemanticContext result = operands[0]; + for (int i = 1; i < operands.length; i++) { + result = SemanticContext.or(result, operands[i]); + } + + return result; + } + + String toString() { + return opnds.join("||"); + } +} diff --git a/runtime/Dart/lib/src/atn/src/transition.dart b/runtime/Dart/lib/src/atn/src/transition.dart new file mode 100644 index 000000000..3a13afa3b --- /dev/null +++ b/runtime/Dart/lib/src/atn/src/transition.dart @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import '../../interval_set.dart'; +import '../../token.dart'; +import 'atn_state.dart'; +import 'semantic_context.dart'; + +enum TransitionType { + INVALID, // 0 is not used + EPSILON, + RANGE, + RULE, + PREDICATE, // e.g., {isType(input.LT(1))}? + ATOM, + ACTION, + SET, // ~(A|B) or ~atom, wildcard, which convert to next 2 + NOT_SET, + WILDCARD, + PRECEDENCE, +} + +/** An ATN transition between any two ATN states. Subclasses define + * atom, set, epsilon, action, predicate, rule transitions. + * + *

This is a one way link. It emanates from a state (usually via a list of + * transitions) and has a target state.

+ * + *

Since we never have to change the ATN transitions once we construct it, + * we can fix these transitions as specific classes. The DFA transitions + * on the other hand need to update the labels as it adds transitions to + * the states. We'll use the term Edge for the DFA to distinguish them from + * ATN transitions.

+ */ +abstract class Transition { + /** The target of this transition. */ + ATNState target; + + Transition(this.target) { + if (target == null) { + throw new ArgumentError.notNull("target cannot be null."); + } + } + + TransitionType get type; + + /** + * Determines if the transition is an "epsilon" transition. + * + *

The default implementation returns [false].

+ * + * @return [true] if traversing this transition in the ATN does not + * consume an input symbol; otherwise, [false] if traversing this + * transition consumes (matches) an input symbol. + */ + bool get isEpsilon => false; + + IntervalSet get label => null; + + bool matches(int symbol, int minVocabSymbol, int maxVocabSymbol); +} + +class EpsilonTransition extends Transition { + /** + * @return the rule index of a precedence rule for which this transition is + * returning from, where the precedence value is 0; otherwise, -1. + * + * @see ATNConfig#isPrecedenceFilterSuppressed() + * @see ParserATNSimulator#applyPrecedenceFilter(ATNConfigSet) + * @since 4.4.1 + */ + final int outermostPrecedenceReturn; + + EpsilonTransition(ATNState target, [this.outermostPrecedenceReturn = -1]) + : super(target); + + get isEpsilon => true; + + bool matches(int symbol, int minVocabSymbol, int maxVocabSymbol) { + return false; + } + + String toString() { + return "epsilon"; + } + + @override + TransitionType get type => TransitionType.EPSILON; +} + +class RangeTransition extends Transition { + final int from; + final int to; + + RangeTransition(ATNState target, this.from, this.to) : super(target); + + IntervalSet get label { + return IntervalSet.ofRange(from, to); + } + + bool matches(int symbol, int minVocabSymbol, int maxVocabSymbol) { + return symbol >= from && symbol <= to; + } + + String toString() { + return "'$from..$to'"; + } + + @override + TransitionType get type => TransitionType.RANGE; +} + +class RuleTransition extends Transition { + /** Ptr to the rule definition object for this rule ref */ + final int ruleIndex; // no Rule object at runtime + + final int precedence; + + /** What node to begin computations following ref to rule */ + ATNState followState; + + RuleTransition(RuleStartState ruleStart, this.ruleIndex, this.precedence, + this.followState) + : super(ruleStart); + + get isEpsilon => true; + + bool matches(int symbol, int minVocabSymbol, int maxVocabSymbol) { + return false; + } + + @override + TransitionType get type => TransitionType.RULE; +} + +abstract class AbstractPredicateTransition extends Transition { + AbstractPredicateTransition(ATNState target) : super(target); +} + +class PredicateTransition extends AbstractPredicateTransition { + final int ruleIndex; + final int predIndex; + final bool isCtxDependent; // e.g., $i ref in pred + + PredicateTransition( + target, this.ruleIndex, this.predIndex, this.isCtxDependent) + : super(target); + + get isEpsilon => true; + + matches(symbol, minVocabSymbol, maxVocabSymbol) { + return false; + } + + get predicate => new Predicate(this.ruleIndex, this.predIndex, this.isCtxDependent); + + + toString() { + return "pred_$ruleIndex:$predIndex"; + } + + @override + TransitionType get type => TransitionType.PREDICATE; +} + +/** TODO: make all transitions sets? no, should remove set edges */ +class AtomTransition extends Transition { + /** The token type or character value; or, signifies special label. */ + final int atomLabel; + + AtomTransition(ATNState target, this.atomLabel) : super(target); + + IntervalSet get label { + return IntervalSet.ofOne(atomLabel); + } + + bool matches(int symbol, int minVocabSymbol, int maxVocabSymbol) { + return atomLabel == symbol; + } + + String toString() { + return label.toString(); + } + + @override + TransitionType get type => TransitionType.ATOM; +} + +class ActionTransition extends Transition { + final int ruleIndex; + final int actionIndex; + final bool isCtxDependent; // e.g., $i ref in pred + + ActionTransition(target, this.ruleIndex, + [this.actionIndex = -1, this.isCtxDependent = false]) + : super(target); + + bool get isEpsilon => + true; // we are to be ignored by analysis 'cept for predicates + + matches(symbol, minVocabSymbol, maxVocabSymbol) => false; + + toString() { + return "action_$ruleIndex:$actionIndex"; + } + + @override + TransitionType get type => TransitionType.ACTION; +} + +// A transition containing a set of values. +class SetTransition extends Transition { + IntervalSet label; + + SetTransition(ATNState target, [IntervalSet st]) : super(target) { + this.label = st ?? IntervalSet.ofOne(Token.INVALID_TYPE); + } + + matches(symbol, minVocabSymbol, maxVocabSymbol) { + return this.label.contains(symbol); + } + + toString() { + return this.label.toString(); + } + + @override + TransitionType get type => TransitionType.SET; +} + +class NotSetTransition extends SetTransition { + NotSetTransition(target, st) : super(target, st); + + matches(symbol, minVocabSymbol, maxVocabSymbol) { + return symbol >= minVocabSymbol && + symbol <= maxVocabSymbol && + !super.matches(symbol, minVocabSymbol, maxVocabSymbol); + } + + toString() { + return '~' + super.toString(); + } + + @override + TransitionType get type => TransitionType.NOT_SET; +} + +class WildcardTransition extends Transition { + WildcardTransition(target) : super(target); + + matches(symbol, minVocabSymbol, maxVocabSymbol) { + return symbol >= minVocabSymbol && symbol <= maxVocabSymbol; + } + + toString() { + return "."; + } + + @override + TransitionType get type => TransitionType.WILDCARD; +} + +class PrecedencePredicateTransition extends AbstractPredicateTransition { + final int precedence; + + PrecedencePredicateTransition(target, this.precedence) : super(target); + + get isEpsilon => true; + + matches(symbol, minVocabSymbol, maxVocabSymbol) => false; + + PrecedencePredicate get predicate { + return new PrecedencePredicate(precedence); + } + + toString() => "$precedence >= _p"; + + @override + TransitionType get type => TransitionType.PRECEDENCE; +} diff --git a/runtime/Dart/lib/src/dfa/dfa.dart b/runtime/Dart/lib/src/dfa/dfa.dart new file mode 100644 index 000000000..552ddef7d --- /dev/null +++ b/runtime/Dart/lib/src/dfa/dfa.dart @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +export 'src/dfa.dart'; +export 'src/dfa_state.dart'; diff --git a/runtime/Dart/lib/src/dfa/src/dfa.dart b/runtime/Dart/lib/src/dfa/src/dfa.dart new file mode 100644 index 000000000..092d63e65 --- /dev/null +++ b/runtime/Dart/lib/src/dfa/src/dfa.dart @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import '../../vocabulary.dart'; +import '../../atn/atn.dart'; +import 'dfa_serializer.dart'; +import 'dfa_state.dart'; + +class DFA { + /** A set of all DFA states. Use [Map] so we can get old state back + * ([Set] only allows you to see if it's there). + */ + + Map states = {}; + + DFAState s0; + + final int decision; + + /** From which ATN state did we create this DFA? */ + + DecisionState atnStartState; + + /** + * [true] if this DFA is for a precedence decision; otherwise, + * [false]. This is the backing field for {@link #isPrecedenceDfa}. + */ + bool precedenceDfa; + + DFA(this.atnStartState, [this.decision]) { + bool precedenceDfa = false; + if (atnStartState is StarLoopEntryState) { + if ((atnStartState as StarLoopEntryState).isPrecedenceDecision) { + precedenceDfa = true; + DFAState precedenceState = new DFAState(configs: ATNConfigSet()); + precedenceState.edges = []; + precedenceState.isAcceptState = false; + precedenceState.requiresFullContext = false; + this.s0 = precedenceState; + } + } + + this.precedenceDfa = precedenceDfa; + } + + /** + * Gets whether this DFA is a precedence DFA. Precedence DFAs use a special + * start state {@link #s0} which is not stored in {@link #states}. The + * {@link DFAState#edges} array for this start state contains outgoing edges + * supplying individual start states corresponding to specific precedence + * values. + * + * @return [true] if this is a precedence DFA; otherwise, + * [false]. + * @see Parser#getPrecedence() + */ + bool isPrecedenceDfa() { + return precedenceDfa; + } + + /** + * Get the start state for a specific precedence value. + * + * @param precedence The current precedence. + * @return The start state corresponding to the specified precedence, or + * null if no start state exists for the specified precedence. + * + * @throws IllegalStateException if this is not a precedence DFA. + * @see #isPrecedenceDfa() + */ + DFAState getPrecedenceStartState(int precedence) { + if (!isPrecedenceDfa()) { + throw new StateError( + "Only precedence DFAs may contain a precedence start state."); + } + + // s0.edges is never null for a precedence DFA + if (precedence < 0 || precedence >= s0.edges.length) { + return null; + } + + return s0.edges[precedence]; + } + + /** + * Set the start state for a specific precedence value. + * + * @param precedence The current precedence. + * @param startState The start state corresponding to the specified + * precedence. + * + * @throws IllegalStateException if this is not a precedence DFA. + * @see #isPrecedenceDfa() + */ + void setPrecedenceStartState(int precedence, DFAState startState) { + if (!isPrecedenceDfa()) { + throw new StateError( + "Only precedence DFAs may contain a precedence start state."); + } + + if (precedence < 0) { + return; + } + + // synchronization on s0 here is ok. when the DFA is turned into a + // precedence DFA, s0 will be initialized once and not updated again + // s0.edges is never null for a precedence DFA + if (precedence >= s0.edges.length) { + final original = s0.edges; + s0.edges = List(precedence + 1); + List.copyRange(s0.edges, 0, original); + } + + s0.edges[precedence] = startState; + } + + /** + * Return a list of all states in this DFA, ordered by state number. + */ + + List getStates() { + List result = states.keys.toList(); + result.sort((DFAState o1, DFAState o2) { + return o1.stateNumber - o2.stateNumber; + }); + + return result; + } + + String toString([Vocabulary vocabulary]) { + vocabulary = vocabulary ?? VocabularyImpl.EMPTY_VOCABULARY; + if (s0 == null) { + return ""; + } + + DFASerializer serializer = new DFASerializer(this, vocabulary); + return serializer.toString(); + } + + String toLexerString() { + if (s0 == null) return ""; + DFASerializer serializer = new LexerDFASerializer(this); + return serializer.toString(); + } +} diff --git a/runtime/Dart/lib/src/dfa/src/dfa_serializer.dart b/runtime/Dart/lib/src/dfa/src/dfa_serializer.dart new file mode 100644 index 000000000..f7661f613 --- /dev/null +++ b/runtime/Dart/lib/src/dfa/src/dfa_serializer.dart @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import '../../vocabulary.dart'; +import '../../util/utils.dart'; +import 'dfa.dart'; +import 'dfa_state.dart'; + +/** A DFA walker that knows how to dump them to serialized strings. */ +class DFASerializer { + final DFA dfa; + + final Vocabulary vocabulary; + + DFASerializer(this.dfa, this.vocabulary); + + String toString() { + if (dfa.s0 == null) return null; + StringBuffer buf = new StringBuffer(); + List states = dfa.getStates(); + for (DFAState s in states) { + int n = 0; + if (s.edges != null) n = s.edges.length; + for (int i = 0; i < n; i++) { + DFAState t = s.edges[i]; + if (t != null && t.stateNumber != 0x7FFFFFFF) { + buf.write(getStateString(s)); + String label = getEdgeLabel(i); + buf.write("-"); + buf.write(label); + buf.write("->"); + buf.write(getStateString(t)); + buf.write('\n'); + } + } + } + + String output = buf.toString(); + if (output.length == 0) return null; + //return Utils.sortLinesInString(output); + return output; + } + + String getEdgeLabel(int i) { + return vocabulary.getDisplayName(i - 1); + } + + String getStateString(DFAState s) { + int n = s.stateNumber; + final String baseStateStr = (s.isAcceptState ? ":" : "") + + "s$n" + + (s.requiresFullContext ? "^" : ""); + if (s.isAcceptState) { + if (s.predicates != null) { + return baseStateStr + "=>${arrayToString(s.predicates)}"; + } else { + return baseStateStr + "=>${s.prediction}"; + } + } else { + return baseStateStr; + } + } +} + +class LexerDFASerializer extends DFASerializer { + LexerDFASerializer(dfa) : super(dfa, VocabularyImpl.EMPTY_VOCABULARY); + + String getEdgeLabel(i) { + return "'" + String.fromCharCode(i) + "'"; + } +} diff --git a/runtime/Dart/lib/src/dfa/src/dfa_state.dart b/runtime/Dart/lib/src/dfa/src/dfa_state.dart new file mode 100644 index 000000000..13926f325 --- /dev/null +++ b/runtime/Dart/lib/src/dfa/src/dfa_state.dart @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import '../../atn/atn.dart'; +import '../../util/murmur_hash.dart'; +import '../../util/utils.dart'; + +/** Map a predicate to a predicted alternative. */ +class PredPrediction { + var alt; + var pred; + + PredPrediction(this.pred, this.alt); + + toString() { + return "($pred, $alt)"; + } +} + +/** A DFA state represents a set of possible ATN configurations. + * As Aho, Sethi, Ullman p. 117 says "The DFA uses its state + * to keep track of all possible states the ATN can be in after + * reading each input symbol. That is to say, after reading + * input a1a2..an, the DFA is in a state that represents the + * subset T of the states of the ATN that are reachable from the + * ATN's start state along some path labeled a1a2..an." + * In conventional NFA→DFA conversion, therefore, the subset T + * would be a bitset representing the set of states the + * ATN could be in. We need to track the alt predicted by each + * state as well, however. More importantly, we need to maintain + * a stack of states, tracking the closure operations as they + * jump from rule to rule, emulating rule invocations (method calls). + * I have to add a stack to simulate the proper lookahead sequences for + * the underlying LL grammar from which the ATN was derived. + * + *

I use a set of ATNConfig objects not simple states. An ATNConfig + * is both a state (ala normal conversion) and a RuleContext describing + * the chain of rules (if any) followed to arrive at that state.

+ * + *

A DFA state may have multiple references to a particular state, + * but with different ATN contexts (with same or different alts) + * meaning that state was reached via a different set of rule invocations.

+ */ +class DFAState { + int stateNumber = -1; + + ATNConfigSet configs = new ATNConfigSet(); + + /** {@code edges[symbol]} points to target of symbol. Shift up by 1 so (-1) + * {@link Token#EOF} maps to {@code edges[0]}. + */ + + List edges; + + bool isAcceptState = false; + + /** if accept state, what ttype do we match or alt do we predict? + * This is set to {@link ATN#INVALID_ALT_NUMBER} when {@link #predicates}{@code !=null} or + * {@link #requiresFullContext}. + */ + int prediction = 0; + + LexerActionExecutor lexerActionExecutor; + + /** + * Indicates that this state was created during SLL prediction that + * discovered a conflict between the configurations in the state. Future + * {@link ParserATNSimulator#execATN} invocations immediately jumped doing + * full context prediction if this field is true. + */ + bool requiresFullContext = false; + + /** During SLL parsing, this is a list of predicates associated with the + * ATN configurations of the DFA state. When we have predicates, + * {@link #requiresFullContext} is [false] since full context prediction evaluates predicates + * on-the-fly. If this is not null, then {@link #prediction} is + * {@link ATN#INVALID_ALT_NUMBER}. + * + *

We only use these for non-{@link #requiresFullContext} but conflicting states. That + * means we know from the context (it's $ or we don't dip into outer + * context) that it's an ambiguity not a conflict.

+ * + *

This list is computed by {@link ParserATNSimulator#predicateDFAState}.

+ */ + + List predicates; + + DFAState({this.stateNumber, this.configs}); + + /** Get the set of all alts mentioned by all ATN configurations in this + * DFA state. + */ + Set get altSet { + Set alts = new Set(); + if (configs != null) { + for (ATNConfig c in configs) { + alts.add(c.alt); + } + } + if (alts.isEmpty) return null; + return alts; + } + + int get hashCode { + int hash = MurmurHash.initialize(7); + hash = MurmurHash.update(hash, configs.hashCode); + hash = MurmurHash.finish(hash, 1); + return hash; + } + + /** + * Two [DFAState] instances are equal if their ATN configuration sets + * are the same. This method is used to see if a state already exists. + * + *

Because the number of alternatives and number of ATN configurations are + * finite, there is a finite number of DFA states that can be processed. + * This is necessary to show that the algorithm terminates.

+ * + *

Cannot test the DFA state numbers here because in + * {@link ParserATNSimulator#addDFAState} we need to know if any other state + * exists that has this exact set of ATN configurations. The + * {@link #stateNumber} is irrelevant.

+ */ + + bool operator ==(Object o) { + // compare set of ATN configurations in this set with other + if (identical(this, o)) return true; + + if (!(o is DFAState)) { + return false; + } + + DFAState other = o; + // TODO (sam): what to do when configs==null? + bool sameSet = this.configs == other.configs; +// System.out.println("DFAState.equals: "+configs+(sameSet?"==":"!=")+other.configs); + return sameSet; + } + + String toString() { + StringBuffer buf = new StringBuffer(); + buf.write("$stateNumber:$configs"); + if (isAcceptState) { + buf.write("=>"); + if (predicates != null) { + buf.write(arrayToString(predicates)); + } else { + buf.write(prediction); + } + } + return buf.toString(); + } +} diff --git a/runtime/Dart/lib/src/error/error.dart b/runtime/Dart/lib/src/error/error.dart new file mode 100644 index 000000000..8b96c26b9 --- /dev/null +++ b/runtime/Dart/lib/src/error/error.dart @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +export 'src/diagnostic_error_listener.dart'; +export 'src/error_listener.dart'; +export 'src/error_strategy.dart'; +export 'src/errors.dart'; diff --git a/runtime/Dart/lib/src/error/src/diagnostic_error_listener.dart b/runtime/Dart/lib/src/error/src/diagnostic_error_listener.dart new file mode 100644 index 000000000..7ded001e4 --- /dev/null +++ b/runtime/Dart/lib/src/error/src/diagnostic_error_listener.dart @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import '../../atn/atn.dart'; +import '../../dfa/dfa.dart'; +import '../../interval_set.dart'; +import '../../parser.dart'; +import '../../util/bit_set.dart'; +import 'error_listener.dart'; + +/** + * This implementation of [ANTLRErrorListener] can be used to identify + * certain potential correctness and performance problems in grammars. "Reports" + * are made by calling {@link Parser#notifyErrorListeners} with the appropriate + * message. + * + *
    + *
  • Ambiguities: These are cases where more than one path through the + * grammar can match the input.
    • + *
  • Weak context sensitivity: These are cases where full-context + * prediction resolved an SLL conflict to a unique alternative which equaled the + * minimum alternative of the SLL conflict.
    • + *
  • Strong (forced) context sensitivity: These are cases where the + * full-context prediction resolved an SLL conflict to a unique alternative, + * and the minimum alternative of the SLL conflict was found to not be + * a truly viable alternative. Two-stage parsing cannot be used for inputs where + * this situation occurs.
    • + *
+ */ +class DiagnosticErrorListener extends BaseErrorListener { + /** + * When [true], only exactly known ambiguities are reported. + */ + final bool exactOnly; + + /** + * Initializes a new instance of [DiagnosticErrorListener], specifying + * whether all ambiguities or only exact ambiguities are reported. + * + * @param exactOnly [true] to report only exact ambiguities, otherwise + * [false] to report all ambiguities. + */ + DiagnosticErrorListener([this.exactOnly = true]); + + void reportAmbiguity(Parser recognizer, DFA dfa, int startIndex, + int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { + if (exactOnly && !exact) { + return; + } + + final decision = getDecisionDescription(recognizer, dfa); + final conflictingAlts = getConflictingAlts(ambigAlts, configs); + final text = + recognizer.tokenStream.getText(Interval.of(startIndex, stopIndex)); + final message = + "reportAmbiguity d=$decision: ambigAlts=$conflictingAlts, input='$text'"; + recognizer.notifyErrorListeners(message); + } + + void reportAttemptingFullContext(Parser recognizer, DFA dfa, int startIndex, + int stopIndex, BitSet conflictingAlts, ATNConfigSet configs) { + final decision = getDecisionDescription(recognizer, dfa); + final text = + recognizer.tokenStream.getText(Interval.of(startIndex, stopIndex)); + final message = "reportAttemptingFullContext d=$decision, input='$text'"; + recognizer.notifyErrorListeners(message); + } + + void reportContextSensitivity(Parser recognizer, DFA dfa, int startIndex, + int stopIndex, int prediction, ATNConfigSet configs) { + String decision = getDecisionDescription(recognizer, dfa); + String text = + recognizer.tokenStream.getText(Interval.of(startIndex, stopIndex)); + String message = "reportContextSensitivity d=$decision, input='$text'"; + recognizer.notifyErrorListeners(message); + } + + String getDecisionDescription(Parser recognizer, DFA dfa) { + int decision = dfa.decision; + int ruleIndex = dfa.atnStartState.ruleIndex; + + final ruleNames = recognizer.ruleNames; + if (ruleIndex < 0 || ruleIndex >= ruleNames.length) { + return decision.toString(); + } + + final ruleName = ruleNames[ruleIndex]; + if (ruleName == null || ruleName.isEmpty) { + return decision.toString(); + } + + return "$decision ($ruleName)"; + } + + /** + * Computes the set of conflicting or ambiguous alternatives from a + * configuration set, if that information was not already provided by the + * parser. + * + * @param reportedAlts The set of conflicting or ambiguous alternatives, as + * reported by the parser. + * @param configs The conflicting or ambiguous configuration set. + * @return Returns [reportedAlts] if it is not null, otherwise + * returns the set of alternatives represented in [configs]. + */ + BitSet getConflictingAlts(BitSet reportedAlts, ATNConfigSet configs) { + if (reportedAlts != null) { + return reportedAlts; + } + + BitSet result = new BitSet(); + for (ATNConfig config in configs) { + result.set(config.alt); + } + + return result; + } +} diff --git a/runtime/Dart/lib/src/error/src/error_listener.dart b/runtime/Dart/lib/src/error/src/error_listener.dart new file mode 100644 index 000000000..b3103a9e7 --- /dev/null +++ b/runtime/Dart/lib/src/error/src/error_listener.dart @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:io'; + +import '../../atn/atn.dart'; +import '../../dfa/dfa.dart'; +import '../../parser.dart'; +import '../../recognizer.dart'; +import '../../util/bit_set.dart'; +import 'errors.dart'; + +abstract class ErrorListener { + /** + * Upon syntax error, notify any interested parties. This is not how to + * recover from errors or compute error messages. [ANTLRErrorStrategy] + * specifies how to recover from syntax errors and how to compute error + * messages. This listener's job is simply to emit a computed message, + * though it has enough information to create its own message in many cases. + * + *

The [RecognitionException] is non-null for all syntax errors except + * when we discover mismatched token errors that we can recover from + * in-line, without returning from the surrounding rule (via the single + * token insertion and deletion mechanism).

+ * + * @param recognizer + * What parser got the error. From this + * object, you can access the context as well + * as the input stream. + * @param offendingSymbol + * The offending token in the input token + * stream, unless recognizer is a lexer (then it's null). If + * no viable alternative error, [e] has token at which we + * started production for the decision. + * @param line + * The line number in the input where the error occurred. + * @param charPositionInLine + * The character position within that line where the error occurred. + * @param msg + * The message to emit. + * @param e + * The exception generated by the parser that led to + * the reporting of an error. It is null in the case where + * the parser was able to recover in line without exiting the + * surrounding rule. + */ + void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, + int charPositionInLine, String msg, RecognitionException e); + + /** + * This method is called by the parser when a full-context prediction + * results in an ambiguity. + * + *

Each full-context prediction which does not result in a syntax error + * will call either {@link #reportContextSensitivity} or + * {@link #reportAmbiguity}.

+ * + *

When [ambigAlts] is not null, it contains the set of potentially + * viable alternatives identified by the prediction algorithm. When + * [ambigAlts] is null, use {@link ATNConfigSet#getAlts} to obtain the + * represented alternatives from the [configs] argument.

+ * + *

When [exact] is [true], all of the potentially + * viable alternatives are truly viable, i.e. this is reporting an exact + * ambiguity. When [exact] is [false], at least two of + * the potentially viable alternatives are viable for the current input, but + * the prediction algorithm terminated as soon as it determined that at + * least the minimum potentially viable alternative is truly + * viable.

+ * + *

When the {@link PredictionMode#LL_EXACT_AMBIG_DETECTION} prediction + * mode is used, the parser is required to identify exact ambiguities so + * [exact] will always be [true].

+ * + *

This method is not used by lexers.

+ * + * @param recognizer the parser instance + * @param dfa the DFA for the current decision + * @param startIndex the input index where the decision started + * @param stopIndex the input input where the ambiguity was identified + * @param exact [true] if the ambiguity is exactly known, otherwise + * [false]. This is always [true] when + * {@link PredictionMode#LL_EXACT_AMBIG_DETECTION} is used. + * @param ambigAlts the potentially ambiguous alternatives, or null + * to indicate that the potentially ambiguous alternatives are the complete + * set of represented alternatives in [configs] + * @param configs the ATN configuration set where the ambiguity was + * identified + */ + void reportAmbiguity(Parser recognizer, DFA dfa, int startIndex, + int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs); + + /** + * This method is called when an SLL conflict occurs and the parser is about + * to use the full context information to make an LL decision. + * + *

If one or more configurations in [configs] contains a semantic + * predicate, the predicates are evaluated before this method is called. The + * subset of alternatives which are still viable after predicates are + * evaluated is reported in [conflictingAlts].

+ * + *

This method is not used by lexers.

+ * + * @param recognizer the parser instance + * @param dfa the DFA for the current decision + * @param startIndex the input index where the decision started + * @param stopIndex the input index where the SLL conflict occurred + * @param conflictingAlts The specific conflicting alternatives. If this is + * null, the conflicting alternatives are all alternatives + * represented in [configs]. At the moment, conflictingAlts is non-null + * (for the reference implementation, but Sam's optimized version can see this + * as null). + * @param configs the ATN configuration set where the SLL conflict was + * detected + */ + void reportAttemptingFullContext(Parser recognizer, DFA dfa, int startIndex, + int stopIndex, BitSet conflictingAlts, ATNConfigSet configs); + + /** + * This method is called by the parser when a full-context prediction has a + * unique result. + * + *

Each full-context prediction which does not result in a syntax error + * will call either {@link #reportContextSensitivity} or + * {@link #reportAmbiguity}.

+ * + *

For prediction implementations that only evaluate full-context + * predictions when an SLL conflict is found (including the default + * [ParserATNSimulator] implementation), this method reports cases + * where SLL conflicts were resolved to unique full-context predictions, + * i.e. the decision was context-sensitive. This report does not necessarily + * indicate a problem, and it may appear even in completely unambiguous + * grammars.

+ * + *

[configs] may have more than one represented alternative if the + * full-context prediction algorithm does not evaluate predicates before + * beginning the full-context prediction. In all cases, the final prediction + * is passed as the [prediction] argument.

+ * + *

Note that the definition of "context sensitivity" in this method + * differs from the concept in {@link DecisionInfo#contextSensitivities}. + * This method reports all instances where an SLL conflict occurred but LL + * parsing produced a unique result, whether or not that unique result + * matches the minimum alternative in the SLL conflicting set.

+ * + *

This method is not used by lexers.

+ * + * @param recognizer the parser instance + * @param dfa the DFA for the current decision + * @param startIndex the input index where the decision started + * @param stopIndex the input index where the context sensitivity was + * finally determined + * @param prediction the unambiguous result of the full-context prediction + * @param configs the ATN configuration set where the unambiguous prediction + * was determined + */ + void reportContextSensitivity(Parser recognizer, DFA dfa, int startIndex, + int stopIndex, int prediction, ATNConfigSet configs); +} + +class BaseErrorListener extends ErrorListener { + @override + void reportAmbiguity(Parser recognizer, DFA dfa, int startIndex, + int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) {} + + @override + void reportAttemptingFullContext(Parser recognizer, DFA dfa, int startIndex, + int stopIndex, BitSet conflictingAlts, ATNConfigSet configs) {} + + @override + void reportContextSensitivity(Parser recognizer, DFA dfa, int startIndex, + int stopIndex, int prediction, ATNConfigSet configs) {} + + @override + void syntaxError(Recognizer recognizer, Object offendingSymbol, + int line, int charPositionInLine, String msg, RecognitionException e) {} +} + +class ConsoleErrorListener extends BaseErrorListener { + /** + * Provides a default instance of [ConsoleErrorListener]. + */ + static final INSTANCE = ConsoleErrorListener(); + + /** + * {@inheritDoc} + * + *

+ * This implementation prints messages to {@link System//err} containing the + * values of [line], [charPositionInLine], and [msg] using + * the following format.

+ * + * 
+   * line line:charPositionInLine msg
+   *
+ */ + syntaxError(recognizer, offendingSymbol, line, column, msg, e) { + stderr.writeln("line $line:$column $msg"); + } +} + +/** + * This implementation of [ErrorListener] dispatches all calls to a + * collection of delegate listeners. This reduces the effort required to support multiple + * listeners. + */ +class ProxyErrorListener implements ErrorListener { + final List delegates; + + ProxyErrorListener(this.delegates) { + if (delegates == null) { + throw new ArgumentError.notNull("delegates"); + } + } + + void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, + int charPositionInLine, String msg, RecognitionException e) { + for (final listener in delegates) { + listener.syntaxError( + recognizer, offendingSymbol, line, charPositionInLine, msg, e); + } + } + + void reportAmbiguity(Parser recognizer, DFA dfa, int startIndex, + int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { + for (final listener in delegates) { + listener.reportAmbiguity( + recognizer, dfa, startIndex, stopIndex, exact, ambigAlts, configs); + } + } + + void reportAttemptingFullContext(Parser recognizer, DFA dfa, int startIndex, + int stopIndex, BitSet conflictingAlts, ATNConfigSet configs) { + for (final listener in delegates) { + listener.reportAttemptingFullContext( + recognizer, dfa, startIndex, stopIndex, conflictingAlts, configs); + } + } + + void reportContextSensitivity(Parser recognizer, DFA dfa, int startIndex, + int stopIndex, int prediction, ATNConfigSet configs) { + for (final listener in delegates) { + listener.reportContextSensitivity( + recognizer, dfa, startIndex, stopIndex, prediction, configs); + } + } +} diff --git a/runtime/Dart/lib/src/error/src/error_strategy.dart b/runtime/Dart/lib/src/error/src/error_strategy.dart new file mode 100644 index 000000000..c9d06dff7 --- /dev/null +++ b/runtime/Dart/lib/src/error/src/error_strategy.dart @@ -0,0 +1,955 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:developer'; + +import 'package:logging/logging.dart'; + +import '../../atn/atn.dart'; +import '../../interval_set.dart'; +import '../../misc/pair.dart'; +import '../../parser.dart'; +import '../../parser_rule_context.dart'; +import '../../rule_context.dart'; +import '../../token.dart'; +import '../../token_stream.dart'; +import '../../tree/tree.dart'; +import 'errors.dart'; + +/** + * The interface for defining strategies to deal with syntax errors encountered + * during a parse by ANTLR-generated parsers. We distinguish between three + * different kinds of errors: + * + *
    + *
  • The parser could not figure out which path to take in the ATN (none of + * the available alternatives could possibly match)
    • + *
  • The current input does not match what we were looking for
    • + *
  • A predicate evaluated to false
    • + *
+ * + * Implementations of this interface report syntax errors by calling + * {@link Parser#notifyErrorListeners}. + * + *

TODO: what to do about lexers

+ */ +abstract class ErrorStrategy { + /** + * Reset the error handler state for the specified [recognizer]. + * @param recognizer the parser instance + */ + void reset(Parser recognizer); + + /** + * This method is called when an unexpected symbol is encountered during an + * inline match operation, such as {@link Parser#match}. If the error + * strategy successfully recovers from the match failure, this method + * returns the [Token] instance which should be treated as the + * successful result of the match. + * + *

This method handles the consumption of any tokens - the caller should + * not call {@link Parser#consume} after a successful recovery.

+ * + *

Note that the calling code will not report an error if this method + * returns successfully. The error strategy implementation is responsible + * for calling {@link Parser#notifyErrorListeners} as appropriate.

+ * + * @param recognizer the parser instance + * @ if the error strategy was not able to + * recover from the unexpected input symbol + */ + Token recoverInline(Parser recognizer); + + /** + * This method is called to recover from exception [e]. This method is + * called after {@link #reportError} by the default exception handler + * generated for a rule method. + * + * @see #reportError + * + * @param recognizer the parser instance + * @param e the recognition exception to recover from + * @ if the error strategy could not recover from + * the recognition exception + */ + void recover(Parser recognizer, RecognitionException e); + + /** + * This method provides the error handler with an opportunity to handle + * syntactic or semantic errors in the input stream before they result in a + * [RecognitionException]. + * + *

The generated code currently contains calls to {@link #sync} after + * entering the decision state of a closure block ({@code (...)*} or + * {@code (...)+}).

+ * + *

For an implementation based on Jim Idle's "magic sync" mechanism, see + * {@link DefaultErrorStrategy#sync}.

+ * + * @see DefaultErrorStrategy#sync + * + * @param recognizer the parser instance + * @ if an error is detected by the error + * strategy but cannot be automatically recovered at the current state in + * the parsing process + */ + void sync(Parser recognizer); + + /** + * Tests whether or not [recognizer] is in the process of recovering + * from an error. In error recovery mode, {@link Parser#consume} adds + * symbols to the parse tree by calling + * {@link Parser#createErrorNode(ParserRuleContext, Token)} then + * {@link ParserRuleContext#addErrorNode(ErrorNode)} instead of + * {@link Parser#createTerminalNode(ParserRuleContext, Token)}. + * + * @param recognizer the parser instance + * @return [true] if the parser is currently recovering from a parse + * error, otherwise [false] + */ + bool inErrorRecoveryMode(Parser recognizer); + + /** + * This method is called by when the parser successfully matches an input + * symbol. + * + * @param recognizer the parser instance + */ + void reportMatch(Parser recognizer); + + /** + * Report any kind of [RecognitionException]. This method is called by + * the default exception handler generated for a rule method. + * + * @param recognizer the parser instance + * @param e the recognition exception to report + */ + void reportError(Parser recognizer, RecognitionException e); +} + +/** + * This is the default implementation of [ANTLRErrorStrategy] used for + * error reporting and recovery in ANTLR parsers. + */ +class DefaultErrorStrategy implements ErrorStrategy { + /** + * Indicates whether the error strategy is currently "recovering from an + * error". This is used to suppress reporting multiple error messages while + * attempting to recover from a detected syntax error. + * + * @see #inErrorRecoveryMode + */ + bool errorRecoveryMode = false; + + /** The index into the input stream where the last error occurred. + * This is used to prevent infinite loops where an error is found + * but no token is consumed during recovery...another error is found, + * ad nauseum. This is a failsafe mechanism to guarantee that at least + * one token/tree node is consumed for two errors. + */ + int lastErrorIndex = -1; + + IntervalSet lastErrorStates; + + /** + * This field is used to propagate information about the lookahead following + * the previous match. Since prediction prefers completing the current rule + * to error recovery efforts, error reporting may occur later than the + * original point where it was discoverable. The original context is used to + * compute the true expected sets as though the reporting occurred as early + * as possible. + */ + ParserRuleContext nextTokensContext; + + /** + * @see #nextTokensContext + */ + int nextTokensState; + + /** + * {@inheritDoc} + * + *

The default implementation simply calls {@link #endErrorCondition} to + * ensure that the handler is not in error recovery mode.

+ */ + + void reset(Parser recognizer) { + endErrorCondition(recognizer); + } + + /** + * This method is called to enter error recovery mode when a recognition + * exception is reported. + * + * @param recognizer the parser instance + */ + void beginErrorCondition(Parser recognizer) { + errorRecoveryMode = true; + } + + /** + * {@inheritDoc} + */ + + bool inErrorRecoveryMode(Parser recognizer) { + return errorRecoveryMode; + } + + /** + * This method is called to leave error recovery mode after recovering from + * a recognition exception. + * + * @param recognizer + */ + void endErrorCondition(Parser recognizer) { + errorRecoveryMode = false; + lastErrorStates = null; + lastErrorIndex = -1; + } + + /** + * {@inheritDoc} + * + *

The default implementation simply calls {@link #endErrorCondition}.

+ */ + + void reportMatch(Parser recognizer) { + endErrorCondition(recognizer); + } + + /** + * {@inheritDoc} + * + *

The default implementation returns immediately if the handler is already + * in error recovery mode. Otherwise, it calls {@link #beginErrorCondition} + * and dispatches the reporting task based on the runtime type of [e] + * according to the following table.

+ * + *
    + *
  • [NoViableAltException]: Dispatches the call to + * {@link #reportNoViableAlternative}
    • + *
  • [InputMismatchException]: Dispatches the call to + * {@link #reportInputMismatch}
    • + *
  • [FailedPredicateException]: Dispatches the call to + * {@link #reportFailedPredicate}
    • + *
  • All other types: calls {@link Parser#notifyErrorListeners} to report + * the exception
    • + *
+ */ + + void reportError(Parser recognizer, RecognitionException e) { + // if we've already reported an error and have not matched a token + // yet successfully, don't report any errors. + if (inErrorRecoveryMode(recognizer)) { +// System.err.print("[SPURIOUS] "); + return; // don't report spurious errors + } + beginErrorCondition(recognizer); + if (e is NoViableAltException) { + reportNoViableAlternative(recognizer, e); + } else if (e is InputMismatchException) { + reportInputMismatch(recognizer, e); + } else if (e is FailedPredicateException) { + reportFailedPredicate(recognizer, e); + } else { + log("unknown recognition error type: ${e.runtimeType}", + level: Level.SEVERE.value); + recognizer.notifyErrorListeners(e.message, e.offendingToken, e); + } + } + + /** + * {@inheritDoc} + * + *

The default implementation resynchronizes the parser by consuming tokens + * until we find one in the resynchronization set--loosely the set of tokens + * that can follow the current rule.

+ */ + + void recover(Parser recognizer, RecognitionException e) { +// System.out.println("recover in "+recognizer.getRuleInvocationStack()+ +// " index="+recognizer.inputStream.index()+ +// ", lastErrorIndex="+ +// lastErrorIndex+ +// ", states="+lastErrorStates); + if (lastErrorIndex == recognizer.inputStream.index && + lastErrorStates != null && + lastErrorStates.contains(recognizer.state)) { + // uh oh, another error at same token index and previously-visited + // state in ATN; must be a case where LT(1) is in the recovery + // token set so nothing got consumed. Consume a single token + // at least to prevent an infinite loop; this is a failsafe. +// log("seen error condition before index=, level: Level.SEVERE.value"+ +// lastErrorIndex+", states="+lastErrorStates); +// log("FAILSAFE consumes "+recognizer.getTokenNames()[recognizer.inputStream.LA(1)], level: Level.SEVERE.value); + recognizer.consume(); + } + lastErrorIndex = recognizer.inputStream.index; + if (lastErrorStates == null) lastErrorStates = new IntervalSet(); + lastErrorStates.addOne(recognizer.state); + IntervalSet followSet = getErrorRecoverySet(recognizer); + consumeUntil(recognizer, followSet); + } + + /** + * The default implementation of {@link ANTLRErrorStrategy#sync} makes sure + * that the current lookahead symbol is consistent with what were expecting + * at this point in the ATN. You can call this anytime but ANTLR only + * generates code to check before subrules/loops and each iteration. + * + *

Implements Jim Idle's magic sync mechanism in closures and optional + * subrules. E.g.,

+ * + * 
+   * a : sync ( stuff sync )* ;
+   * sync : {consume to what can follow sync} ;
+   *
+ * + * At the start of a sub rule upon error, {@link #sync} performs single + * token deletion, if possible. If it can't do that, it bails on the current + * rule and uses the default error recovery, which consumes until the + * resynchronization set of the current rule. + * + *

If the sub rule is optional ({@code (...)?}, {@code (...)*}, or block + * with an empty alternative), then the expected set includes what follows + * the subrule.

+ * + *

During loop iteration, it consumes until it sees a token that can start a + * sub rule or what follows loop. Yes, that is pretty aggressive. We opt to + * stay in the loop as long as possible.

+ * + *

ORIGINS

+ * + *

Previous versions of ANTLR did a poor job of their recovery within loops. + * A single mismatch token or missing token would force the parser to bail + * out of the entire rules surrounding the loop. So, for rule

+ * + * 
+   * classDef : 'class' ID '{' member* '}'
+   *
+ * + * input with an extra token between members would force the parser to + * consume until it found the next class definition rather than the next + * member definition of the current class. + * + *

This functionality cost a little bit of effort because the parser has to + * compare token set at the start of the loop and at each iteration. If for + * some reason speed is suffering for you, you can turn off this + * functionality by simply overriding this method as a blank { }.

+ */ + + void sync(Parser recognizer) { + ATNState s = recognizer.interpreter.atn.states[recognizer.state]; +// log("sync @ "+s.stateNumber+"="+s.getClass().getSimpleName(), level: Level.SEVERE.value); + // If already recovering, don't try to sync + if (inErrorRecoveryMode(recognizer)) { + return; + } + + TokenStream tokens = recognizer.inputStream; + int la = tokens.LA(1); + + // try cheaper subset first; might get lucky. seems to shave a wee bit off + IntervalSet nextTokens = recognizer.getATN().nextTokens(s); + if (nextTokens.contains(la)) { + // We are sure the token matches + nextTokensContext = null; + nextTokensState = ATNState.INVALID_STATE_NUMBER; + return; + } + + if (nextTokens.contains(Token.EPSILON)) { + if (nextTokensContext == null) { + // It's possible the next token won't match; information tracked + // by sync is restricted for performance. + nextTokensContext = recognizer.context; + nextTokensState = recognizer.state; + } + return; + } + + switch (s.stateType) { + case StateType.BLOCK_START: + case StateType.STAR_BLOCK_START: + case StateType.PLUS_BLOCK_START: + case StateType.STAR_LOOP_ENTRY: + // report error and recover if possible + if (singleTokenDeletion(recognizer) != null) { + return; + } + + throw new InputMismatchException(recognizer); + + case StateType.PLUS_LOOP_BACK: + case StateType.STAR_LOOP_BACK: +// log("at loop back: "+s.getClass().getSimpleName(), level: Level.SEVERE.value); + reportUnwantedToken(recognizer); + IntervalSet expecting = recognizer.expectedTokens; + IntervalSet whatFollowsLoopIterationOrRule = + expecting | getErrorRecoverySet(recognizer); + consumeUntil(recognizer, whatFollowsLoopIterationOrRule); + break; + + default: + // do nothing if we can't identify the exact kind of ATN state + break; + } + } + + /** + * This is called by {@link #reportError} when the exception is a + * [NoViableAltException]. + * + * @see #reportError + * + * @param recognizer the parser instance + * @param e the recognition exception + */ + void reportNoViableAlternative(Parser recognizer, NoViableAltException e) { + TokenStream tokens = recognizer.inputStream; + String input; + if (tokens != null) { + if (e.startToken.type == Token.EOF) + input = ""; + else + input = tokens.getTextRange(e.startToken, e.offendingToken); + } else { + input = ""; + } + String msg = "no viable alternative at input " + escapeWSAndQuote(input); + recognizer.notifyErrorListeners(msg, e.offendingToken, e); + } + + /** + * This is called by {@link #reportError} when the exception is an + * [InputMismatchException]. + * + * @see #reportError + * + * @param recognizer the parser instance + * @param e the recognition exception + */ + void reportInputMismatch(Parser recognizer, InputMismatchException e) { + String msg = "mismatched input " + + getTokenErrorDisplay(e.offendingToken) + + " expecting " + + e.expectedTokens.toString(vocabulary: recognizer.vocabulary); + recognizer.notifyErrorListeners(msg, e.offendingToken, e); + } + + /** + * This is called by {@link #reportError} when the exception is a + * [FailedPredicateException]. + * + * @see #reportError + * + * @param recognizer the parser instance + * @param e the recognition exception + */ + void reportFailedPredicate(Parser recognizer, FailedPredicateException e) { + String ruleName = + recognizer.ruleNames[recognizer.context.ruleIndex]; + String msg = "rule " + ruleName + " " + e.message; + recognizer.notifyErrorListeners(msg, e.offendingToken, e); + } + + /** + * This method is called to report a syntax error which requires the removal + * of a token from the input stream. At the time this method is called, the + * erroneous symbol is current {@code LT(1)} symbol and has not yet been + * removed from the input stream. When this method returns, + * [recognizer] is in error recovery mode. + * + *

This method is called when {@link #singleTokenDeletion} identifies + * single-token deletion as a viable recovery strategy for a mismatched + * input error.

+ * + *

The default implementation simply returns if the handler is already in + * error recovery mode. Otherwise, it calls {@link #beginErrorCondition} to + * enter error recovery mode, followed by calling + * {@link Parser#notifyErrorListeners}.

+ * + * @param recognizer the parser instance + */ + void reportUnwantedToken(Parser recognizer) { + if (inErrorRecoveryMode(recognizer)) { + return; + } + + beginErrorCondition(recognizer); + + Token t = recognizer.currentToken; + String tokenName = getTokenErrorDisplay(t); + IntervalSet expecting = getExpectedTokens(recognizer); + String msg = "extraneous input " + + tokenName + + " expecting " + + expecting.toString(vocabulary: recognizer.vocabulary); + recognizer.notifyErrorListeners(msg, t, null); + } + + /** + * This method is called to report a syntax error which requires the + * insertion of a missing token into the input stream. At the time this + * method is called, the missing token has not yet been inserted. When this + * method returns, [recognizer] is in error recovery mode. + * + *

This method is called when {@link #singleTokenInsertion} identifies + * single-token insertion as a viable recovery strategy for a mismatched + * input error.

+ * + *

The default implementation simply returns if the handler is already in + * error recovery mode. Otherwise, it calls {@link #beginErrorCondition} to + * enter error recovery mode, followed by calling + * {@link Parser#notifyErrorListeners}.

+ * + * @param recognizer the parser instance + */ + void reportMissingToken(Parser recognizer) { + if (inErrorRecoveryMode(recognizer)) { + return; + } + + beginErrorCondition(recognizer); + + Token t = recognizer.currentToken; + IntervalSet expecting = getExpectedTokens(recognizer); + String msg = "missing " + + expecting.toString(vocabulary: recognizer.vocabulary) + + " at " + + getTokenErrorDisplay(t); + + recognizer.notifyErrorListeners(msg, t, null); + } + + /** + * {@inheritDoc} + * + *

The default implementation attempts to recover from the mismatched input + * by using single token insertion and deletion as described below. If the + * recovery attempt fails, this method throws an + * [InputMismatchException].

+ * + *

EXTRA TOKEN (single token deletion)

+ * + *

{@code LA(1)} is not what we are looking for. If {@code LA(2)} has the + * right token, however, then assume {@code LA(1)} is some extra spurious + * token and delete it. Then consume and return the next token (which was + * the {@code LA(2)} token) as the successful result of the match operation.

+ * + *

This recovery strategy is implemented by {@link #singleTokenDeletion}.

+ * + *

MISSING TOKEN (single token insertion)

+ * + *

If current token (at {@code LA(1)}) is consistent with what could come + * after the expected {@code LA(1)} token, then assume the token is missing + * and use the parser's [TokenFactory] to create it on the fly. The + * "insertion" is performed by returning the created token as the successful + * result of the match operation.

+ * + *

This recovery strategy is implemented by {@link #singleTokenInsertion}.

+ * + *

EXAMPLE

+ * + *

For example, Input {@code i=(3;} is clearly missing the {@code ')'}. When + * the parser returns from the nested call to [expr], it will have + * call chain:

+ * + * 
+   * stat → expr → atom
+   *
+ * + * and it will be trying to match the {@code ')'} at this point in the + * derivation: + * + * 
+   * => ID '=' '(' INT ')' ('+' atom)* ';'
+   *                    ^
+   *
+ * + * The attempt to match {@code ')'} will fail when it sees {@code ';'} and + * call {@link #recoverInline}. To recover, it sees that {@code LA(1)==';'} + * is in the set of tokens that can follow the {@code ')'} token reference + * in rule [atom]. It can assume that you forgot the {@code ')'}. + */ + + Token recoverInline(Parser recognizer) { +// SINGLE TOKEN DELETION + Token matchedSymbol = singleTokenDeletion(recognizer); + if (matchedSymbol != null) { +// we have deleted the extra token. +// now, move past ttype token as if all were ok + recognizer.consume(); + return matchedSymbol; + } + +// SINGLE TOKEN INSERTION + if (singleTokenInsertion(recognizer)) { + return getMissingSymbol(recognizer); + } + +// even that didn't work; must throw the exception + InputMismatchException e; + if (nextTokensContext == null) { + e = new InputMismatchException(recognizer); + } else { + e = new InputMismatchException( + recognizer, nextTokensState, nextTokensContext); + } + + throw e; + } + + /** + * This method implements the single-token insertion inline error recovery + * strategy. It is called by {@link #recoverInline} if the single-token + * deletion strategy fails to recover from the mismatched input. If this + * method returns [true], [recognizer] will be in error recovery + * mode. + * + *

This method determines whether or not single-token insertion is viable by + * checking if the {@code LA(1)} input symbol could be successfully matched + * if it were instead the {@code LA(2)} symbol. If this method returns + * [true], the caller is responsible for creating and inserting a + * token with the correct type to produce this behavior.

+ * + * @param recognizer the parser instance + * @return [true] if single-token insertion is a viable recovery + * strategy for the current mismatched input, otherwise [false] + */ + bool singleTokenInsertion(Parser recognizer) { + int currentSymbolType = recognizer.inputStream.LA(1); + // if current token is consistent with what could come after current + // ATN state, then we know we're missing a token; error recovery + // is free to conjure up and insert the missing token + ATNState currentState = + recognizer.interpreter.atn.states[recognizer.state]; + ATNState next = currentState.transition(0).target; + ATN atn = recognizer.interpreter.atn; + IntervalSet expectingAtLL2 = atn.nextTokens(next, recognizer.context); +// System.out.println("LT(2) set="+expectingAtLL2.toString(recognizer.getTokenNames())); + if (expectingAtLL2.contains(currentSymbolType)) { + reportMissingToken(recognizer); + return true; + } + return false; + } + + /** + * This method implements the single-token deletion inline error recovery + * strategy. It is called by {@link #recoverInline} to attempt to recover + * from mismatched input. If this method returns null, the parser and error + * handler state will not have changed. If this method returns non-null, + * [recognizer] will not be in error recovery mode since the + * returned token was a successful match. + * + *

If the single-token deletion is successful, this method calls + * {@link #reportUnwantedToken} to report the error, followed by + * {@link Parser#consume} to actually "delete" the extraneous token. Then, + * before returning {@link #reportMatch} is called to signal a successful + * match.

+ * + * @param recognizer the parser instance + * @return the successfully matched [Token] instance if single-token + * deletion successfully recovers from the mismatched input, otherwise + * null + */ + Token singleTokenDeletion(Parser recognizer) { + int nextTokenType = recognizer.inputStream.LA(2); + IntervalSet expecting = getExpectedTokens(recognizer); + if (expecting.contains(nextTokenType)) { + reportUnwantedToken(recognizer); + /* + log("recoverFromMismatchedToken deleting , level: Level.SEVERE.value"+ + ((TokenStream)recognizer.inputStream).LT(1)+ + " since "+((TokenStream)recognizer.inputStream).LT(2)+ + " is what we want"); + */ + recognizer.consume(); // simply delete extra token + // we want to return the token we're actually matching + Token matchedSymbol = recognizer.currentToken; + reportMatch(recognizer); // we know current token is correct + return matchedSymbol; + } + return null; + } + + /** Conjure up a missing token during error recovery. + * + * The recognizer attempts to recover from single missing + * symbols. But, actions might refer to that missing symbol. + * For example, x=ID {f($x);}. The action clearly assumes + * that there has been an identifier matched previously and that + * $x points at that token. If that token is missing, but + * the next token in the stream is what we want we assume that + * this token is missing and we keep going. Because we + * have to return some token to replace the missing token, + * we have to conjure one up. This method gives the user control + * over the tokens returned for missing tokens. Mostly, + * you will want to create something special for identifier + * tokens. For literals such as '{' and ',', the default + * action in the parser or tree parser works. It simply creates + * a CommonToken of the appropriate type. The text will be the token. + * If you change what tokens must be created by the lexer, + * override this method to create the appropriate tokens. + */ + Token getMissingSymbol(Parser recognizer) { + Token currentSymbol = recognizer.currentToken; + IntervalSet expecting = getExpectedTokens(recognizer); + int expectedTokenType = Token.INVALID_TYPE; + if (!expecting.isNil) { + expectedTokenType = expecting.minElement; // get any element + } + String tokenText; + if (expectedTokenType == Token.EOF) + tokenText = ""; + else + tokenText = ""; + Token current = currentSymbol; + Token lookback = recognizer.inputStream.LT(-1); + if (current.type == Token.EOF && lookback != null) { + current = lookback; + } + return recognizer.tokenFactory.create( + expectedTokenType, + tokenText, + Pair(current.tokenSource, current.tokenSource.inputStream), + Token.DEFAULT_CHANNEL, + -1, + -1, + current.line, + current.charPositionInLine); + } + + IntervalSet getExpectedTokens(Parser recognizer) { + return recognizer.expectedTokens; + } + + /** How should a token be displayed in an error message? The default + * is to display just the text, but during development you might + * want to have a lot of information spit out. Override in that case + * to use t.toString() (which, for CommonToken, dumps everything about + * the token). This is better than forcing you to override a method in + * your token objects because you don't have to go modify your lexer + * so that it creates a new Java type. + */ + String getTokenErrorDisplay(Token t) { + if (t == null) return ""; + String s = getSymbolText(t); + if (s == null) { + if (getSymbolType(t) == Token.EOF) { + s = ""; + } else { + s = "<${getSymbolType(t)}>"; + } + } + return escapeWSAndQuote(s); + } + + String getSymbolText(Token symbol) { + return symbol.text; + } + + int getSymbolType(Token symbol) { + return symbol.type; + } + + String escapeWSAndQuote(String s) { +// if ( s==null ) return s; + s = s.replaceAll("\n", r"\n"); + s = s.replaceAll("\r", r"\r"); + s = s.replaceAll("\t", r"\t"); + return "'" + s + "'"; + } + +/* Compute the error recovery set for the current rule. During + * rule invocation, the parser pushes the set of tokens that can + * follow that rule reference on the stack; this amounts to + * computing FIRST of what follows the rule reference in the + * enclosing rule. See LinearApproximator.FIRST(). + * This local follow set only includes tokens + * from within the rule; i.e., the FIRST computation done by + * ANTLR stops at the end of a rule. + * + * EXAMPLE + * + * When you find a "no viable alt exception", the input is not + * consistent with any of the alternatives for rule r. The best + * thing to do is to consume tokens until you see something that + * can legally follow a call to r *or* any rule that called r. + * You don't want the exact set of viable next tokens because the + * input might just be missing a token--you might consume the + * rest of the input looking for one of the missing tokens. + * + * Consider grammar: + * + * a : '[' b ']' + * | '(' b ')' + * ; + * b : c '^' INT ; + * c : ID + * | INT + * ; + * + * At each rule invocation, the set of tokens that could follow + * that rule is pushed on a stack. Here are the various + * context-sensitive follow sets: + * + * FOLLOW(b1_in_a) = FIRST(']') = ']' + * FOLLOW(b2_in_a) = FIRST(')') = ')' + * FOLLOW(c_in_b) = FIRST('^') = '^' + * + * Upon erroneous input "[]", the call chain is + * + * a -> b -> c + * + * and, hence, the follow context stack is: + * + * depth follow set start of rule execution + * 0 a (from main()) + * 1 ']' b + * 2 '^' c + * + * Notice that ')' is not included, because b would have to have + * been called from a different context in rule a for ')' to be + * included. + * + * For error recovery, we cannot consider FOLLOW(c) + * (context-sensitive or otherwise). We need the combined set of + * all context-sensitive FOLLOW sets--the set of all tokens that + * could follow any reference in the call chain. We need to + * resync to one of those tokens. Note that FOLLOW(c)='^' and if + * we resync'd to that token, we'd consume until EOF. We need to + * sync to context-sensitive FOLLOWs for a, b, and c: {']','^'}. + * In this case, for input "[]", LA(1) is ']' and in the set, so we would + * not consume anything. After printing an error, rule c would + * return normally. Rule b would not find the required '^' though. + * At this point, it gets a mismatched token error and throws an + * exception (since LA(1) is not in the viable following token + * set). The rule exception handler tries to recover, but finds + * the same recovery set and doesn't consume anything. Rule b + * exits normally returning to rule a. Now it finds the ']' (and + * with the successful match exits errorRecovery mode). + * + * So, you can see that the parser walks up the call chain looking + * for the token that was a member of the recovery set. + * + * Errors are not generated in errorRecovery mode. + * + * ANTLR's error recovery mechanism is based upon original ideas: + * + * "Algorithms + Data Structures = Programs" by Niklaus Wirth + * + * and + * + * "A note on error recovery in recursive descent parsers": + * http://portal.acm.org/citation.cfm?id=947902.947905 + * + * Later, Josef Grosch had some good ideas: + * + * "Efficient and Comfortable Error Recovery in Recursive Descent + * Parsers": + * ftp://www.cocolab.com/products/cocktail/doca4.ps/ell.ps.zip + * + * Like Grosch I implement context-sensitive FOLLOW sets that are combined + * at run-time upon error to avoid overhead during parsing. + */ + IntervalSet getErrorRecoverySet(Parser recognizer) { + ATN atn = recognizer.interpreter.atn; + RuleContext ctx = recognizer.context; + IntervalSet recoverSet = new IntervalSet(); + while (ctx != null && ctx.invokingState >= 0) { + // compute what follows who invoked us + ATNState invokingState = atn.states[ctx.invokingState]; + RuleTransition rt = invokingState.transition(0); + IntervalSet follow = atn.nextTokens(rt.followState); + recoverSet.addAll(follow); + ctx = ctx.parent; + } + recoverSet.remove(Token.EPSILON); +// System.out.println("recover set "+recoverSet.toString(recognizer.getTokenNames())); + return recoverSet; + } + + /** Consume tokens until one matches the given token set. */ + void consumeUntil(Parser recognizer, IntervalSet set) { +// log("consumeUntil("+set.toString(recognizer.getTokenNames())+")", level: Level.SEVERE.value); + int ttype = recognizer.inputStream.LA(1); + while (ttype != Token.EOF && !set.contains(ttype)) { + //System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]); +// recognizer.inputStream.consume(); + recognizer.consume(); + ttype = recognizer.inputStream.LA(1); + } + } +} + +/** + * This implementation of [ANTLRErrorStrategy] responds to syntax errors + * by immediately canceling the parse operation with a + * [ParseCancellationException]. The implementation ensures that the + * {@link ParserRuleContext#exception} field is set for all parse tree nodes + * that were not completed prior to encountering the error. + * + *

+ * This error strategy is useful in the following scenarios.

+ * + *
    + *
  • Two-stage parsing: This error strategy allows the first + * stage of two-stage parsing to immediately terminate if an error is + * encountered, and immediately fall back to the second stage. In addition to + * avoiding wasted work by attempting to recover from errors here, the empty + * implementation of {@link BailErrorStrategy#sync} improves the performance of + * the first stage.
    • + *
  • Silent validation: When syntax errors are not being + * reported or logged, and the parse result is simply ignored if errors occur, + * the [BailErrorStrategy] avoids wasting work on recovering from errors + * when the result will be ignored either way.
    • + *
+ * + *

+ * {@code myparser.setErrorHandler(new BailErrorStrategy());}

+ * + * @see Parser#setErrorHandler(ANTLRErrorStrategy) + */ +class BailErrorStrategy extends DefaultErrorStrategy { + /** Instead of recovering from exception [e], re-throw it wrapped + * in a [ParseCancellationException] so it is not caught by the + * rule function catches. Use {@link Exception#getCause()} to get the + * original [RecognitionException]. + */ + + void recover(Parser recognizer, RecognitionException e) { + for (ParserRuleContext context = recognizer.context; + context != null; + context = context.parent) { + context.exception = e; + } + + throw new ParseCancellationException(e.message); + } + + /** Make sure we don't attempt to recover inline; if the parser + * successfully recovers, it won't throw an exception. + */ + + Token recoverInline(Parser recognizer) { + InputMismatchException e = new InputMismatchException(recognizer); + for (ParserRuleContext context = recognizer.context; + context != null; + context = context.parent) { + context.exception = e; + } + + throw new ParseCancellationException(e.message); + } + + /** Make sure we don't attempt to recover from problems in subrules. */ + + void sync(Parser recognizer) {} +} diff --git a/runtime/Dart/lib/src/error/src/errors.dart b/runtime/Dart/lib/src/error/src/errors.dart new file mode 100644 index 000000000..3acde4b22 --- /dev/null +++ b/runtime/Dart/lib/src/error/src/errors.dart @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import '../../atn/atn.dart'; +import '../../input_stream.dart'; +import '../../interval_set.dart'; +import '../../lexer.dart'; +import '../../parser.dart'; +import '../../parser_rule_context.dart'; +import '../../recognizer.dart'; +import '../../rule_context.dart'; +import '../../token.dart'; +import '../../token_stream.dart'; +import '../../util/utils.dart'; + +/** The root of the ANTLR exception hierarchy. In general, ANTLR tracks just + * 3 kinds of errors: prediction errors, failed predicate errors, and + * mismatched input errors. In each case, the parser knows where it is + * in the input, where it is in the ATN, the rule invocation stack, + * and what kind of problem occurred. + */ +class RecognitionException extends StateError { + /** + * Gets the [Recognizer] where this exception occurred. + * + *

If the recognizer is not available, this method returns null.

+ * + * @return The recognizer where this exception occurred, or null if + * the recognizer is not available. + */ + final Recognizer recognizer; + + /** + * Gets the [RuleContext] at the time this exception was thrown. + * + *

If the context is not available, this method returns null.

+ * + * @return The [RuleContext] at the time this exception was thrown. + * If the context is not available, this method returns null. + */ + final RuleContext ctx; + + /** + * Gets the input stream which is the symbol source for the recognizer where + * this exception was thrown. + * + *

If the input stream is not available, this method returns null.

+ * + * @return The input stream which is the symbol source for the recognizer + * where this exception was thrown, or null if the stream is not + * available. + */ + final IntStream inputStream; + + /** + * The current [Token] when an error occurred. Since not all streams + * support accessing symbols by index, we have to track the [Token] + * instance itself. + */ + Token offendingToken; + + /** + * Get the ATN state number the parser was in at the time the error + * occurred. For [NoViableAltException] and + * [LexerNoViableAltException] exceptions, this is the + * [DecisionState] number. For others, it is the state whose outgoing + * edge we couldn't match. + * + *

If the state number is not known, this method returns -1.

+ */ + int offendingState = -1; + + RecognitionException(this.recognizer, this.inputStream, this.ctx, + [String message = ""]) + : super(message) { + if (recognizer != null) this.offendingState = recognizer.state; + } + + /** + * Gets the set of input symbols which could potentially follow the + * previously matched symbol at the time this exception was thrown. + * + *

If the set of expected tokens is not known and could not be computed, + * this method returns null.

+ * + * @return The set of token types that could potentially follow the current + * state in the ATN, or null if the information is not available. + */ + IntervalSet get expectedTokens { + if (recognizer != null) { + return recognizer.getATN().getExpectedTokens(offendingState, ctx); + } + return null; + } +} + +class LexerNoViableAltException extends RecognitionException { + /** Matching attempted at what input index? */ + final int startIndex; + + /** Which configurations did we try at input.index() that couldn't match input.LA(1)? */ + final ATNConfigSet deadEndConfigs; + + LexerNoViableAltException( + Lexer lexer, CharStream input, this.startIndex, this.deadEndConfigs) + : super(lexer, input, null); + + CharStream get inputStream { + return super.inputStream; + } + + String toString() { + String symbol = ""; + if (startIndex >= 0 && startIndex < inputStream.size) { + symbol = inputStream.getText(Interval.of(startIndex, startIndex)); + symbol = escapeWhitespace(symbol); + } + + return "${LexerNoViableAltException}('${symbol}')"; + } +} + +/** Indicates that the parser could not decide which of two or more paths + * to take based upon the remaining input. It tracks the starting token + * of the offending input and also knows where the parser was + * in the various paths when the error. Reported by reportNoViableAlternative() + */ +class NoViableAltException extends RecognitionException { + /** Which configurations did we try at input.index() that couldn't match input.LT(1)? */ + + final ATNConfigSet deadEndConfigs; + + /** The token object at the start index; the input stream might + * not be buffering tokens so get a reference to it. (At the + * time the error occurred, of course the stream needs to keep a + * buffer all of the tokens but later we might not have access to those.) + */ + + final Token startToken; + +// NoViableAltException(Parser recognizer) { // LL(1) error +// this(recognizer, +// recognizer.inputStream, +// recognizer.getCurrentToken(), +// recognizer.getCurrentToken(), +// null, +// recognizer._ctx); +// } + + NoViableAltException._(Parser recognizer, TokenStream input, this.startToken, + Token offendingToken, this.deadEndConfigs, ParserRuleContext ctx) + : super(recognizer, input, ctx) { + this.offendingToken = offendingToken; + } + + NoViableAltException(Parser recognizer, + [TokenStream input, + Token startToken, + Token offendingToken, + ATNConfigSet deadEndConfigs, + ParserRuleContext ctx]) + : this._( + recognizer, + input ?? recognizer.inputStream, + startToken ?? recognizer.currentToken, + offendingToken ?? recognizer.currentToken, + deadEndConfigs ?? null, + ctx ?? recognizer.context); +} + +/** This signifies any kind of mismatched input exceptions such as + * when the current input does not match the expected token. + */ +class InputMismatchException extends RecognitionException { + InputMismatchException(Parser recognizer, + [int state = -1, ParserRuleContext ctx]) + : super(recognizer, recognizer.inputStream, ctx ?? recognizer.context) { + if (state != -1 && ctx != null) { + this.offendingState = state; + } + this.offendingToken = recognizer.currentToken; + } +} + +/** A semantic predicate failed during validation. Validation of predicates + * occurs when normally parsing the alternative just like matching a token. + * Disambiguating predicate evaluation occurs when we test a predicate during + * prediction. + */ +class FailedPredicateException extends RecognitionException { + int ruleIndex; + int predIndex; + final String predicate; + + FailedPredicateException(Parser recognizer, + [this.predicate = null, String message = null]) + : super(recognizer, recognizer.inputStream, recognizer.context, + formatMessage(predicate, message)) { + ATNState s = recognizer.interpreter.atn.states[recognizer.state]; + + AbstractPredicateTransition trans = s.transition(0); + if (trans is PredicateTransition) { + this.ruleIndex = (trans as PredicateTransition).ruleIndex; + this.predIndex = (trans as PredicateTransition).predIndex; + } + this.offendingToken = recognizer.currentToken; + } + + static String formatMessage(String predicate, String message) { + if (message != null) { + return message; + } + + return "failed predicate: {$predicate}?"; + } +} diff --git a/runtime/Dart/lib/src/input_stream.dart b/runtime/Dart/lib/src/input_stream.dart new file mode 100644 index 000000000..580c2a3ee --- /dev/null +++ b/runtime/Dart/lib/src/input_stream.dart @@ -0,0 +1,350 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:math'; + +import 'interval_set.dart'; +import 'token.dart'; + +/** + * A simple stream of symbols whose values are represented as integers. This + * interface provides marked ranges with support for a minimum level + * of buffering necessary to implement arbitrary lookahead during prediction. + * For more information on marked ranges, see {@link #mark}. + * + *

Initializing Methods: Some methods in this interface have + * unspecified behavior if no call to an initializing method has occurred after + * the stream was constructed. The following is a list of initializing methods:

+ * + *
    + *
  • {@link #LA}
    • + *
  • {@link #consume}
    • + *
  • {@link #size}
    • + *
+ */ +abstract class IntStream { + /** + * The value returned by {@link #LA LA()} when the end of the stream is + * reached. + */ + static const int EOF = -1; + + /** + * The value returned by {@link #getSourceName} when the actual name of the + * underlying source is not known. + */ + static const UNKNOWN_SOURCE_NAME = ""; + + /** + * Consumes the current symbol in the stream. This method has the following + * effects: + * + *
    + *
  • Forward movement: The value of {@link #index index()} + * before calling this method is less than the value of {@code index()} + * after calling this method.
    • + *
  • Ordered lookahead: The value of {@code LA(1)} before + * calling this method becomes the value of {@code LA(-1)} after calling + * this method.
    • + *
+ * + * Note that calling this method does not guarantee that {@code index()} is + * incremented by exactly 1, as that would preclude the ability to implement + * filtering streams (e.g. [CommonTokenStream] which distinguishes + * between "on-channel" and "off-channel" tokens). + * + * @throws IllegalStateException if an attempt is made to consume the + * end of the stream (i.e. if {@code LA(1)==}{@link #EOF EOF} before calling + * [consume]). + */ + void consume(); + + /** + * Gets the value of the symbol at offset [i] from the current + * position. When {@code i==1}, this method returns the value of the current + * symbol in the stream (which is the next symbol to be consumed). When + * {@code i==-1}, this method returns the value of the previously read + * symbol in the stream. It is not valid to call this method with + * {@code i==0}, but the specific behavior is unspecified because this + * method is frequently called from performance-critical code. + * + *

This method is guaranteed to succeed if any of the following are true:

+ * + *
    + *
  • {@code i>0}
    • + *
  • {@code i==-1} and {@link #index index()} returns a value greater + * than the value of {@code index()} after the stream was constructed + * and {@code LA(1)} was called in that order. Specifying the current + * {@code index()} relative to the index after the stream was created + * allows for filtering implementations that do not return every symbol + * from the underlying source. Specifying the call to {@code LA(1)} + * allows for lazily initialized streams.
    • + *
  • {@code LA(i)} refers to a symbol consumed within a marked region + * that has not yet been released.
    • + *
+ * + *

If [i] represents a position at or beyond the end of the stream, + * this method returns {@link #EOF}.

+ * + *

The return value is unspecified if {@code i<0} and fewer than {@code -i} + * calls to {@link #consume consume()} have occurred from the beginning of + * the stream before calling this method.

+ * + * @throws UnsupportedOperationException if the stream does not support + * retrieving the value of the specified symbol + */ + int LA(int i); + + /** + * A mark provides a guarantee that {@link #seek seek()} operations will be + * valid over a "marked range" extending from the index where {@code mark()} + * was called to the current {@link #index index()}. This allows the use of + * streaming input sources by specifying the minimum buffering requirements + * to support arbitrary lookahead during prediction. + * + *

The returned mark is an opaque handle (type [int]) which is passed + * to {@link #release release()} when the guarantees provided by the marked + * range are no longer necessary. When calls to + * {@code mark()}/{@code release()} are nested, the marks must be released + * in reverse order of which they were obtained. Since marked regions are + * used during performance-critical sections of prediction, the specific + * behavior of invalid usage is unspecified (i.e. a mark is not released, or + * a mark is released twice, or marks are not released in reverse order from + * which they were created).

+ * + *

The behavior of this method is unspecified if no call to an + * {@link IntStream initializing method} has occurred after this stream was + * constructed.

+ * + *

This method does not change the current position in the input stream.

+ * + *

The following example shows the use of {@link #mark mark()}, + * {@link #release release(mark)}, {@link #index index()}, and + * {@link #seek seek(index)} as part of an operation to safely work within a + * marked region, then restore the stream position to its original value and + * release the mark.

+ * 
+   * IntStream stream = ...;
+   * int index = -1;
+   * int mark = stream.mark();
+   * try {
+   *   index = stream.index();
+   *   // perform work here...
+   * } finally {
+   *   if (index != -1) {
+   *     stream.seek(index);
+   *   }
+   *   stream.release(mark);
+   * }
+   *
+ * + * @return An opaque marker which should be passed to + * {@link #release release()} when the marked range is no longer required. + */ + int mark(); + + /** + * This method releases a marked range created by a call to + * {@link #mark mark()}. Calls to {@code release()} must appear in the + * reverse order of the corresponding calls to {@code mark()}. If a mark is + * released twice, or if marks are not released in reverse order of the + * corresponding calls to {@code mark()}, the behavior is unspecified. + * + *

For more information and an example, see {@link #mark}.

+ * + * @param marker A marker returned by a call to {@code mark()}. + * @see #mark + */ + void release(int marker); + + /** + * Return the index into the stream of the input symbol referred to by + * {@code LA(1)}. + * + *

The behavior of this method is unspecified if no call to an + * {@link IntStream initializing method} has occurred after this stream was + * constructed.

+ */ + int get index; + + /** + * Set the input cursor to the position indicated by [index]. If the + * specified index lies past the end of the stream, the operation behaves as + * though [index] was the index of the EOF symbol. After this method + * returns without throwing an exception, then at least one of the following + * will be true. + * + *
    + *
  • {@link #index index()} will return the index of the first symbol + * appearing at or after the specified [index]. Specifically, + * implementations which filter their sources should automatically + * adjust [index] forward the minimum amount required for the + * operation to target a non-ignored symbol.
    • + *
  • {@code LA(1)} returns {@link #EOF}
    • + *
+ * + * This operation is guaranteed to not throw an exception if [index] + * lies within a marked region. For more information on marked regions, see + * {@link #mark}. The behavior of this method is unspecified if no call to + * an {@link IntStream initializing method} has occurred after this stream + * was constructed. + * + * @param index The absolute index to seek to. + * + * @throws IllegalArgumentException if [index] is less than 0 + * @throws UnsupportedOperationException if the stream does not support + * seeking to the specified index + */ + void seek(int index); + + /** + * Returns the total number of symbols in the stream, including a single EOF + * symbol. + * + * @throws UnsupportedOperationException if the size of the stream is + * unknown. + */ + int get size; + + /** + * Gets the name of the underlying symbol source. This method returns a + * non-null, non-empty string. If such a name is not known, this method + * returns {@link #UNKNOWN_SOURCE_NAME}. + */ + + String get sourceName; +} + +/** A source of characters for an ANTLR lexer. */ +abstract class CharStream extends IntStream { + /** + * This method returns the text for a range of characters within this input + * stream. This method is guaranteed to not throw an exception if the + * specified [interval] lies entirely within a marked range. For more + * information about marked ranges, see {@link IntStream#mark}. + * + * @param interval an interval within the stream + * @return the text of the specified interval + * + * @throws NullPointerException if [interval] is null + * @throws IllegalArgumentException if {@code interval.a < 0}, or if + * {@code interval.b < interval.a - 1}, or if {@code interval.b} lies at or + * past the end of the stream + * @throws UnsupportedOperationException if the stream does not support + * getting the text of the specified interval + */ + String getText(Interval interval); +} + +// Vacuum all input from a string and then treat it like a buffer. +class InputStream extends CharStream { + var name = ""; + List data; + int _index = 0; + bool decodeToUnicodeCodePoints = false; + + InputStream(List data) { + this.data = data; + } + + InputStream.fromString(String data) { + this.data = data.runes.toList(growable: false); + } + + static Future fromStringStream(Stream stream) async { + final data = StringBuffer(); + await stream.listen((buf) { + data.write(buf); + }).asFuture(); + return InputStream.fromString(data.toString()); + } + + static Future fromStream(Stream> stream, + {Encoding encoding = utf8}) { + final data = stream.transform(encoding.decoder); + return fromStringStream(data); + } + + static Future fromPath(String path, {Encoding encoding = utf8}) { + return fromStream(File(path).openRead()); + } + + get index { + return this._index; + } + + get size { + return this.data.length; + } + + /// Reset the stream so that it's in the same state it was + /// when the object was created *except* the data array is not + /// touched. + reset() { + this._index = 0; + } + + consume() { + if (this._index >= this.size) { + // assert this.LA(1) == Token.EOF + throw ("cannot consume EOF"); + } + this._index += 1; + } + + int LA(int offset) { + if (offset == 0) { + return 0; // undefined + } + if (offset < 0) { + offset += 1; // e.g., translate LA(-1) to use offset=0 + } + var pos = this._index + offset - 1; + if (pos < 0 || pos >= this.size) { + // invalid + return Token.EOF; + } + return this.data[pos]; + } + + /// mark/release do nothing; we have entire buffer + int mark() { + return -1; + } + + release(int marker) {} + + /// consume() ahead until p==_index; can't just set p=_index as we must + /// update line and column. If we seek backwards, just set p + seek(int _index) { + if (_index <= this._index) { + this._index = _index; // just jump; don't update stream state (line, + // ...) + return; + } + // seek forward + this._index = min(_index, this.size); + } + + String getText(Interval interval) { + final startIdx = min(interval.a, size); + final len = min(interval.b - interval.a + 1, size - startIdx); + return String.fromCharCodes(this.data, startIdx, startIdx + len); + } + + toString() { + return String.fromCharCodes(this.data); + } + + @override + String get sourceName { + // TODO: implement getSourceName + return IntStream.UNKNOWN_SOURCE_NAME; + } +} diff --git a/runtime/Dart/lib/src/interval_set.dart b/runtime/Dart/lib/src/interval_set.dart new file mode 100644 index 000000000..ae4276563 --- /dev/null +++ b/runtime/Dart/lib/src/interval_set.dart @@ -0,0 +1,750 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:math'; + +import 'package:collection/collection.dart'; + +import 'lexer.dart'; +import 'token.dart'; +import 'util/murmur_hash.dart'; +import 'vocabulary.dart'; + +/** An immutable inclusive interval a..b */ +class Interval { + static final int INTERVAL_POOL_MAX_VALUE = 1000; + + static final Interval INVALID = new Interval(-1, -2); + + static List cache = List(INTERVAL_POOL_MAX_VALUE + 1); + + int a; + int b; + + static int creates = 0; + static int misses = 0; + static int hits = 0; + static int outOfRange = 0; + + Interval(this.a, this.b); + + /** Interval objects are used readonly so share all with the + * same single value a==b up to some max size. Use an array as a perfect hash. + * Return shared object for 0..INTERVAL_POOL_MAX_VALUE or a new + * Interval object with a..a in it. On Java.g4, 218623 IntervalSets + * have a..a (set with 1 element). + */ + static Interval of(int a, int b) { + // cache just a..a + if (a != b || a < 0 || a > INTERVAL_POOL_MAX_VALUE) { + return new Interval(a, b); + } + if (cache[a] == null) { + cache[a] = new Interval(a, a); + } + return cache[a]; + } + + /** return number of elements between a and b inclusively. x..x is length 1. + * if b < a, then length is 0. 9..10 has length 2. + */ + int get length { + if (b < a) return 0; + return b - a + 1; + } + + bool operator ==(Object o) { + if (o == null || !(o is Interval)) { + return false; + } + Interval other = o; + return this.a == other.a && this.b == other.b; + } + + int get hashCode { + int hash = 23; + hash = hash * 31 + a; + hash = hash * 31 + b; + return hash; + } + + /** Does this start completely before other? Disjoint */ + bool startsBeforeDisjoint(Interval other) { + return this.a < other.a && this.b < other.a; + } + + /** Does this start at or before other? Nondisjoint */ + bool startsBeforeNonDisjoint(Interval other) { + return this.a <= other.a && this.b >= other.a; + } + + /** Does this.a start after other.b? May or may not be disjoint */ + bool startsAfter(Interval other) { + return this.a > other.a; + } + + /** Does this start completely after other? Disjoint */ + bool startsAfterDisjoint(Interval other) { + return this.a > other.b; + } + + /** Does this start after other? NonDisjoint */ + bool startsAfterNonDisjoint(Interval other) { + return this.a > other.a && this.a <= other.b; // this.b>=other.b implied + } + + /** Are both ranges disjoint? I.e., no overlap? */ + bool disjoint(Interval other) { + return startsBeforeDisjoint(other) || startsAfterDisjoint(other); + } + + /** Are two intervals adjacent such as 0..41 and 42..42? */ + bool adjacent(Interval other) { + return this.a == other.b + 1 || this.b == other.a - 1; + } + + bool properlyContains(Interval other) { + return other.a >= this.a && other.b <= this.b; + } + + /** Return the interval computed from combining this and other */ + Interval union(Interval other) { + return Interval.of(min(a, other.a), max(b, other.b)); + } + + /** Return the interval in common between this and o */ + Interval intersection(Interval other) { + return Interval.of(max(a, other.a), min(b, other.b)); + } + + /** Return the interval with elements from this not in other; + * other must not be totally enclosed (properly contained) + * within this, which would result in two disjoint intervals + * instead of the single one returned by this method. + */ + Interval differenceNotProperlyContained(Interval other) { + Interval diff = null; + // other.a to left of this.a (or same) + if (other.startsBeforeNonDisjoint(this)) { + diff = Interval.of(max(this.a, other.b + 1), this.b); + } + + // other.a to right of this.a + else if (other.startsAfterNonDisjoint(this)) { + diff = Interval.of(this.a, other.a - 1); + } + return diff; + } + + String toString() { + return "$a..$b"; + } +} + +/** + * This class implements the [IntervalSet] backed by a sorted array of + * non-overlapping intervals. It is particularly efficient for representing + * large collections of numbers, where the majority of elements appear as part + * of a sequential range of numbers that are all part of the set. For example, + * the set { 1, 2, 3, 4, 7, 8 } may be represented as { [1, 4], [7, 8] }. + * + *

+ * This class is able to represent sets containing any combination of values in + * the range {@link int#MIN_VALUE} to {@link int#MAX_VALUE} + * (inclusive).

+ */ +class IntervalSet { + static final IntervalSet COMPLETE_CHAR_SET = + IntervalSet.ofRange(Lexer.MIN_CHAR_VALUE, Lexer.MAX_CHAR_VALUE) + ..setReadonly(true); + + static final IntervalSet EMPTY_SET = new IntervalSet([])..setReadonly(true); + + /** The list of sorted, disjoint intervals. */ + List intervals = []; + + bool readonly = false; + + IntervalSet([List intervals]) { + this.intervals = intervals ?? []; + } + + IntervalSet.ofSet(IntervalSet set) { + addAll(set); + } + +// TODO +// IntervalSet(int... els) { +//if ( els==null ) { +//intervals = new ArrayList(2); // most sets are 1 or 2 elements +//} +//else { +//intervals = new ArrayList(els.length); +//for (int e : els) add(e); +//} +//} + + /** Create a set with a single element, el. */ + + IntervalSet.ofOne(int a) { + addOne(a); + } + + /** Create a set with all ints within range [a..b] (inclusive) */ + static IntervalSet ofRange(int a, int b) { + IntervalSet s = new IntervalSet(); + s.addRange(a, b); + return s; + } + + void clear() { + if (readonly) throw new StateError("can't alter readonly IntervalSet"); + intervals.clear(); + } + + /** Add a single element to the set. An isolated element is stored + * as a range el..el. + */ + + void addOne(int el) { + if (readonly) throw new StateError("can't alter readonly IntervalSet"); + addRange(el, el); + } + + /** Add interval; i.e., add all integers from a to b to set. + * If b<a, do nothing. + * Keep list in sorted order (by left range value). + * If overlap, combine ranges. For example, + * If this is {1..5, 10..20}, adding 6..7 yields + * {1..5, 6..7, 10..20}. Adding 4..8 yields {1..8, 10..20}. + */ + void addRange(int a, int b) { + add(Interval.of(a, b)); + } + + // copy on write so we can cache a..a intervals and sets of that + void add(Interval addition) { + if (readonly) throw new StateError("can't alter readonly IntervalSet"); + //System.out.println("add "+addition+" to "+intervals.toString()); + if (addition.b < addition.a) { + return; + } + for (int i = 0; i < intervals.length; i++) { + Interval r = intervals[i]; + if (addition == r) { + return; + } + if (addition.adjacent(r) || !addition.disjoint(r)) { + // next to each other, make a single larger interval + Interval bigger = addition.union(r); + intervals[i] = bigger; + + // make sure we didn't just create an interval that + // should be merged with next interval in list + for (i++; i < intervals.length; i++) { + Interval next = intervals[i]; + if (!bigger.adjacent(next) && bigger.disjoint(next)) { + break; + } + + // if we bump up against or overlap next, merge + intervals.removeAt(i); // remove this one + intervals[i - 1] = + bigger.union(next); // set previous to 3 merged ones + } + return; + } + if (addition.startsBeforeDisjoint(r)) { + // insert before r + intervals.insert(i, addition); + return; + } + // if disjoint and after r, a future iteration will handle it + + } + // ok, must be after last interval (and disjoint from last interval) + // just add it + intervals.add(addition); + } + + /** combine all sets in the array returned the or'd value */ + static IntervalSet or(List sets) { + IntervalSet r = new IntervalSet(); + for (IntervalSet s in sets) r.addAll(s); + return r; + } + + IntervalSet operator |(IntervalSet a) { + IntervalSet o = new IntervalSet(); + o.addAll(this); + o.addAll(a); + return o; + } + + IntervalSet addAll(IntervalSet set) { + if (set == null) { + return this; + } + + if (set is IntervalSet) { + IntervalSet other = set; + // walk set and add each interval + int n = other.intervals.length; + for (int i = 0; i < n; i++) { + Interval I = other.intervals[i]; + this.addRange(I.a, I.b); + } + } else { + for (int value in set.toList()) { + addOne(value); + } + } + + return this; + } + + IntervalSet complementRange(int minElement, int maxElement) { + return this.complement(IntervalSet.ofRange(minElement, maxElement)); + } + + /** {@inheritDoc} */ + IntervalSet complement(IntervalSet vocabulary) { + if (vocabulary == null || vocabulary.isNil) { + return null; // nothing in common with null set + } + IntervalSet vocabularyIS; + if (vocabulary is IntervalSet) { + vocabularyIS = vocabulary; + } else { + vocabularyIS = new IntervalSet(); + vocabularyIS.addAll(vocabulary); + } + + return vocabularyIS - this; + } + + IntervalSet operator -(IntervalSet a) { + if (a == null || a.isNil) { + return new IntervalSet.ofSet(this); + } + + if (a is IntervalSet) { + return subtract(this, a); + } + + IntervalSet other = new IntervalSet(); + other.addAll(a); + return subtract(this, other); + } + + /** + * Compute the set difference between two interval sets. The specific + * operation is {@code left - right}. If either of the input sets is + * null, it is treated as though it was an empty set. + */ + static IntervalSet subtract(IntervalSet left, IntervalSet right) { + if (left == null || left.isNil) { + return new IntervalSet(); + } + + IntervalSet result = new IntervalSet.ofSet(left); + if (right == null || right.isNil) { + // right set has no elements; just return the copy of the current set + return result; + } + + int resultI = 0; + int rightI = 0; + while ( + resultI < result.intervals.length && rightI < right.intervals.length) { + Interval resultInterval = result.intervals[resultI]; + Interval rightInterval = right.intervals[rightI]; + +// operation: (resultInterval - rightInterval) and update indexes + + if (rightInterval.b < resultInterval.a) { + rightI++; + continue; + } + + if (rightInterval.a > resultInterval.b) { + resultI++; + continue; + } + + Interval beforeCurrent = null; + Interval afterCurrent = null; + if (rightInterval.a > resultInterval.a) { + beforeCurrent = new Interval(resultInterval.a, rightInterval.a - 1); + } + + if (rightInterval.b < resultInterval.b) { + afterCurrent = new Interval(rightInterval.b + 1, resultInterval.b); + } + + if (beforeCurrent != null) { + if (afterCurrent != null) { +// split the current interval into two + result.intervals[resultI] = beforeCurrent; + result.intervals.insert(resultI + 1, afterCurrent); + resultI++; + rightI++; + continue; + } else { +// replace the current interval + result.intervals[resultI] = beforeCurrent; + resultI++; + continue; + } + } else { + if (afterCurrent != null) { +// replace the current interval + result.intervals[resultI] = afterCurrent; + rightI++; + continue; + } else { +// remove the current interval (thus no need to increment resultI) + result.intervals.removeAt(resultI); + continue; + } + } + } + +// If rightI reached right.intervals.length, no more intervals to subtract from result. +// If resultI reached result.intervals.length, we would be subtracting from an empty set. +// Either way, we are done. + return result; + } + + /** {@inheritDoc} */ + IntervalSet operator +(IntervalSet other) { + if (other == null) { + //|| !(other is IntervalSet) ) { + return null; // nothing in common with null set + } + + List myIntervals = this.intervals; + List theirIntervals = (other).intervals; + IntervalSet intersection = null; + int mySize = myIntervals.length; + int theirSize = theirIntervals.length; + int i = 0; + int j = 0; +// iterate down both interval lists looking for nondisjoint intervals + while (i < mySize && j < theirSize) { + Interval mine = myIntervals[i]; + Interval theirs = theirIntervals[j]; +//System.out.println("mine="+mine+" and theirs="+theirs); + if (mine.startsBeforeDisjoint(theirs)) { +// move this iterator looking for interval that might overlap + i++; + } else if (theirs.startsBeforeDisjoint(mine)) { +// move other iterator looking for interval that might overlap + j++; + } else if (mine.properlyContains(theirs)) { +// overlap, add intersection, get next theirs + if (intersection == null) { + intersection = new IntervalSet(); + } + intersection.add(mine.intersection(theirs)); + j++; + } else if (theirs.properlyContains(mine)) { +// overlap, add intersection, get next mine + if (intersection == null) { + intersection = new IntervalSet(); + } + intersection.add(mine.intersection(theirs)); + i++; + } else if (!mine.disjoint(theirs)) { +// overlap, add intersection + if (intersection == null) { + intersection = new IntervalSet(); + } + intersection.add(mine.intersection(theirs)); +// Move the iterator of lower range [a..b], but not +// the upper range as it may contain elements that will collide +// with the next iterator. So, if mine=[0..115] and +// theirs=[115..200], then intersection is 115 and move mine +// but not theirs as theirs may collide with the next range +// in thisIter. +// move both iterators to next ranges + if (mine.startsAfterNonDisjoint(theirs)) { + j++; + } else if (theirs.startsAfterNonDisjoint(mine)) { + i++; + } + } + } + if (intersection == null) { + return new IntervalSet(); + } + return intersection; + } + + /** {@inheritDoc} */ + + bool contains(int el) { + int n = intervals.length; + int l = 0; + int r = n - 1; +// Binary search for the element in the (sorted, +// disjoint) array of intervals. + while (l <= r) { + int m = ((l + r) / 2).floor(); + Interval I = intervals[m]; + int a = I.a; + int b = I.b; + if (b < el) { + l = m + 1; + } else if (a > el) { + r = m - 1; + } else { + // el >= a && el <= b + return true; + } + } + return false; + } + + /** {@inheritDoc} */ + + bool get isNil { + return intervals == null || intervals.isEmpty; + } + + /** + * Returns the maximum value contained in the set if not isNil(). + * + * @return the maximum value contained in the set. + * @throws RuntimeException if set is empty + */ + int get maxElement { + if (isNil) { + throw new StateError("set is empty"); + } + return intervals.last.b; + } + + /** + * Returns the minimum value contained in the set if not isNil(). + * + * @return the minimum value contained in the set. + * @throws RuntimeException if set is empty + */ + int get minElement { + if (isNil) { + throw new StateError("set is empty"); + } + + return intervals.first.a; + } + + int get hashCode { + int hash = MurmurHash.initialize(); + for (Interval I in intervals) { + hash = MurmurHash.update(hash, I.a); + hash = MurmurHash.update(hash, I.b); + } + + hash = MurmurHash.finish(hash, intervals.length * 2); + return hash; + } + + /** Are two IntervalSets equal? Because all intervals are sorted + * and disjoint, equals is a simple linear walk over both lists + * to make sure they are the same. Interval.equals() is used + * by the List.equals() method to check the ranges. + */ + + bool operator ==(Object obj) { + if (obj == null || !(obj is IntervalSet)) { + return false; + } + IntervalSet other = obj; + return ListEquality().equals(this.intervals, other.intervals); + } + + String toString({bool elemAreChar = false, Vocabulary vocabulary}) { + if (this.intervals == null || this.intervals.isEmpty) { + return "{}"; + } + + final elemStr = this.intervals.map((I) { + StringBuffer buf = new StringBuffer(); + int a = I.a; + int b = I.b; + if (a == b) { + if (vocabulary != null) { + buf.write(elementName(vocabulary, a)); + } else { + if (a == Token.EOF) { + buf.write(""); + } else if (elemAreChar) { + buf.write("'"); + buf.writeCharCode(a); + buf.write("'"); + } else { + buf.write(a); + } + } + } else { + if (vocabulary != null) { + for (int i = a; i <= b; i++) { + if (i > a) buf.write(", "); + buf.write(elementName(vocabulary, i)); + } + } else { + if (elemAreChar) { + buf.write("'"); + buf.writeCharCode(a); + buf.write("'..'"); + buf.writeCharCode(b); + buf.write("'"); + } else { + buf.write(a); + buf.write(".."); + buf.write(b); + } + } + } + return buf; + }).join(", "); + if (this.length > 1) { + return "{$elemStr}"; + } + return elemStr; + } + + String elementName(Vocabulary vocabulary, int a) { + if (a == Token.EOF) { + return ""; + } else if (a == Token.EPSILON) { + return ""; + } else { + return vocabulary.getDisplayName(a); + } + } + + int get length { + int n = 0; + int numIntervals = intervals.length; + if (numIntervals == 1) { + Interval firstInterval = this.intervals[0]; + return firstInterval.b - firstInterval.a + 1; + } + for (int i = 0; i < numIntervals; i++) { + Interval I = intervals[i]; + n += (I.b - I.a + 1); + } + return n; + } + + List toIntegerList() { + List values = new List(length); + int n = intervals.length; + for (int i = 0; i < n; i++) { + Interval I = intervals[i]; + int a = I.a; + int b = I.b; + for (int v = a; v <= b; v++) { + values.add(v); + } + } + return values; + } + + List toList() { + List values = []; + int n = intervals.length; + for (int i = 0; i < n; i++) { + Interval I = intervals[i]; + int a = I.a; + int b = I.b; + for (int v = a; v <= b; v++) { + values.add(v); + } + } + return values; + } + + Set toSet() { + Set s = new Set(); + for (Interval I in intervals) { + int a = I.a; + int b = I.b; + for (int v = a; v <= b; v++) { + s.add(v); + } + } + return s; + } + + /** Get the ith element of ordered set. Used only by RandomPhrase so + * don't bother to implement if you're not doing that for a new + * ANTLR code gen target. + */ + int get(int i) { + int n = intervals.length; + int index = 0; + for (int j = 0; j < n; j++) { + Interval I = intervals[j]; + int a = I.a; + int b = I.b; + for (int v = a; v <= b; v++) { + if (index == i) { + return v; + } + index++; + } + } + return -1; + } + + void remove(int el) { + if (readonly) throw new StateError("can't alter readonly IntervalSet"); + int n = intervals.length; + for (int i = 0; i < n; i++) { + Interval I = intervals[i]; + int a = I.a; + int b = I.b; + if (el < a) { + break; // list is sorted and el is before this interval; not here + } +// if whole interval x..x, rm + if (el == a && el == b) { + intervals.removeAt(i); + break; + } +// if on left edge x..b, adjust left + if (el == a) { + I.a++; + break; + } +// if on right edge a..x, adjust right + if (el == b) { + I.b--; + break; + } +// if in middle a..x..b, split interval + if (el > a && el < b) { + // found in this interval + int oldb = I.b; + I.b = el - 1; // [a..x-1] + addRange(el + 1, oldb); // add [x+1..b] + } + } + } + + bool isReadonly() { + return readonly; + } + + void setReadonly(bool readonly) { + if (this.readonly && !readonly) + throw new StateError("can't alter readonly IntervalSet"); + this.readonly = readonly; + } +} diff --git a/runtime/Dart/lib/src/lexer.dart b/runtime/Dart/lib/src/lexer.dart new file mode 100644 index 000000000..b8c61ad00 --- /dev/null +++ b/runtime/Dart/lib/src/lexer.dart @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:developer'; + +import 'atn/atn.dart'; +import 'error/error.dart'; +import 'input_stream.dart'; +import 'interval_set.dart'; +import 'misc/pair.dart'; +import 'recognizer.dart'; +import 'token.dart'; +import 'token_factory.dart'; +import 'token_source.dart'; +import 'util/utils.dart'; + +abstract class Lexer extends Recognizer + implements TokenSource { + static final DEFAULT_MODE = 0; + static final MORE = -2; + static final SKIP = -3; + + static final DEFAULT_TOKEN_CHANNEL = Token.DEFAULT_CHANNEL; + static final HIDDEN = Token.HIDDEN_CHANNEL; + static final MIN_CHAR_VALUE = 0x0000; + static final MAX_CHAR_VALUE = 0x10FFFF; + + CharStream _input; + + Pair _tokenFactorySourcePair; + TokenFactory tokenFactory = CommonTokenFactory.DEFAULT; + + // The goal of all lexer rules/methods is to create a token object. + // this is an instance variable as multiple rules may collaborate to + // create a single token. nextToken will return this object after + // matching lexer rule(s). If you subclass to allow multiple token + // emissions, then set this to the last token to be matched or + // something nonnull so that the auto token emit mechanism will not + // emit another token. + Token _token = null; + + // What character index in the stream did the current token start at? + // Needed, for example, to get the text for current token. Set at + // the start of nextToken. + int tokenStartCharIndex = -1; + + // The line on which the first character of the token resides/// + int tokenStartLine = -1; + + // The character position of first character within the line/// + int tokenStartCharPositionInLine = -1; + + // Once we see EOF on char stream, next token will be EOF. + // If you have DONE : EOF ; then you see DONE EOF. + bool _hitEOF = false; + + // The channel number for the current token/// + int channel = Token.DEFAULT_CHANNEL; + + // The token type for the current token/// + int type = Token.INVALID_TYPE; + + List _modeStack = []; + int mode_ = Lexer.DEFAULT_MODE; + + /// You can set the text for the current token to override what is in + /// the input char buffer. Use setText() or can set this instance var. + String _text = null; + + Lexer(CharStream input) { + this._input = input; + this._tokenFactorySourcePair = Pair(this, input); + } + + reset() { + // wack Lexer state variables + if (_input != null) { + _input.seek(0); // rewind the input + } + _token = null; + type = Token.INVALID_TYPE; + channel = Token.DEFAULT_CHANNEL; + tokenStartCharIndex = -1; + tokenStartCharPositionInLine = -1; + tokenStartLine = -1; + _text = null; + + _hitEOF = false; + mode_ = Lexer.DEFAULT_MODE; + _modeStack.clear(); + + interpreter.reset(); + } + + /// Return a token from this source; i.e., match a token on the char stream. + Token nextToken() { + if (_input == null) { + throw new StateError("nextToken requires a non-null input stream."); + } + + // Mark start location in char stream so unbuffered streams are + // guaranteed at least have text of current token + int tokenStartMarker = _input.mark(); + try { + outer: + while (true) { + if (_hitEOF) { + emitEOF(); + return _token; + } + + _token = null; + channel = Token.DEFAULT_CHANNEL; + tokenStartCharIndex = _input.index; + tokenStartCharPositionInLine = interpreter.charPositionInLine; + tokenStartLine = interpreter.line; + _text = null; + do { + type = Token.INVALID_TYPE; +// System.out.println("nextToken line "+tokenStartLine+" at "+((char)input.LA(1))+ +// " in mode "+mode+ +// " at index "+input.index()); + int ttype; + try { + ttype = interpreter.match(_input, mode_); + } on LexerNoViableAltException catch (e) { + notifyListeners(e); // report error + recover(e); + ttype = SKIP; + } + if (_input.LA(1) == IntStream.EOF) { + _hitEOF = true; + } + if (type == Token.INVALID_TYPE) type = ttype; + if (type == SKIP) { + continue outer; + } + } while (type == MORE); + if (_token == null) emit(); + return _token; + } + } finally { + // make sure we release marker after match or + // unbuffered char stream will keep buffering + _input.release(tokenStartMarker); + } + } + + /// Instruct the lexer to skip creating a token for current lexer rule + /// and look for another token. nextToken() knows to keep looking when + /// a lexer rule finishes with token set to SKIP_TOKEN. Recall that + /// if token==null at end of any token rule, it creates one for you + /// and emits it. + skip() { + this.type = Lexer.SKIP; + } + + more() { + this.type = Lexer.MORE; + } + + mode(int m) { + this.mode_ = m; + } + + pushMode(int m) { + if (LexerATNSimulator.debug) { + log("pushMode $m"); + } + _modeStack.add(mode_); + mode(m); + } + + int popMode() { + if (_modeStack.isEmpty) throw new StateError(""); + if (LexerATNSimulator.debug) log("popMode back to ${_modeStack.last}"); + mode(_modeStack.removeLast()); + return mode_; + } + + /** Set the char stream and reset the lexer */ + void set inputStream(IntStream input) { + this._input = null; + this._tokenFactorySourcePair = + new Pair(this, _input); + reset(); + this._input = input; + this._tokenFactorySourcePair = + new Pair(this, _input); + } + + String get sourceName { + return _input.sourceName; + } + + CharStream get inputStream { + return _input; + } + + /** By default does not support multiple emits per nextToken invocation + * for efficiency reasons. Subclass and override this method, nextToken, + * and getToken (to push tokens into a list and pull from that list + * rather than a single variable as this implementation does). + */ + void emitToken(Token token) { + //System.err.println("emit "+token); + this._token = token; + } + + /** The standard method called to automatically emit a token at the + * outermost lexical rule. The token object should point into the + * char buffer start..stop. If there is a text override in 'text', + * use that to set the token's text. Override this method to emit + * custom Token objects or provide a new factory. + */ + Token emit() { + Token t = tokenFactory.create( + type, + _text, + _tokenFactorySourcePair, + channel, + tokenStartCharIndex, + charIndex - 1, + tokenStartLine, + tokenStartCharPositionInLine); + emitToken(t); + return t; + } + + Token emitEOF() { + int cpos = charPositionInLine; + Token eof = tokenFactory.create(Token.EOF, null, _tokenFactorySourcePair, + Token.DEFAULT_CHANNEL, _input.index, _input.index - 1, line, cpos); + emitToken(eof); + return eof; + } + + int get charPositionInLine { + return interpreter.charPositionInLine; + } + + int get line { + return interpreter.line; + } + + void set line(int line) { + interpreter.line = line; + } + + void set charPositionInLine(int charPositionInLine) { + interpreter.charPositionInLine = charPositionInLine; + } + + /** What is the index of the current character of lookahead? */ + int get charIndex { + return _input.index; + } + + /** Return the text matched so far for the current token or any + * text override. + */ + String get text { + if (_text != null) { + return _text; + } + return interpreter.getText(_input); + } + + /** Set the complete text of this token; it wipes any previous + * changes to the text. + */ + void set text(String text) { + this._text = text; + } + + /** Override if emitting multiple tokens. */ + Token get token { + return _token; + } + + void setToken(Token _token) { + this._token = _token; + } + + List get channelNames => null; + + List get modeNames => null; + + /** Return a list of all Token objects in input char stream. + * Forces load of all tokens. Does not include EOF token. + */ + List get allTokens { + List tokens = []; + Token t = nextToken(); + while (t.type != Token.EOF) { + tokens.add(t); + t = nextToken(); + } + return tokens; + } + + void notifyListeners(LexerNoViableAltException e) { + String text = + _input.getText(Interval.of(tokenStartCharIndex, _input.index)); + String msg = "token recognition error at: '" + getErrorDisplay(text) + "'"; + + ErrorListener listener = errorListenerDispatch; + listener.syntaxError( + this, null, tokenStartLine, tokenStartCharPositionInLine, msg, e); + } + + String getErrorDisplay(String s) { + return escapeWhitespace(s); + } + + String getCharErrorDisplay(int c) { + String s = getErrorDisplay(String.fromCharCode(c)); + return "'$s'"; + } + + /** Lexers can normally match any char in it's vocabulary after matching + * a token, so do the easy thing and just kill a character and hope + * it all works out. You can instead use the rule invocation stack + * to do sophisticated error recovery if you are in a fragment rule. + */ + void recover(RecognitionException re) { + if (re is LexerNoViableAltException) { + if (_input.LA(1) != IntStream.EOF) { + // skip a char and try again + interpreter.consume(_input); + } + } else { + //System.out.println("consuming char "+(char)input.LA(1)+" during recovery"); + //re.printStackTrace(); + // TODO: Do we lose character or line position information? + _input.consume(); + } + } +} diff --git a/runtime/Dart/lib/src/ll1_analyzer.dart b/runtime/Dart/lib/src/ll1_analyzer.dart new file mode 100644 index 000000000..45646299d --- /dev/null +++ b/runtime/Dart/lib/src/ll1_analyzer.dart @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import './util/bit_set.dart'; +import 'atn/atn.dart'; +import 'interval_set.dart'; +import 'prediction_context.dart'; +import 'rule_context.dart'; +import 'token.dart'; +import 'util/bit_set.dart'; + +class LL1Analyzer { + /** Special value added to the lookahead sets to indicate that we hit + * a predicate during analysis if {@code seeThruPreds==false}. + */ + static final int HIT_PRED = Token.INVALID_TYPE; + + final ATN atn; + + LL1Analyzer(this.atn); + + /** + * Calculates the SLL(1) expected lookahead set for each outgoing transition + * of an [ATNState]. The returned array has one element for each + * outgoing transition in [s]. If the closure from transition + * i leads to a semantic predicate before matching a symbol, the + * element at index i of the result will be null. + * + * @param s the ATN state + * @return the expected symbols for each outgoing transition of [s]. + */ + List getDecisionLookahead(ATNState s) { +// System.out.println("LOOK("+s.stateNumber+")"); + if (s == null) { + return null; + } + + List look = List(s.numberOfTransitions); + for (int alt = 0; alt < s.numberOfTransitions; alt++) { + look[alt] = new IntervalSet(); + Set lookBusy = Set(); + bool seeThruPreds = false; // fail to get lookahead upon pred + _LOOK(s.transition(alt).target, null, PredictionContext.EMPTY, look[alt], + lookBusy, new BitSet(), seeThruPreds, false); + // Wipe out lookahead for this alternative if we found nothing + // or we had a predicate when we !seeThruPreds + if (look[alt].length == 0 || look[alt].contains(HIT_PRED)) { + look[alt] = null; + } + } + return look; + } + + /** + * Compute set of tokens that can follow [s] in the ATN in the + * specified [ctx]. + * + *

If [ctx] is null and the end of the rule containing + * [s] is reached, {@link Token#EPSILON} is added to the result set. + * If [ctx] is not null and the end of the outermost rule is + * reached, {@link Token#EOF} is added to the result set.

+ * + * @param s the ATN state + * @param stopState the ATN state to stop at. This can be a + * [BlockEndState] to detect epsilon paths through a closure. + * @param ctx the complete parser context, or null if the context + * should be ignored + * + * @return The set of tokens that can follow [s] in the ATN in the + * specified [ctx]. + */ + + IntervalSet LOOK(ATNState s, RuleContext ctx, [ATNState stopState = null]) { + IntervalSet r = new IntervalSet(); + bool seeThruPreds = true; // ignore preds; get all lookahead + PredictionContext lookContext = + ctx != null ? PredictionContext.fromRuleContext(s.atn, ctx) : null; + _LOOK( + s, stopState, lookContext, r, Set(), new BitSet(), seeThruPreds, true); + return r; + } + + /** + * Compute set of tokens that can follow [s] in the ATN in the + * specified [ctx]. + * + *

If [ctx] is null and [stopState] or the end of the + * rule containing [s] is reached, {@link Token#EPSILON} is added to + * the result set. If [ctx] is not null and [addEOF] is + * [true] and [stopState] or the end of the outermost rule is + * reached, {@link Token#EOF} is added to the result set.

+ * + * @param s the ATN state. + * @param stopState the ATN state to stop at. This can be a + * [BlockEndState] to detect epsilon paths through a closure. + * @param ctx The outer context, or null if the outer context should + * not be used. + * @param look The result lookahead set. + * @param lookBusy A set used for preventing epsilon closures in the ATN + * from causing a stack overflow. Outside code should pass + * {@code new HashSet} for this argument. + * @param calledRuleStack A set used for preventing left recursion in the + * ATN from causing a stack overflow. Outside code should pass + * {@code new BitSet()} for this argument. + * @param seeThruPreds [true] to true semantic predicates as + * implicitly [true] and "see through them", otherwise [false] + * to treat semantic predicates as opaque and add {@link #HIT_PRED} to the + * result if one is encountered. + * @param addEOF Add {@link Token#EOF} to the result if the end of the + * outermost context is reached. This parameter has no effect if [ctx] + * is null. + */ + void _LOOK( + ATNState s, + ATNState stopState, + PredictionContext ctx, + IntervalSet look, + Set lookBusy, + BitSet calledRuleStack, + bool seeThruPreds, + bool addEOF) { +// System.out.println("_LOOK("+s.stateNumber+", ctx="+ctx); + ATNConfig c = new ATNConfig(s, 0, ctx); + if (!lookBusy.add(c)) return; + + if (s == stopState) { + if (ctx == null) { + look.addOne(Token.EPSILON); + return; + } else if (ctx.isEmpty && addEOF) { + look.addOne(Token.EOF); + return; + } + } + + if (s is RuleStopState) { + if (ctx == null) { + look.addOne(Token.EPSILON); + return; + } else if (ctx.isEmpty && addEOF) { + look.addOne(Token.EOF); + return; + } + + if (ctx != PredictionContext.EMPTY) { + // run thru all possible stack tops in ctx + bool removed = calledRuleStack[s.ruleIndex]; + try { + calledRuleStack.clear(s.ruleIndex); + for (int i = 0; i < ctx.length; i++) { + ATNState returnState = atn.states[ctx.getReturnState(i)]; +// System.out.println("popping back to "+retState); + _LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, + calledRuleStack, seeThruPreds, addEOF); + } + } finally { + if (removed) { + calledRuleStack.set(s.ruleIndex); + } + } + return; + } + } + + for (int i = 0; i < s.numberOfTransitions; i++) { + Transition t = s.transition(i); + if (t is RuleTransition) { + if (calledRuleStack[t.target.ruleIndex]) { + continue; + } + + PredictionContext newContext = + SingletonPredictionContext.create(ctx, t.followState.stateNumber); + + try { + calledRuleStack.set(t.target.ruleIndex); + _LOOK(t.target, stopState, newContext, look, lookBusy, + calledRuleStack, seeThruPreds, addEOF); + } finally { + calledRuleStack.clear(t.target.ruleIndex); + } + } else if (t is AbstractPredicateTransition) { + if (seeThruPreds) { + _LOOK(t.target, stopState, ctx, look, lookBusy, calledRuleStack, + seeThruPreds, addEOF); + } else { + look.addOne(HIT_PRED); + } + } else if (t.isEpsilon) { + _LOOK(t.target, stopState, ctx, look, lookBusy, calledRuleStack, + seeThruPreds, addEOF); + } else if (t is WildcardTransition) { + look.addAll( + IntervalSet.ofRange(Token.MIN_USER_TOKEN_TYPE, atn.maxTokenType)); + } else { +// System.out.println("adding "+ t); + IntervalSet set = t.label; + if (set != null) { + if (t is NotSetTransition) { + set = set.complement(IntervalSet.ofRange( + Token.MIN_USER_TOKEN_TYPE, atn.maxTokenType)); + } + look.addAll(set); + } + } + } + } +} diff --git a/runtime/Dart/lib/src/misc/multi_map.dart b/runtime/Dart/lib/src/misc/multi_map.dart new file mode 100644 index 000000000..27bcfdca4 --- /dev/null +++ b/runtime/Dart/lib/src/misc/multi_map.dart @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:collection'; + +import 'package:collection/collection.dart'; + +import 'pair.dart'; + +class MultiMap extends DelegatingMap> { + MultiMap() : super(LinkedHashMap()); + + void put(K key, V value) { + List elementsForKey = this[key]; + if (elementsForKey == null) { + elementsForKey = []; + this[key] = elementsForKey; + } + elementsForKey.add(value); + } + + List> get pairs { + List> pairs = []; + for (K key in keys) { + for (V value in this[key]) { + pairs.add(new Pair(key, value)); + } + } + return pairs; + } +} diff --git a/runtime/Dart/lib/src/misc/pair.dart b/runtime/Dart/lib/src/misc/pair.dart new file mode 100644 index 000000000..7041da97b --- /dev/null +++ b/runtime/Dart/lib/src/misc/pair.dart @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +class Pair { + final A a; + final B b; + + const Pair(this.a, this.b); + + @override + bool operator ==(other) { + return other is Pair && a == other.a && b == other.b; + } + + String toString() { + return "($a, $b)"; + } + + @override + int get hashCode { + return a.hashCode ^ b.hashCode; + } +} diff --git a/runtime/Dart/lib/src/parser.dart b/runtime/Dart/lib/src/parser.dart new file mode 100644 index 000000000..584c23144 --- /dev/null +++ b/runtime/Dart/lib/src/parser.dart @@ -0,0 +1,839 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:developer'; +import 'dart:io'; + +import 'atn/atn.dart'; +import 'dfa/dfa.dart'; +import 'error/error.dart'; +import 'input_stream.dart'; +import 'interval_set.dart'; +import 'lexer.dart'; +import 'parser_rule_context.dart'; +import 'recognizer.dart'; +import 'rule_context.dart'; +import 'token.dart'; +import 'token_factory.dart'; +import 'token_source.dart'; +import 'token_stream.dart'; +import 'tree/tree.dart'; + +/** This is all the parsing support code essentially; most of it is error recovery stuff. */ +abstract class Parser extends Recognizer { + /** + * This field maps from the serialized ATN string to the deserialized [ATN] with + * bypass alternatives. + * + * @see ATNDeserializationOptions#isGenerateRuleBypassTransitions() + */ + static final Map bypassAltsAtnCache = {}; + + /** + * The error handling strategy for the parser. The default value is a new + * instance of [DefaultErrorStrategy]. + * + * @see #getErrorHandler + * @see #setErrorHandler + */ + + ErrorStrategy errorHandler = new DefaultErrorStrategy(); + + /** + * The input stream. + * + * @see #getInputStream + * @see #setInputStream + */ + TokenStream _input; + + final List _precedenceStack = [0]; + + /** + * The [ParserRuleContext] object for the currently executing rule. + * This is always non-null during the parsing process. + */ + ParserRuleContext _ctx; + + /** + * Specifies whether or not the parser should construct a parse tree during + * the parsing process. The default value is [true]. + * + * @see #getBuildParseTree + * @see #setBuildParseTree + */ + bool _buildParseTrees = true; + + /** + * When {@link #setTrace}{@code (true)} is called, a reference to the + * [TraceListener] is stored here so it can be easily removed in a + * later call to {@link #setTrace}{@code (false)}. The listener itself is + * implemented as a parser listener so this field is not directly used by + * other parser methods. + */ + TraceListener _tracer; + + /** + * The list of [ParseTreeListener] listeners registered to receive + * events during the parse. + * + * @see #addParseListener + */ + List _parseListeners; + + /** + * The number of syntax errors reported during parsing. This value is + * incremented each time {@link #notifyErrorListeners} is called. + */ + int _syntaxErrors = 0; + + /** Indicates parser has match()ed EOF token. See {@link #exitRule()}. */ + bool matchedEOF = false; + + Parser(TokenStream input) { + inputStream = input; + } + + /** reset the parser's state */ + void reset() { + if (inputStream != null) inputStream.seek(0); + errorHandler.reset(this); + _ctx = null; + _syntaxErrors = 0; + matchedEOF = false; + setTrace(false); + _precedenceStack.clear(); + _precedenceStack.add(0); + if (interpreter != null) { + interpreter.reset(); + } + } + + /** + * Match current input symbol against [ttype]. If the symbol type + * matches, {@link ANTLRErrorStrategy#reportMatch} and {@link #consume} are + * called to complete the match process. + * + *

If the symbol type does not match, + * {@link ANTLRErrorStrategy#recoverInline} is called on the current error + * strategy to attempt recovery. If {@link #getBuildParseTree} is + * [true] and the token index of the symbol returned by + * {@link ANTLRErrorStrategy#recoverInline} is -1, the symbol is added to + * the parse tree by calling {@link #createErrorNode(ParserRuleContext, Token)} then + * {@link ParserRuleContext#addErrorNode(ErrorNode)}.

+ * + * @param ttype the token type to match + * @return the matched symbol + * @throws RecognitionException if the current input symbol did not match + * [ttype] and the error strategy could not recover from the + * mismatched symbol + */ + Token match(int ttype) { + Token t = currentToken; + if (t.type == ttype) { + if (ttype == Token.EOF) { + matchedEOF = true; + } + errorHandler.reportMatch(this); + consume(); + } else { + t = errorHandler.recoverInline(this); + if (_buildParseTrees && t.tokenIndex == -1) { + // we must have conjured up a new token during single token insertion + // if it's not the current symbol + _ctx.addErrorNode(createErrorNode(_ctx, t)); + } + } + return t; + } + + /** + * Match current input symbol as a wildcard. If the symbol type matches + * (i.e. has a value greater than 0), {@link ANTLRErrorStrategy#reportMatch} + * and {@link #consume} are called to complete the match process. + * + *

If the symbol type does not match, + * {@link ANTLRErrorStrategy#recoverInline} is called on the current error + * strategy to attempt recovery. If {@link #getBuildParseTree} is + * [true] and the token index of the symbol returned by + * {@link ANTLRErrorStrategy#recoverInline} is -1, the symbol is added to + * the parse tree by calling {@link Parser#createErrorNode(ParserRuleContext, Token)}. then + * {@link ParserRuleContext#addErrorNode(ErrorNode)}

+ * + * @return the matched symbol + * @throws RecognitionException if the current input symbol did not match + * a wildcard and the error strategy could not recover from the mismatched + * symbol + */ + Token matchWildcard() { + Token t = currentToken; + if (t.type > 0) { + errorHandler.reportMatch(this); + consume(); + } else { + t = errorHandler.recoverInline(this); + if (_buildParseTrees && t.tokenIndex == -1) { + // we must have conjured up a new token during single token insertion + // if it's not the current symbol + _ctx.addErrorNode(createErrorNode(_ctx, t)); + } + } + + return t; + } + + /** + * Track the [ParserRuleContext] objects during the parse and hook + * them up using the {@link ParserRuleContext#children} list so that it + * forms a parse tree. The [ParserRuleContext] returned from the start + * rule represents the root of the parse tree. + * + *

Note that if we are not building parse trees, rule contexts only point + * upwards. When a rule exits, it returns the context but that gets garbage + * collected if nobody holds a reference. It points upwards but nobody + * points at it.

+ * + *

When we build parse trees, we are adding all of these contexts to + * {@link ParserRuleContext#children} list. Contexts are then not candidates + * for garbage collection.

+ */ + void set buildParseTree(bool buildParseTrees) { + this._buildParseTrees = buildParseTrees; + } + + /** + * Gets whether or not a complete parse tree will be constructed while + * parsing. This property is [true] for a newly constructed parser. + * + * @return [true] if a complete parse tree will be constructed while + * parsing, otherwise [false] + */ + bool get buildParseTree { + return _buildParseTrees; + } + + /** + * Trim the internal lists of the parse tree during parsing to conserve memory. + * This property is set to [false] by default for a newly constructed parser. + * + * @param trimParseTrees [true] to trim the capacity of the {@link ParserRuleContext#children} + * list to its size after a rule is parsed. + */ + void set trimParseTree(bool trimParseTrees) { + if (trimParseTrees) { + if (trimParseTree) return; + addParseListener(TrimToSizeListener.INSTANCE); + } else { + removeParseListener(TrimToSizeListener.INSTANCE); + } + } + + /** + * @return [true] if the {@link ParserRuleContext#children} list is trimmed + * using the default {@link Parser.TrimToSizeListener} during the parse process. + */ + bool get trimParseTree { + return parseListeners.contains(TrimToSizeListener.INSTANCE); + } + + List get parseListeners => _parseListeners; + + /** + * Registers [listener] to receive events during the parsing process. + * + *

To support output-preserving grammar transformations (including but not + * limited to left-recursion removal, automated left-factoring, and + * optimized code generation), calls to listener methods during the parse + * may differ substantially from calls made by + * {@link ParseTreeWalker#DEFAULT} used after the parse is complete. In + * particular, rule entry and exit events may occur in a different order + * during the parse than after the parser. In addition, calls to certain + * rule entry methods may be omitted.

+ * + *

With the following specific exceptions, calls to listener events are + * deterministic, i.e. for identical input the calls to listener + * methods will be the same.

+ * + *
    + *
  • Alterations to the grammar used to generate code may change the + * behavior of the listener calls.
    • + *
  • Alterations to the command line options passed to ANTLR 4 when + * generating the parser may change the behavior of the listener calls.
    • + *
  • Changing the version of the ANTLR Tool used to generate the parser + * may change the behavior of the listener calls.
    • + *
+ * + * @param listener the listener to add + * + * @throws NullPointerException if {@code} listener is null + */ + void addParseListener(ParseTreeListener listener) { + if (listener == null) { + throw new ArgumentError.notNull("listener"); + } + + if (_parseListeners == null) { + _parseListeners = []; + } + + this._parseListeners.add(listener); + } + + /** + * Remove [listener] from the list of parse listeners. + * + *

If [listener] is null or has not been added as a parse + * listener, this method does nothing.

+ * + * @see #addParseListener + * + * @param listener the listener to remove + */ + void removeParseListener(ParseTreeListener listener) { + if (_parseListeners != null) { + if (_parseListeners.remove(listener)) { + if (_parseListeners.isEmpty) { + _parseListeners = null; + } + } + } + } + + /** + * Remove all parse listeners. + * + * @see #addParseListener + */ + void removeParseListeners() { + _parseListeners = null; + } + + /** + * Notify any parse listeners of an enter rule event. + * + * @see #addParseListener + */ + void triggerEnterRuleEvent() { + for (ParseTreeListener listener in _parseListeners) { + listener.enterEveryRule(_ctx); + _ctx.enterRule(listener); + } + } + + /** + * Notify any parse listeners of an exit rule event. + * + * @see #addParseListener + */ + void triggerExitRuleEvent() { + // reverse order walk of listeners + for (int i = _parseListeners.length - 1; i >= 0; i--) { + ParseTreeListener listener = _parseListeners[i]; + _ctx.exitRule(listener); + listener.exitEveryRule(_ctx); + } + } + + /** + * Gets the number of syntax errors reported during parsing. This value is + * incremented each time {@link #notifyErrorListeners} is called. + * + * @see #notifyErrorListeners + */ + int get numberOfSyntaxErrors { + return _syntaxErrors; + } + + TokenFactory get tokenFactory { + return _input.tokenSource.tokenFactory; + } + + /** Tell our token source and error strategy about a new way to create tokens. */ + + void set tokenFactory(TokenFactory factory) { + _input.tokenSource.tokenFactory = factory; + } + + /** + * The ATN with bypass alternatives is expensive to create so we create it + * lazily. + * + * @throws UnsupportedOperationException if the current parser does not + * implement the {@link #getSerializedATN()} method. + */ + ATN get ATNWithBypassAlts { + String serializedAtn = serializedATN; + if (serializedAtn == null) { + throw new UnsupportedError( + "The current parser does not support an ATN with bypass alternatives."); + } + + ATN result = bypassAltsAtnCache[serializedAtn]; + if (result == null) { + ATNDeserializationOptions deserializationOptions = + new ATNDeserializationOptions(); + deserializationOptions.setGenerateRuleBypassTransitions(true); + result = new ATNDeserializer(deserializationOptions) + .deserialize(serializedAtn.codeUnits); + bypassAltsAtnCache[serializedAtn] = result; + } + + return result; + } + + /** + * The preferred method of getting a tree pattern. For example, here's a + * sample use: + * + * 
+   * ParseTree t = parser.expr();
+   * ParseTreePattern p = parser.compileParseTreePattern("<ID>+0", MyParser.RULE_expr);
+   * ParseTreeMatch m = p.match(t);
+   * String id = m.get("ID");
+   *
+ */ + ParseTreePattern compileParseTreePattern(String pattern, int patternRuleIndex, + [Lexer lexer]) { + if (lexer == null) { + TokenSource tokenSource = tokenStream?.tokenSource; + if (tokenSource == null || !(tokenSource is Lexer)) { + throw new UnsupportedError("Parser can't discover a lexer to use"); + } + lexer = tokenSource; + } + + ParseTreePatternMatcher m = new ParseTreePatternMatcher(lexer, this); + return m.compile(pattern, patternRuleIndex); + } + + TokenStream get inputStream => tokenStream; + + void set inputStream(IntStream input) { + setTokenStream(input); + } + + TokenStream get tokenStream => _input; + + /** Set the token stream and reset the parser. */ + void setTokenStream(TokenStream input) { + this._input = null; + reset(); + this._input = input; + } + + /** Match needs to return the current input symbol, which gets put + * into the label for the associated token ref; e.g., x=ID. + */ + + Token get currentToken { + return _input.LT(1); + } + + void notifyErrorListeners(String msg, + [Token offendingToken = null, RecognitionException e = null]) { + offendingToken = offendingToken ?? currentToken; + _syntaxErrors++; + int line = -1; + int charPositionInLine = -1; + line = offendingToken.line; + charPositionInLine = offendingToken.charPositionInLine; + + ErrorListener listener = errorListenerDispatch; + listener.syntaxError( + this, offendingToken, line, charPositionInLine, msg, e); + } + + /** + * Consume and return the {@linkplain #getCurrentToken current symbol}. + * + *

E.g., given the following input with [A] being the current + * lookahead symbol, this function moves the cursor to [B] and returns + * [A].

+ * + * 
+   *  A B
+   *  ^
+   *
+ * + * If the parser is not in error recovery mode, the consumed symbol is added + * to the parse tree using {@link ParserRuleContext#addChild}, and + * {@link ParseTreeListener#visitTerminal} is called on any parse listeners. + * If the parser is in error recovery mode, the consumed symbol is + * added to the parse tree using {@link #createErrorNode(ParserRuleContext, Token)} then + * {@link ParserRuleContext#addErrorNode(ErrorNode)} and + * {@link ParseTreeListener#visitErrorNode} is called on any parse + * listeners. + */ + Token consume() { + Token o = currentToken; + if (o.type != IntStream.EOF) { + inputStream.consume(); + } + bool hasListener = _parseListeners != null && !_parseListeners.isEmpty; + if (_buildParseTrees || hasListener) { + if (errorHandler.inErrorRecoveryMode(this)) { + ErrorNode node = _ctx.addErrorNode(createErrorNode(_ctx, o)); + if (_parseListeners != null) { + for (ParseTreeListener listener in _parseListeners) { + listener.visitErrorNode(node); + } + } + } else { + TerminalNode node = _ctx.addChild(createTerminalNode(_ctx, o)); + if (_parseListeners != null) { + for (ParseTreeListener listener in _parseListeners) { + listener.visitTerminal(node); + } + } + } + } + return o; + } + + /** How to create a token leaf node associated with a parent. + * Typically, the terminal node to create is not a function of the parent. + * + * @since 4.7 + */ + TerminalNode createTerminalNode(ParserRuleContext parent, Token t) { + return new TerminalNodeImpl(t); + } + + /** How to create an error node, given a token, associated with a parent. + * Typically, the error node to create is not a function of the parent. + * + * @since 4.7 + */ + ErrorNode createErrorNode(ParserRuleContext parent, Token t) { + return new ErrorNodeImpl(t); + } + + void addContextToParseTree() { + ParserRuleContext parent = _ctx.parent; + // add current context to parent if we have a parent + if (parent != null) { + parent.addAnyChild(_ctx); + } + } + + /** + * Always called by generated parsers upon entry to a rule. Access field + * {@link #_ctx} get the current context. + */ + void enterRule(ParserRuleContext localctx, int state, int ruleIndex) { + this.state = state; + _ctx = localctx; + _ctx.start = _input.LT(1); + if (_buildParseTrees) addContextToParseTree(); + if (_parseListeners != null) triggerEnterRuleEvent(); + } + + void exitRule() { + if (matchedEOF) { + // if we have matched EOF, it cannot consume past EOF so we use LT(1) here + _ctx.stop = _input.LT(1); // LT(1) will be end of file + } else { + _ctx.stop = _input.LT(-1); // stop node is what we just matched + } + // trigger event on _ctx, before it reverts to parent + if (_parseListeners != null) triggerExitRuleEvent(); + state = _ctx.invokingState; + _ctx = _ctx.parent; + } + + void enterOuterAlt(ParserRuleContext localctx, int altNum) { + localctx.altNumber = altNum; + // if we have new localctx, make sure we replace existing ctx + // that is previous child of parse tree + if (_buildParseTrees && _ctx != localctx) { + ParserRuleContext parent = _ctx.parent; + if (parent != null) { + parent.removeLastChild(); + parent.addAnyChild(localctx); + } + } + _ctx = localctx; + } + + /** + * Get the precedence level for the top-most precedence rule. + * + * @return The precedence level for the top-most precedence rule, or -1 if + * the parser context is not nested within a precedence rule. + */ + int get precedence { + if (_precedenceStack.isEmpty) { + return -1; + } + + return _precedenceStack.last; + } + + void enterRecursionRule( + ParserRuleContext localctx, int state, int ruleIndex, int precedence) { + this.state = state; + _precedenceStack.add(precedence); + _ctx = localctx; + _ctx.start = _input.LT(1); + if (_parseListeners != null) { + triggerEnterRuleEvent(); // simulates rule entry for left-recursive rules + } + } + + /** Like {@link #enterRule} but for recursive rules. + * Make the current context the child of the incoming localctx. + */ + void pushNewRecursionContext( + ParserRuleContext localctx, int state, int ruleIndex) { + ParserRuleContext previous = _ctx; + previous.parent = localctx; + previous.invokingState = state; + previous.stop = _input.LT(-1); + + _ctx = localctx; + _ctx.start = previous.start; + if (_buildParseTrees) { + _ctx.addAnyChild(previous); + } + + if (_parseListeners != null) { + triggerEnterRuleEvent(); // simulates rule entry for left-recursive rules + } + } + + void unrollRecursionContexts(ParserRuleContext _parentctx) { + _precedenceStack.removeLast(); + _ctx.stop = _input.LT(-1); + ParserRuleContext retctx = _ctx; // save current ctx (return value) + + // unroll so _ctx is as it was before call to recursive method + if (_parseListeners != null) { + while (_ctx != _parentctx) { + triggerExitRuleEvent(); + _ctx = _ctx.parent; + } + } else { + _ctx = _parentctx; + } + + // hook into tree + retctx.parent = _parentctx; + + if (_buildParseTrees && _parentctx != null) { + // add return ctx into invoking rule's tree + _parentctx.addAnyChild(retctx); + } + } + + ParserRuleContext getInvokingContext(int ruleIndex) { + ParserRuleContext p = _ctx; + while (p != null) { + if (p.ruleIndex == ruleIndex) return p; + p = p.parent; + } + return null; + } + + ParserRuleContext get context { + return _ctx; + } + + set context(ParserRuleContext ctx) { + _ctx = ctx; + } + + bool precpred(RuleContext localctx, int precedence) { + return precedence >= _precedenceStack.last; + } + + bool inContext(String context) { + // TODO: useful in parser? + return false; + } + + /** + * Checks whether or not [symbol] can follow the current state in the + * ATN. The behavior of this method is equivalent to the following, but is + * implemented such that the complete context-sensitive follow set does not + * need to be explicitly constructed. + * + * 
+   * return expectedTokens.contains(symbol);
+   *
+ * + * @param symbol the symbol type to check + * @return [true] if [symbol] can follow the current state in + * the ATN, otherwise [false]. + */ + bool isExpectedToken(int symbol) { +// return interpreter.atn.nextTokens(_ctx); + ATN atn = interpreter.atn; + ParserRuleContext ctx = _ctx; + ATNState s = atn.states[state]; + IntervalSet following = atn.nextTokens(s); + if (following.contains(symbol)) { + return true; + } +// log("following "+s+"="+following); + if (!following.contains(Token.EPSILON)) return false; + + while (ctx != null && + ctx.invokingState >= 0 && + following.contains(Token.EPSILON)) { + ATNState invokingState = atn.states[ctx.invokingState]; + RuleTransition rt = invokingState.transition(0); + following = atn.nextTokens(rt.followState); + if (following.contains(symbol)) { + return true; + } + + ctx = ctx.parent; + } + + if (following.contains(Token.EPSILON) && symbol == Token.EOF) { + return true; + } + + return false; + } + + bool isMatchedEOF() { + return matchedEOF; + } + + /** + * Computes the set of input symbols which could follow the current parser + * state and context, as given by {@link #getState} and {@link #getContext}, + * respectively. + * + * @see ATN#getExpectedTokens(int, RuleContext) + */ + IntervalSet get expectedTokens { + return getATN().getExpectedTokens(state, context); + } + + IntervalSet get expectedTokensWithinCurrentRule { + ATN atn = interpreter.atn; + ATNState s = atn.states[state]; + return atn.nextTokens(s); + } + + /** Get a rule's index (i.e., {@code RULE_ruleName} field) or -1 if not found. */ + int getRuleIndex(String ruleName) { + int ruleIndex = ruleIndexMap[ruleName]; + if (ruleIndex != null) return ruleIndex; + return -1; + } + + ParserRuleContext get ruleContext { + return _ctx; + } + + List get ruleInvocationStack => getRuleInvocationStack(); + + /** Return List<String> of the rule names in your parser instance + * leading up to a call to the current rule. You could override if + * you want more details such as the file/line info of where + * in the ATN a rule is invoked. + * + * This is very useful for error messages. + */ + List getRuleInvocationStack([RuleContext p]) { + p = p ?? _ctx; + final _ruleNames = ruleNames; + List stack = []; + while (p != null) { + // compute what follows who invoked us + int ruleIndex = p.ruleIndex; + if (ruleIndex < 0) + stack.add("n/a"); + else + stack.add(_ruleNames[ruleIndex]); + p = p.parent; + } + return stack; + } + + /** For debugging and other purposes. */ + List get dfaStrings { + List s = []; + for (int d = 0; d < interpreter.decisionToDFA.length; d++) { + DFA dfa = interpreter.decisionToDFA[d]; + s.add(dfa.toString(vocabulary)); + } + return s; + } + + /** For debugging and other purposes. */ + void dumpDFA() { + bool seenOne = false; + for (int d = 0; d < interpreter.decisionToDFA.length; d++) { + DFA dfa = interpreter.decisionToDFA[d]; + if (!dfa.states.isEmpty) { + if (seenOne) print(""); + print("Decision ${dfa.decision}:"); + stdout.write(dfa.toString(vocabulary)); + seenOne = true; + } + } + } + + String get sourceName { + return _input.sourceName; + } + + ParseInfo get parseInfo { + ParserATNSimulator interp = interpreter; + if (interp is ProfilingATNSimulator) { + return new ParseInfo(interp); + } + return null; + } + + /** + * @since 4.3 + */ + void setProfile(bool profile) { + ParserATNSimulator interp = interpreter; + PredictionMode saveMode = interp.predictionMode; + if (profile) { + if (!(interp is ProfilingATNSimulator)) { + interpreter = new ProfilingATNSimulator(this); + } + } else if (interp is ProfilingATNSimulator) { + ParserATNSimulator sim = new ParserATNSimulator( + this, getATN(), interp.decisionToDFA, interp.sharedContextCache); + interpreter = sim; + } + interpreter.predictionMode = saveMode; + } + + /** During a parse is sometimes useful to listen in on the rule entry and exit + * events as well as token matches. This is for quick and dirty debugging. + */ + void setTrace(bool trace) { + if (!trace) { + removeParseListener(_tracer); + _tracer = null; + } else { + if (_tracer != null) + removeParseListener(_tracer); + else + _tracer = new TraceListener(this); + addParseListener(_tracer); + } + } + + /** + * Gets whether a [TraceListener] is registered as a parse listener + * for the parser. + * + * @see #setTrace(bool) + */ + bool isTrace() { + return _tracer != null; + } +} diff --git a/runtime/Dart/lib/src/parser_interpreter.dart b/runtime/Dart/lib/src/parser_interpreter.dart new file mode 100644 index 000000000..238b9442c --- /dev/null +++ b/runtime/Dart/lib/src/parser_interpreter.dart @@ -0,0 +1,396 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:collection'; + +import 'atn/atn.dart'; +import 'dfa/dfa.dart'; +import 'error/error.dart'; +import 'misc/pair.dart'; +import 'parser.dart'; +import 'parser_rule_context.dart'; +import 'token.dart'; +import 'token_stream.dart'; +import 'vocabulary.dart'; + +/** A parser simulator that mimics what ANTLR's generated + * parser code does. A ParserATNSimulator is used to make + * predictions via adaptivePredict but this class moves a pointer through the + * ATN to simulate parsing. ParserATNSimulator just + * makes us efficient rather than having to backtrack, for example. + * + * This properly creates parse trees even for left recursive rules. + * + * We rely on the left recursive rule invocation and special predicate + * transitions to make left recursive rules work. + * + * See TestParserInterpreter for examples. + */ +class ParserInterpreter extends Parser { + final String grammarFileName; + final ATN atn; + + List decisionToDFA; // not shared like it is for generated parsers + final PredictionContextCache sharedContextCache = + new PredictionContextCache(); + + final List ruleNames; + + final Vocabulary vocabulary; + + /** This stack corresponds to the _parentctx, _parentState pair of locals + * that would exist on call stack frames with a recursive descent parser; + * in the generated function for a left-recursive rule you'd see: + * + * EContext e(int _p) throws RecognitionException { + * ParserRuleContext _parentctx = context; // Pair.a + * int _parentState = state; // Pair.b + * ... + * } + * + * Those values are used to create new recursive rule invocation contexts + * associated with left operand of an alt like "expr '*' expr". + */ + final DoubleLinkedQueue> _parentContextStack = + new DoubleLinkedQueue(); + + /** We need a map from (decision,inputIndex)->forced alt for computing ambiguous + * parse trees. For now, we allow exactly one override. + */ + int overrideDecision = -1; + int overrideDecisionInputIndex = -1; + int overrideDecisionAlt = -1; + bool overrideDecisionReached = + false; // latch and only override once; error might trigger infinite loop + + /** What is the current context when we override a decisions? This tells + * us what the root of the parse tree is when using override + * for an ambiguity/lookahead check. + */ + InterpreterRuleContext overrideDecisionRoot = null; + + /** Return the root of the parse, which can be useful if the parser + * bails out. You still can access the top node. Note that, + * because of the way left recursive rules add children, it's possible + * that the root will not have any children if the start rule immediately + * called and left recursive rule that fails. + * + * @since 4.5.1 + */ + InterpreterRuleContext rootContext; + + ParserInterpreter(this.grammarFileName, this.vocabulary, this.ruleNames, + this.atn, TokenStream input) + : super(input) { + // init decision DFA + int numberOfDecisions = atn.numberOfDecisions; + this.decisionToDFA = new List(numberOfDecisions); + for (int i = 0; i < numberOfDecisions; i++) { + DecisionState decisionState = atn.getDecisionState(i); + decisionToDFA[i] = new DFA(decisionState, i); + } + + // get atn simulator that knows how to do predictions + interpreter = + new ParserATNSimulator(this, atn, decisionToDFA, sharedContextCache); + } + + void reset() { + super.reset(); + overrideDecisionReached = false; + overrideDecisionRoot = null; + } + + ATN getATN() { + return atn; + } + + /** Begin parsing at startRuleIndex */ + ParserRuleContext parse(int startRuleIndex) { + RuleStartState startRuleStartState = atn.ruleToStartState[startRuleIndex]; + + rootContext = createInterpreterRuleContext( + null, ATNState.INVALID_STATE_NUMBER, startRuleIndex); + if (startRuleStartState.isLeftRecursiveRule) { + enterRecursionRule( + rootContext, startRuleStartState.stateNumber, startRuleIndex, 0); + } else { + enterRule(rootContext, startRuleStartState.stateNumber, startRuleIndex); + } + + while (true) { + ATNState p = atnState; + switch (p.stateType) { + case StateType.RULE_STOP: + // pop; return from rule + if (context.isEmpty) { + if (startRuleStartState.isLeftRecursiveRule) { + ParserRuleContext result = context; + Pair parentContext = + _parentContextStack.removeLast(); + unrollRecursionContexts(parentContext.a); + return result; + } else { + exitRule(); + return rootContext; + } + } + + visitRuleStopState(p); + break; + + default: + try { + visitState(p); + } on RecognitionException catch (e) { + state = atn.ruleToStopState[p.ruleIndex].stateNumber; + context.exception = e; + errorHandler.reportError(this, e); + recover(e); + } + + break; + } + } + } + + void enterRecursionRule( + ParserRuleContext localctx, int state, int ruleIndex, int precedence) { + Pair pair = + new Pair(context, localctx.invokingState); + _parentContextStack.add(pair); + super.enterRecursionRule(localctx, state, ruleIndex, precedence); + } + + ATNState get atnState { + return atn.states[state]; + } + + void visitState(ATNState p) { +// System.out.println("visitState "+p.stateNumber); + int predictedAlt = 1; + if (p is DecisionState) { + predictedAlt = visitDecisionState(p); + } + + Transition transition = p.transition(predictedAlt - 1); + switch (transition.type) { + case TransitionType.EPSILON: + if (p.stateType == StateType.STAR_LOOP_ENTRY && + (p as StarLoopEntryState).isPrecedenceDecision && + !(transition.target is LoopEndState)) { + // We are at the start of a left recursive rule's (...)* loop + // and we're not taking the exit branch of loop. + InterpreterRuleContext localctx = createInterpreterRuleContext( + _parentContextStack.last.a, + _parentContextStack.last.b, + context.ruleIndex); + pushNewRecursionContext(localctx, + atn.ruleToStartState[p.ruleIndex].stateNumber, context.ruleIndex); + } + break; + + case TransitionType.ATOM: + match((transition as AtomTransition).atomLabel); + break; + + case TransitionType.RANGE: + case TransitionType.SET: + case TransitionType.NOT_SET: + if (!transition.matches( + inputStream.LA(1), Token.MIN_USER_TOKEN_TYPE, 65535)) { + recoverInline(); + } + matchWildcard(); + break; + + case TransitionType.WILDCARD: + matchWildcard(); + break; + + case TransitionType.RULE: + RuleStartState ruleStartState = transition.target; + int ruleIndex = ruleStartState.ruleIndex; + InterpreterRuleContext newctx = + createInterpreterRuleContext(context, p.stateNumber, ruleIndex); + if (ruleStartState.isLeftRecursiveRule) { + enterRecursionRule(newctx, ruleStartState.stateNumber, ruleIndex, + (transition as RuleTransition).precedence); + } else { + enterRule(newctx, transition.target.stateNumber, ruleIndex); + } + break; + + case TransitionType.PREDICATE: + PredicateTransition predicateTransition = transition; + if (!sempred(context, predicateTransition.ruleIndex, + predicateTransition.predIndex)) { + throw new FailedPredicateException(this); + } + + break; + + case TransitionType.ACTION: + ActionTransition actionTransition = transition; + action( + context, actionTransition.ruleIndex, actionTransition.actionIndex); + break; + + case TransitionType.PRECEDENCE: + if (!precpred(context, + (transition as PrecedencePredicateTransition).precedence)) { + throw new FailedPredicateException(this, + "precpred(context, ${(transition as PrecedencePredicateTransition).precedence})"); + } + break; + + default: + throw new UnsupportedError("Unrecognized ATN transition type."); + } + + state = transition.target.stateNumber; + } + + /** Method visitDecisionState() is called when the interpreter reaches + * a decision state (instance of DecisionState). It gives an opportunity + * for subclasses to track interesting things. + */ + int visitDecisionState(DecisionState p) { + int predictedAlt = 1; + if (p.numberOfTransitions > 1) { + errorHandler.sync(this); + int decision = p.decision; + if (decision == overrideDecision && + inputStream.index == overrideDecisionInputIndex && + !overrideDecisionReached) { + predictedAlt = overrideDecisionAlt; + overrideDecisionReached = true; + } else { + predictedAlt = + interpreter.adaptivePredict(inputStream, decision, context); + } + } + return predictedAlt; + } + + /** Provide simple "factory" for InterpreterRuleContext's. + * @since 4.5.1 + */ + InterpreterRuleContext createInterpreterRuleContext( + ParserRuleContext parent, int invokingStateNumber, int ruleIndex) { + return new InterpreterRuleContext(parent, invokingStateNumber, ruleIndex); + } + + void visitRuleStopState(ATNState p) { + RuleStartState ruleStartState = atn.ruleToStartState[p.ruleIndex]; + if (ruleStartState.isLeftRecursiveRule) { + Pair parentContext = + _parentContextStack.removeLast(); + unrollRecursionContexts(parentContext.a); + state = parentContext.b; + } else { + exitRule(); + } + + RuleTransition ruleTransition = atn.states[state].transition(0); + state = ruleTransition.followState.stateNumber; + } + + /** Override this parser interpreters normal decision-making process + * at a particular decision and input token index. Instead of + * allowing the adaptive prediction mechanism to choose the + * first alternative within a block that leads to a successful parse, + * force it to take the alternative, 1..n for n alternatives. + * + * As an implementation limitation right now, you can only specify one + * override. This is sufficient to allow construction of different + * parse trees for ambiguous input. It means re-parsing the entire input + * in general because you're never sure where an ambiguous sequence would + * live in the various parse trees. For example, in one interpretation, + * an ambiguous input sequence would be matched completely in expression + * but in another it could match all the way back to the root. + * + * s : e '!'? ; + * e : ID + * | ID '!' + * ; + * + * Here, x! can be matched as (s (e ID) !) or (s (e ID !)). In the first + * case, the ambiguous sequence is fully contained only by the root. + * In the second case, the ambiguous sequences fully contained within just + * e, as in: (e ID !). + * + * Rather than trying to optimize this and make + * some intelligent decisions for optimization purposes, I settled on + * just re-parsing the whole input and then using + * {link Trees#getRootOfSubtreeEnclosingRegion} to find the minimal + * subtree that contains the ambiguous sequence. I originally tried to + * record the call stack at the point the parser detected and ambiguity but + * left recursive rules create a parse tree stack that does not reflect + * the actual call stack. That impedance mismatch was enough to make + * it it challenging to restart the parser at a deeply nested rule + * invocation. + * + * Only parser interpreters can override decisions so as to avoid inserting + * override checking code in the critical ALL(*) prediction execution path. + * + * @since 4.5.1 + */ + void addDecisionOverride(int decision, int tokenIndex, int forcedAlt) { + overrideDecision = decision; + overrideDecisionInputIndex = tokenIndex; + overrideDecisionAlt = forcedAlt; + } + + /** Rely on the error handler for this parser but, if no tokens are consumed + * to recover, add an error node. Otherwise, nothing is seen in the parse + * tree. + */ + void recover(RecognitionException e) { + int i = inputStream.index; + errorHandler.recover(this, e); + if (inputStream.index == i) { + // no input consumed, better add an error node + if (e is InputMismatchException) { + InputMismatchException ime = e; + Token tok = e.offendingToken; + int expectedTokenType = Token.INVALID_TYPE; + if (!ime.expectedTokens.isNil) { + expectedTokenType = ime.expectedTokens.minElement; // get any element + } + Token errToken = tokenFactory.create( + expectedTokenType, + tok.text, + new Pair(tok.tokenSource, tok.tokenSource.inputStream), + Token.DEFAULT_CHANNEL, + -1, + -1, + // invalid start/stop + tok.line, + tok.charPositionInLine); + context.addErrorNode(createErrorNode(context, errToken)); + } else { + // NoViableAlt + Token tok = e.offendingToken; + Token errToken = tokenFactory.create( + Token.INVALID_TYPE, + tok.text, + new Pair(tok.tokenSource, tok.tokenSource.inputStream), + Token.DEFAULT_CHANNEL, + -1, + -1, + // invalid start/stop + tok.line, + tok.charPositionInLine); + context.addErrorNode(createErrorNode(context, errToken)); + } + } + } + + Token recoverInline() { + return errorHandler.recoverInline(this); + } +} diff --git a/runtime/Dart/lib/src/parser_rule_context.dart b/runtime/Dart/lib/src/parser_rule_context.dart new file mode 100644 index 000000000..e6743bd1e --- /dev/null +++ b/runtime/Dart/lib/src/parser_rule_context.dart @@ -0,0 +1,288 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'error/error.dart'; +import 'interval_set.dart'; +import 'parser.dart'; +import 'rule_context.dart'; +import 'token.dart'; +import 'tree/tree.dart'; + +/** A rule invocation record for parsing. + * + * Contains all of the information about the current rule not stored in the + * RuleContext. It handles parse tree children list, Any ATN state + * tracing, and the default values available for rule invocations: + * start, stop, rule index, current alt number. + * + * Subclasses made for each rule and grammar track the parameters, + * return values, locals, and labels specific to that rule. These + * are the objects that are returned from rules. + * + * Note text is not an actual field of a rule return value; it is computed + * from start and stop using the input stream's toString() method. I + * could add a ctor to this so that we can pass in and store the input + * stream, but I'm not sure we want to do that. It would seem to be undefined + * to get the .text property anyway if the rule matches tokens from multiple + * input streams. + * + * I do not use getters for fields of objects that are used simply to + * group values such as this aggregate. The getters/setters are there to + * satisfy the superclass interface. + */ +class ParserRuleContext extends RuleContext { + /** + * If we are debugging or building a parse tree for a visitor, + * we need to track all of the tokens and rule invocations associated + * with this rule's context. This is empty for parsing w/o tree constr. + * operation because we don't the need to track the details about + * how we parse this rule. + */ + List children; + + /** + * Get the initial/final token in this context. + * Note that the range from start to stop is inclusive, so for rules that do not consume anything + * (for example, zero length or error productions) this token may exceed stop. + */ + Token start, stop; + + /// The exception that forced this rule to return. If the rule successfully + /// completed, this is null. + RecognitionException exception = null; + + ParserRuleContext([RuleContext parent, int invokingStateNumber]) + : super(parent: parent, invokingState: invokingStateNumber); + + /** COPY a ctx (I'm deliberately not using copy constructor) to avoid + * confusion with creating node with parent. Does not copy children + * (except error leaves). + * + * This is used in the generated parser code to flip a generic XContext + * node for rule X to a YContext for alt label Y. In that sense, it is + * not really a generic copy function. + * + * If we do an error sync() at start of a rule, we might add error nodes + * to the generic XContext so this function must copy those nodes to + * the YContext as well else they are lost! + */ + void copyFrom(ParserRuleContext ctx) { + this.parent = ctx.parent; + this.invokingState = ctx.invokingState; + + this.start = ctx.start; + this.stop = ctx.stop; + + // copy any error nodes to alt label node + if (ctx.children != null) { + this.children = []; + // reset parent pointer for any error nodes + for (ParseTree child in ctx.children) { + if (child is ErrorNode) { + addChild(child); + } + } + } + } + + // Double dispatch methods for listeners + + void enterRule(ParseTreeListener listener) {} + + void exitRule(ParseTreeListener listener) {} + + /** Add a parse tree node to this as a child. Works for + * internal and leaf nodes. Does not set parent link; + * other add methods must do that. Other addChild methods + * call this. + * + * We cannot set the parent pointer of the incoming node + * because the existing interfaces do not have a setParent() + * method and I don't want to break backward compatibility for this. + * + * @since 4.7 + */ + T addAnyChild(T t) { + if (children == null) children = []; + children.add(t); + return t; + } + + /** Add a token leaf node child and force its parent to be this node. */ + TerminalNode addChild(TerminalNode t) { + t.parent = this; + return addAnyChild(t); + } + + /** Add an error node child and force its parent to be this node. + * + * @since 4.7 + */ + ErrorNode addErrorNode(ErrorNode errorNode) { + errorNode.parent=this; + return addAnyChild(errorNode); + } + + /** Used by enterOuterAlt to toss out a RuleContext previously added as + * we entered a rule. If we have # label, we will need to remove + * generic ruleContext object. + */ + void removeLastChild() { + if (children != null) { + children.removeLast(); + } + } + + // Override to make type more specific + ParserRuleContext get parent { + return super.parent; + } + + @override + ParseTree getChild(int i) { + if (children == null || i < 0 || i >= children.length) { + return null; + } + + if (T == null) { + return children[i]; + } + int j = -1; // what element have we found with ctxType? + for (ParseTree o in children) { + if (o is T) { + j++; + if (j == i) { + return o; + } + } + } + return null; + } + + TerminalNode getToken(int ttype, int i) { + if (children == null || i < 0 || i >= children.length) { + return null; + } + + int j = -1; // what token with ttype have we found? + for (ParseTree o in children) { + if (o is TerminalNode) { + TerminalNode tnode = o; + Token symbol = tnode.symbol; + if (symbol.type == ttype) { + j++; + if (j == i) { + return tnode; + } + } + } + } + + return null; + } + + List getTokens(int ttype) { + if (children == null) { + return []; + } + + List tokens = null; + for (ParseTree o in children) { + if (o is TerminalNode) { + TerminalNode tnode = o; + Token symbol = tnode.symbol; + if (symbol.type == ttype) { + if (tokens == null) { + tokens = []; + } + tokens.add(tnode); + } + } + } + + if (tokens == null) { + return []; + } + + return tokens; + } + + T getRuleContext(int i) { + return getChild(i); + } + + List getRuleContexts() { + if (children == null) { + return []; + } + + List contexts = null; + for (ParseTree o in children) { + if (o is T) { + if (contexts == null) { + contexts = []; + } + + contexts.add(o); + } + } + + if (contexts == null) { + return []; + } + + return contexts; + } + + int get childCount => children?.length ?? 0; + + Interval get sourceInterval { + if (start == null) { + return Interval.INVALID; + } + if (stop == null || stop.tokenIndex < start.tokenIndex) { + return Interval(start.tokenIndex, start.tokenIndex - 1); // empty + } + return Interval(start.tokenIndex, stop.tokenIndex); + } + + /** Used for rule context info debugging during parse-time, not so much for ATN debugging */ + String toInfoString(Parser recognizer) { + List rules = recognizer.getRuleInvocationStack(this); + + return "ParserRuleContext${rules.reversed}{start=$start, stop=$stop}'"; + } + + static final EMPTY = ParserRuleContext(); +} + +/** + * This class extends [ParserRuleContext] by allowing the value of + * {@link #getRuleIndex} to be explicitly set for the context. + * + *

+ * [ParserRuleContext] does not include field storage for the rule index + * since the context classes created by the code generator override the + * {@link #getRuleIndex} method to return the correct value for that context. + * Since the parser interpreter does not use the context classes generated for a + * parser, this class (with slightly more memory overhead per node) is used to + * provide equivalent functionality.

+ */ +class InterpreterRuleContext extends ParserRuleContext { + int ruleIndex = -1; + + /** + * Constructs a new [InterpreterRuleContext] with the specified + * parent, invoking state, and rule index. + * + * @param parent The parent context. + * @param invokingStateNumber The invoking state number. + * @param ruleIndex The rule index for the current context. + */ + InterpreterRuleContext( + ParserRuleContext parent, int invokingStateNumber, this.ruleIndex) + : super(parent, invokingStateNumber); +} diff --git a/runtime/Dart/lib/src/prediction_context.dart b/runtime/Dart/lib/src/prediction_context.dart new file mode 100644 index 000000000..c2bc71c5a --- /dev/null +++ b/runtime/Dart/lib/src/prediction_context.dart @@ -0,0 +1,874 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'package:collection/collection.dart'; + +import 'atn/atn.dart'; +import 'misc/pair.dart'; +import 'recognizer.dart'; +import 'rule_context.dart'; +import 'util/murmur_hash.dart'; + +abstract class PredictionContext { + /** + * Represents {@code $} in local context prediction, which means wildcard. + * {@code *+x = *}. + */ + static final EmptyPredictionContext EMPTY = new EmptyPredictionContext(); + + /** + * Represents {@code $} in an array in full context mode, when {@code $} + * doesn't mean wildcard: {@code $ + x = [$,x]}. Here, + * {@code $} = {@link #EMPTY_RETURN_STATE}. + */ + static final int EMPTY_RETURN_STATE = 0x7FFFFFFF; + + static final int INITIAL_HASH = 1; + + static int globalNodeCount = 0; + int id = globalNodeCount++; + + /** + * Stores the computed hash code of this [PredictionContext]. The hash + * code is computed in parts to match the following reference algorithm. + * + * 
+   *   int referenceHashCode() {
+   *      int hash = {@link MurmurHash#initialize MurmurHash.initialize}({@link #INITIAL_HASH});
+   *
+   *      for (int i = 0; i < {@link #size()}; i++) {
+   *          hash = {@link MurmurHash#update MurmurHash.update}(hash, {@link #getParent getParent}(i));
+   *      }
+   *
+   *      for (int i = 0; i < {@link #size()}; i++) {
+   *          hash = {@link MurmurHash#update MurmurHash.update}(hash, {@link #getReturnState getReturnState}(i));
+   *      }
+   *
+   *      hash = {@link MurmurHash#finish MurmurHash.finish}(hash, 2 * {@link #size()});
+   *      return hash;
+   *  }
+   *
+ */ + final int cachedHashCode; + + PredictionContext(this.cachedHashCode); + + /** Convert a [RuleContext] tree to a [PredictionContext] graph. + * Return {@link #EMPTY} if [outerContext] is empty or null. + */ + static PredictionContext fromRuleContext(ATN atn, RuleContext outerContext) { + if (outerContext == null) outerContext = RuleContext.EMPTY; + + // if we are in RuleContext of start rule, s, then PredictionContext + // is EMPTY. Nobody called us. (if we are empty, return empty) + if (outerContext.parent == null || outerContext == RuleContext.EMPTY) { + return PredictionContext.EMPTY; + } + + // If we have a parent, convert it to a PredictionContext graph + PredictionContext parent = EMPTY; + parent = PredictionContext.fromRuleContext(atn, outerContext.parent); + + ATNState state = atn.states[outerContext.invokingState]; + RuleTransition transition = state.transition(0); + return SingletonPredictionContext.create( + parent, transition.followState.stateNumber); + } + + int get length; + + PredictionContext getParent(int index); + + int getReturnState(int index); + + /** This means only the {@link #EMPTY} (wildcard? not sure) context is in set. */ + bool get isEmpty { + return this == EMPTY; + } + + bool hasEmptyPath() { + // since EMPTY_RETURN_STATE can only appear in the last position, we check last one + return getReturnState(length - 1) == EMPTY_RETURN_STATE; + } + + int get hashCode { + return cachedHashCode; + } + + bool operator ==(Object obj); + + static int calculateEmptyHashCode() { + int hash = MurmurHash.initialize(INITIAL_HASH); + hash = MurmurHash.finish(hash, 0); + return hash; + } + + static int calculateHashCode( + List parents, List returnStates) { + int hash = MurmurHash.initialize(INITIAL_HASH); + + for (PredictionContext parent in parents) { + hash = MurmurHash.update(hash, parent); + } + + for (int returnState in returnStates) { + hash = MurmurHash.update(hash, returnState); + } + + hash = MurmurHash.finish(hash, 2 * parents.length); + return hash; + } + + // dispatch + static PredictionContext merge( + PredictionContext a, + PredictionContext b, + bool rootIsWildcard, + Map, PredictionContext> + mergeCache) { + assert(a != null && b != null); // must be empty context, never null + + // share same graph if both same + if (a == b || a == b) return a; + + if (a is SingletonPredictionContext && b is SingletonPredictionContext) { + return mergeSingletons(a, b, rootIsWildcard, mergeCache); + } + + // At least one of a or b is array + // If one is $ and rootIsWildcard, return $ as * wildcard + if (rootIsWildcard) { + if (a is EmptyPredictionContext) return a; + if (b is EmptyPredictionContext) return b; + } + + // convert singleton so both are arrays to normalize + if (a is SingletonPredictionContext) { + a = new ArrayPredictionContext.of(a); + } + if (b is SingletonPredictionContext) { + b = new ArrayPredictionContext.of(b); + } + return mergeArrays(a, b, rootIsWildcard, mergeCache); + } + + /** + * Merge two [SingletonPredictionContext] instances. + * + *

Stack tops equal, parents merge is same; return left graph.
+ *

+ * + *

Same stack top, parents differ; merge parents giving array node, then + * remainders of those graphs. A new root node is created to point to the + * merged parents.
+ *

+ * + *

Different stack tops pointing to same parent. Make array node for the + * root where both element in the root point to the same (original) + * parent.
+ *

+ * + *

Different stack tops pointing to different parents. Make array node for + * the root where each element points to the corresponding original + * parent.
+ *

+ * + * @param a the first [SingletonPredictionContext] + * @param b the second [SingletonPredictionContext] + * @param rootIsWildcard [true] if this is a local-context merge, + * otherwise false to indicate a full-context merge + * @param mergeCache + */ + static PredictionContext mergeSingletons( + SingletonPredictionContext a, + SingletonPredictionContext b, + bool rootIsWildcard, + Map, PredictionContext> + mergeCache) { + if (mergeCache != null) { + PredictionContext previous = mergeCache[Pair(a, b)]; + if (previous != null) return previous; + previous = mergeCache[Pair(b, a)]; + if (previous != null) return previous; + } + + PredictionContext rootMerge = mergeRoot(a, b, rootIsWildcard); + if (rootMerge != null) { + if (mergeCache != null) mergeCache[Pair(a, b)] = rootMerge; + return rootMerge; + } + + if (a.returnState == b.returnState) { + // a == b + PredictionContext parent = + merge(a.parent, b.parent, rootIsWildcard, mergeCache); + // if parent is same as existing a or b parent or reduced to a parent, return it + if (parent == a.parent) return a; // ax + bx = ax, if a=b + if (parent == b.parent) return b; // ax + bx = bx, if a=b + // else: ax + ay = a'[x,y] + // merge parents x and y, giving array node with x,y then remainders + // of those graphs. dup a, a' points at merged array + // new joined parent so create new singleton pointing to it, a' + PredictionContext a_ = + SingletonPredictionContext.create(parent, a.returnState); + if (mergeCache != null) mergeCache[Pair(a, b)] = a_; + return a_; + } else { + // a != b payloads differ + // see if we can collapse parents due to $+x parents if local ctx + PredictionContext singleParent = null; + if (a == b || (a.parent != null && a.parent == b.parent)) { + // ax + bx = [a,b]x + singleParent = a.parent; + } + if (singleParent != null) { + // parents are same + // sort payloads and use same parent + List payloads = [a.returnState, b.returnState]; + if (a.returnState > b.returnState) { + payloads[0] = b.returnState; + payloads[1] = a.returnState; + } + List parents = [singleParent, singleParent]; + PredictionContext a_ = new ArrayPredictionContext(parents, payloads); + if (mergeCache != null) mergeCache[Pair(a, b)] = a_; + return a_; + } + // parents differ and can't merge them. Just pack together + // into array; can't merge. + // ax + by = [ax,by] + List payloads = [a.returnState, b.returnState]; + List parents = [a.parent, b.parent]; + if (a.returnState > b.returnState) { + // sort by payload + payloads[0] = b.returnState; + payloads[1] = a.returnState; + parents = [b.parent, a.parent]; + } + PredictionContext a_ = new ArrayPredictionContext(parents, payloads); + if (mergeCache != null) mergeCache[Pair(a, b)] = a_; + return a_; + } + } + + /** + * Handle case where at least one of [a] or [b] is + * {@link #EMPTY}. In the following diagrams, the symbol {@code $} is used + * to represent {@link #EMPTY}. + * + *

Local-Context Merges

+ * + *

These local-context merge operations are used when [rootIsWildcard] + * is true.

+ * + *

{@link #EMPTY} is superset of any graph; return {@link #EMPTY}.
+ *

+ * + *

{@link #EMPTY} and anything is {@code #EMPTY}, so merged parent is + * {@code #EMPTY}; return left graph.
+ *

+ * + *

Special case of last merge if local context.
+ *

+ * + *

Full-Context Merges

+ * + *

These full-context merge operations are used when [rootIsWildcard] + * is false.

+ * + *

+ * + *

Must keep all contexts; {@link #EMPTY} in array is a special value (and + * null parent).
+ *

+ * + *

+ * + * @param a the first [SingletonPredictionContext] + * @param b the second [SingletonPredictionContext] + * @param rootIsWildcard [true] if this is a local-context merge, + * otherwise false to indicate a full-context merge + */ + static PredictionContext mergeRoot(SingletonPredictionContext a, + SingletonPredictionContext b, bool rootIsWildcard) { + if (rootIsWildcard) { + if (a == EMPTY) return EMPTY; // * + b = * + if (b == EMPTY) return EMPTY; // a + * = * + } else { + if (a == EMPTY && b == EMPTY) return EMPTY; // $ + $ = $ + if (a == EMPTY) { + // $ + x = [x,$] + List payloads = [b.returnState, EMPTY_RETURN_STATE]; + List parents = [b.parent, null]; + PredictionContext joined = + new ArrayPredictionContext(parents, payloads); + return joined; + } + if (b == EMPTY) { + // x + $ = [x,$] ($ is always last if present) + List payloads = [a.returnState, EMPTY_RETURN_STATE]; + final parents = [a.parent, null]; + PredictionContext joined = + new ArrayPredictionContext(parents, payloads); + return joined; + } + } + return null; + } + + /** + * Merge two [ArrayPredictionContext] instances. + * + *

Different tops, different parents.
+ *

+ * + *

Shared top, same parents.
+ *

+ * + *

Shared top, different parents.
+ *

+ * + *

Shared top, all shared parents.
+ *

+ * + *

Equal tops, merge parents and reduce top to + * [SingletonPredictionContext].
+ *

+ */ + static PredictionContext mergeArrays( + ArrayPredictionContext a, + ArrayPredictionContext b, + bool rootIsWildcard, + Map, PredictionContext> + mergeCache) { + if (mergeCache != null) { + PredictionContext previous = mergeCache[Pair(a, b)]; + if (previous != null) return previous; + previous = mergeCache[Pair(b, a)]; + if (previous != null) return previous; + } + + // merge sorted payloads a + b => M + int i = 0; // walks a + int j = 0; // walks b + int k = 0; // walks target M array + + List mergedReturnStates = List( + a.returnStates.length + b.returnStates.length); // TODO Will it grow? + var mergedParents = List( + a.returnStates.length + b.returnStates.length); // TODO Will it grow? + // walk and merge to yield mergedParents, mergedReturnStates + while (i < a.returnStates.length && j < b.returnStates.length) { + PredictionContext a_parent = a.parents[i]; + PredictionContext b_parent = b.parents[j]; + if (a.returnStates[i] == b.returnStates[j]) { + // same payload (stack tops are equal), must yield merged singleton + int payload = a.returnStates[i]; + // $+$ = $ + bool both$ = payload == EMPTY_RETURN_STATE && + a_parent == null && + b_parent == null; + bool ax_ax = (a_parent != null && b_parent != null) && + a_parent == b_parent; // ax+ax -> ax + if (both$ || ax_ax) { + mergedParents[k] = a_parent; // choose left + mergedReturnStates[k] = payload; + } else { + // ax+ay -> a'[x,y] + PredictionContext mergedParent = + merge(a_parent, b_parent, rootIsWildcard, mergeCache); + mergedParents[k] = mergedParent; + mergedReturnStates[k] = payload; + } + i++; // hop over left one as usual + j++; // but also skip one in right side since we merge + } else if (a.returnStates[i] < b.returnStates[j]) { + // copy a[i] to M + mergedParents[k] = a_parent; + mergedReturnStates[k] = a.returnStates[i]; + i++; + } else { + // b > a, copy b[j] to M + mergedParents[k] = b_parent; + mergedReturnStates[k] = b.returnStates[j]; + j++; + } + k++; + } + + // copy over any payloads remaining in either array + if (i < a.returnStates.length) { + for (int p = i; p < a.returnStates.length; p++) { + mergedParents[k] = a.parents[p]; + mergedReturnStates[k] = a.returnStates[p]; + k++; + } + } else { + for (int p = j; p < b.returnStates.length; p++) { + mergedParents[k] = b.parents[p]; + mergedReturnStates[k] = b.returnStates[p]; + k++; + } + } + + // trim merged if we combined a few that had same stack tops + if (k < mergedParents.length) { + // write index < last position; trim + if (k == 1) { + // for just one merged element, return singleton top + PredictionContext a_ = SingletonPredictionContext.create( + mergedParents[0], mergedReturnStates[0]); + if (mergeCache != null) mergeCache[Pair(a, b)] = a_; + return a_; + } + mergedParents = List(k)..setRange(0, k, mergedParents); + mergedReturnStates = List(k)..setRange(0, k, mergedReturnStates); + } + + PredictionContext M = + new ArrayPredictionContext(mergedParents, mergedReturnStates); + + // if we created same array as a or b, return that instead + // TODO: track whether this is possible above during merge sort for speed + if (M == a) { + if (mergeCache != null) mergeCache[Pair(a, b)] = a; + return a; + } + if (M == b) { + if (mergeCache != null) mergeCache[Pair(a, b)] = b; + return b; + } + + combineCommonParents(mergedParents); + + if (mergeCache != null) mergeCache[Pair(a, b)] = M; + return M; + } + + /** + * Make pass over all M [parents]; merge any {@code equals()} + * ones. + */ + static void combineCommonParents(List parents) { + Map uniqueParents = + new Map(); + + for (int p = 0; p < parents.length; p++) { + PredictionContext parent = parents[p]; + if (!uniqueParents.containsKey(parent)) { + // don't replace + uniqueParents[parent] = parent; + } + } + + for (int p = 0; p < parents.length; p++) { + parents[p] = uniqueParents[parents[p]]; + } + } + + static String toDOTString(PredictionContext context) { + if (context == null) return ""; + StringBuffer buf = new StringBuffer(); + buf.write("digraph G {\n"); + buf.write("rankdir=LR;\n"); + + List nodes = getAllContextNodes(context); + nodes.sort((PredictionContext o1, PredictionContext o2) { + return o1.id - o2.id; + }); + + for (PredictionContext current in nodes) { + if (current is SingletonPredictionContext) { + String s = current.id.toString(); + buf.write(" s"); + buf.write(s); + String returnState = current.getReturnState(0).toString(); + if (current is EmptyPredictionContext) returnState = r"$"; + buf.write(" [label=\""); + buf.write(returnState); + buf.write("\"];\n"); + continue; + } + ArrayPredictionContext arr = current; + buf.write(" s"); + buf.write(arr.id); + buf.write(" [shape=box, label=\""); + buf.write("["); + bool first = true; + for (int inv in arr.returnStates) { + if (!first) buf.write(", "); + if (inv == EMPTY_RETURN_STATE) + buf.write(r"$"); + else + buf.write(inv); + first = false; + } + buf.write("]"); + buf.write("\"];\n"); + } + + for (PredictionContext current in nodes) { + if (current == EMPTY) continue; + for (int i = 0; i < current.length; i++) { + if (current.getParent(i) == null) continue; + String s = current.id.toString(); + buf.write(" s"); + buf.write(s); + buf.write("->"); + buf.write("s"); + buf.write(current.getParent(i).id); + if (current.length > 1) + buf.write(" [label=\"parent[$i]\"];\n"); + else + buf.write(";\n"); + } + } + + buf.write("}\n"); + return buf.toString(); + } + + // From Sam + static PredictionContext getCachedContext( + PredictionContext context, + PredictionContextCache contextCache, + Map visited) { + if (context.isEmpty) { + return context; + } + + PredictionContext existing = visited[context]; + if (existing != null) { + return existing; + } + + existing = contextCache[context]; + if (existing != null) { + visited[context] = existing; + return existing; + } + + bool changed = false; + var parents = List(context.length); + for (int i = 0; i < parents.length; i++) { + PredictionContext parent = + getCachedContext(context.getParent(i), contextCache, visited); + if (changed || parent != context.getParent(i)) { + if (!changed) { + parents = List(context.length); + for (int j = 0; j < context.length; j++) { + parents[j] = context.getParent(j); + } + + changed = true; + } + + parents[i] = parent; + } + } + + if (!changed) { + contextCache.add(context); + visited[context] = context; + return context; + } + + PredictionContext updated; + if (parents.length == 0) { + updated = EMPTY; + } else if (parents.length == 1) { + updated = SingletonPredictionContext.create( + parents[0], context.getReturnState(0)); + } else { + ArrayPredictionContext arrayPredictionContext = context; + updated = new ArrayPredictionContext( + parents, arrayPredictionContext.returnStates); + } + + contextCache.add(updated); + visited[updated] = updated; + visited[context] = updated; + + return updated; + } + +// // extra structures, but cut/paste/morphed works, so leave it. +// // seems to do a breadth-first walk +// static List getAllNodes(PredictionContext context) { +// Map visited = +// new IdentityHashMap(); +// Deque workList = new ArrayDeque(); +// workList.add(context); +// visited.put(context, context); +// List nodes = new ArrayList(); +// while (!workList.isEmpty) { +// PredictionContext current = workList.pop(); +// nodes.add(current); +// for (int i = 0; i < current.length; i++) { +// PredictionContext parent = current.getParent(i); +// if ( parent!=null && visited.put(parent, parent) == null) { +// workList.push(parent); +// } +// } +// } +// return nodes; +// } + + // ter's recursive version of Sam's getAllNodes() + static List getAllContextNodes(PredictionContext context) { + List nodes = List(); + Map visited = + Map(); + getAllContextNodes_(context, nodes, visited); + return nodes; + } + + static void getAllContextNodes_( + PredictionContext context, + List nodes, + Map visited) { + if (context == null || visited.containsKey(context)) return; + visited[context] = context; + nodes.add(context); + for (int i = 0; i < context.length; i++) { + getAllContextNodes_(context.getParent(i), nodes, visited); + } + } + + // FROM SAM + List toStrings( + Recognizer recognizer, PredictionContext stop, int currentState) { + List result = []; + + outer: + for (int perm = 0;; perm++) { + int offset = 0; + bool last = true; + PredictionContext p = this; + int stateNumber = currentState; + StringBuffer localBuffer = new StringBuffer(); + localBuffer.write("["); + while (!p.isEmpty && p != stop) { + int index = 0; + if (p.length > 0) { + int bits = 1; + while ((1 << bits) < p.length) { + bits++; + } + + int mask = (1 << bits) - 1; + index = (perm >> offset) & mask; + last &= index >= p.length - 1; + if (index >= p.length) { + continue outer; + } + offset += bits; + } + + if (recognizer != null) { + if (localBuffer.length > 1) { + // first char is '[', if more than that this isn't the first rule + localBuffer.write(' '); + } + + ATN atn = recognizer.getATN(); + ATNState s = atn.states[stateNumber]; + String ruleName = recognizer.ruleNames[s.ruleIndex]; + localBuffer.write(ruleName); + } else if (p.getReturnState(index) != EMPTY_RETURN_STATE) { + if (!p.isEmpty) { + if (localBuffer.length > 1) { + // first char is '[', if more than that this isn't the first rule + localBuffer.write(' '); + } + + localBuffer.write(p.getReturnState(index)); + } + } + stateNumber = p.getReturnState(index); + p = p.getParent(index); + } + localBuffer.write("]"); + result.add(localBuffer.toString()); + + if (last) { + break; + } + } + + return result; + } +} + +class SingletonPredictionContext extends PredictionContext { + final PredictionContext parent; + final int returnState; + + SingletonPredictionContext(PredictionContext this.parent, this.returnState) + : super(parent != null + ? PredictionContext.calculateHashCode([parent], [returnState]) + : PredictionContext.calculateEmptyHashCode()) { + assert(this.returnState != ATNState.INVALID_STATE_NUMBER); + } + + static SingletonPredictionContext create( + PredictionContext parent, int returnState) { + if (returnState == PredictionContext.EMPTY_RETURN_STATE && parent == null) { + // someone can pass in the bits of an array ctx that mean $ + return PredictionContext.EMPTY; + } + return new SingletonPredictionContext(parent, returnState); + } + + int get length { + return 1; + } + + PredictionContext getParent(int index) { + assert(index == 0); + return parent; + } + + int getReturnState(int index) { + assert(index == 0); + return returnState; + } + + bool operator ==(Object o) { + if (identical(this, o)) { + return true; + } else if (o is SingletonPredictionContext) { + if (this.hashCode != o.hashCode) { + return false; // can't be same if hash is different + } + + SingletonPredictionContext s = o; + return returnState == s.returnState && + (parent != null && parent == s.parent); + } + return false; + } + + String toString() { + String up = parent != null ? parent.toString() : ""; + if (up.length == 0) { + if (returnState == PredictionContext.EMPTY_RETURN_STATE) { + return r"$"; + } + return returnState.toString(); + } + return "$returnState $up"; + } +} + +class EmptyPredictionContext extends SingletonPredictionContext { + EmptyPredictionContext() : super(null, PredictionContext.EMPTY_RETURN_STATE); + + bool get isEmpty { + return true; + } + + int get length { + return 1; + } + + PredictionContext getParent(int index) { + return null; + } + + int getReturnState(int index) { + return returnState; + } + + String toString() { + return r"$"; + } +} + +class ArrayPredictionContext extends PredictionContext { + /** Parent can be null only if full ctx mode and we make an array + * from {@link #EMPTY} and non-empty. We merge {@link #EMPTY} by using null parent and + * returnState == {@link #EMPTY_RETURN_STATE}. + */ + List parents; + + /** Sorted for merge, no duplicates; if present, + * {@link #EMPTY_RETURN_STATE} is always last. + */ + List returnStates; + + ArrayPredictionContext.of(SingletonPredictionContext a) + : this([a.parent], [a.returnState]); + + ArrayPredictionContext( + List parents, List returnStates) + : super(PredictionContext.calculateHashCode(parents, returnStates)) { + assert(parents != null && parents.length > 0); + assert(returnStates != null && returnStates.length > 0); +// System.err.println("CREATE ARRAY: "+Arrays.toString(parents)+", "+Arrays.toString(returnStates)); + this.parents = parents; + this.returnStates = returnStates; + } + + bool get isEmpty { + // since EMPTY_RETURN_STATE can only appear in the last position, we + // don't need to verify that size==1 + return returnStates[0] == PredictionContext.EMPTY_RETURN_STATE; + } + + int get length { + return returnStates.length; + } + + PredictionContext getParent(int index) { + return parents[index]; + } + + int getReturnState(int index) { + return returnStates[index]; + } + +// int findReturnState(int returnState) { +// return Arrays.binarySearch(returnStates, returnState); +// } + + bool operator ==(Object o) { + if (identical(this, o)) { + return true; + } else if (o is ArrayPredictionContext) { + if (this.hashCode != o.hashCode) { + return false; // can't be same if hash is different + } + + ArrayPredictionContext a = o; + return ListEquality().equals(returnStates, a.returnStates) && + ListEquality().equals(parents, a.parents); + } + return false; + } + + String toString() { + if (isEmpty) return "[]"; + StringBuffer buf = new StringBuffer(); + buf.write("["); + for (int i = 0; i < returnStates.length; i++) { + if (i > 0) buf.write(", "); + if (returnStates[i] == PredictionContext.EMPTY_RETURN_STATE) { + buf.write(r"$"); + continue; + } + buf.write(returnStates[i]); + if (parents[i] != null) { + buf.write(' '); + buf.write(parents[i].toString()); + } else { + buf.write("null"); + } + } + buf.write("]"); + return buf.toString(); + } +} diff --git a/runtime/Dart/lib/src/recognizer.dart b/runtime/Dart/lib/src/recognizer.dart new file mode 100644 index 000000000..ff575c4ca --- /dev/null +++ b/runtime/Dart/lib/src/recognizer.dart @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'vocabulary.dart'; +import 'atn/atn.dart'; +import 'error/error.dart'; +import 'input_stream.dart'; +import 'rule_context.dart'; +import 'token.dart'; +import 'token_factory.dart'; +import 'util/utils.dart'; + +abstract class Recognizer { + static const EOF = -1; + + static final Map> tokenTypeMapCache = {}; + static final Map, Map> ruleIndexMapCache = {}; + List _listeners = [ConsoleErrorListener.INSTANCE]; + + /// The ATN interpreter used by the recognizer for prediction. + ATNInterpreter interpreter; + int _stateNumber = -1; + + List get ruleNames; + + /** + * Get the vocabulary used by the recognizer. + * + * @return A [Vocabulary] instance providing information about the + * vocabulary used by the grammar. + */ + Vocabulary get vocabulary; + + /** + * Get a map from token names to token types. + * + *

Used for XPath and tree pattern compilation.

+ */ + Map get tokenTypeMap { + Vocabulary _vocabulary = vocabulary; + + Map result = tokenTypeMapCache[_vocabulary]; + if (result == null) { + result = {}; + for (int i = 0; i <= getATN().maxTokenType; i++) { + String literalName = _vocabulary.getLiteralName(i); + if (literalName != null) { + result[literalName] = i; + } + + String symbolicName = _vocabulary.getSymbolicName(i); + if (symbolicName != null) { + result[symbolicName] = i; + } + } + + result["EOF"] = Token.EOF; + result = Map.unmodifiable(result); + tokenTypeMapCache[_vocabulary] = result; + } + + return result; + } + + /** + * Get a map from rule names to rule indexes. + * + *

Used for XPath and tree pattern compilation.

+ */ + Map get ruleIndexMap { + final _ruleNames = ruleNames; + if (_ruleNames == null) { + throw UnsupportedError( + "The current recognizer does not provide a list of rule names."); + } + + var result = ruleIndexMapCache[_ruleNames]; + if (result == null) { + result = Map.unmodifiable(toMap(_ruleNames)); + ruleIndexMapCache[_ruleNames] = result; + } + + return result; + } + + int getTokenType(String tokenName) { + final ttype = tokenTypeMap[tokenName]; + if (ttype != null) return ttype; + return Token.INVALID_TYPE; + } + + /** + * If this recognizer was generated, it will have a serialized ATN + * representation of the grammar. + * + *

For interpreters, we don't know their serialized ATN despite having + * created the interpreter from it.

+ */ + String get serializedATN { + throw new UnsupportedError("there is no serialized ATN"); + } + + /** For debugging and other purposes, might want the grammar name. + * Have ANTLR generate an implementation for this method. + */ + String get grammarFileName; + + /** + * Get the [ATN] used by the recognizer for prediction. + * + * @return The [ATN] used by the recognizer for prediction. + */ + ATN getATN(); + + /** If profiling during the parse/lex, this will return DecisionInfo records + * for each decision in recognizer in a ParseInfo object. + * + * @since 4.3 + */ + ParseInfo get parseInfo { + return null; + } + + /** What is the error header, normally line/character position information? */ + String getErrorHeader(RecognitionException e) { + int line = e.offendingToken.line; + int charPositionInLine = e.offendingToken.charPositionInLine; + return "line $line:$charPositionInLine"; + } + + /** + * @exception NullPointerException if [listener] is null. + */ + void addErrorListener(ErrorListener listener) { + if (listener == null) { + throw new ArgumentError.notNull("listener"); + } + + _listeners.add(listener); + } + + void removeErrorListener(ErrorListener listener) { + _listeners.remove(listener); + } + + void removeErrorListeners() { + _listeners.clear(); + } + + List get errorListeners { + return _listeners; + } + + ErrorListener get errorListenerDispatch { + return new ProxyErrorListener(errorListeners); + } + + // subclass needs to override these if there are sempreds or actions + // that the ATN interp needs to execute + bool sempred(RuleContext _localctx, int ruleIndex, int actionIndex) { + return true; + } + + bool precpred(RuleContext localctx, int precedence) { + return true; + } + + void action(RuleContext _localctx, int ruleIndex, int actionIndex) {} + + int get state { + return _stateNumber; + } + + /** Indicate that the recognizer has changed internal state that is + * consistent with the ATN state passed in. This way we always know + * where we are in the ATN as the parser goes along. The rule + * context objects form a stack that lets us see the stack of + * invoking rules. Combine this and we have complete ATN + * configuration information. + */ + void set state(int atnState) { +// System.err.println("setState "+atnState); + _stateNumber = atnState; +// if ( traceATNStates ) _ctx.trace(atnState); + } + + IntStream get inputStream; + + void set inputStream(IntStream input); + + TokenFactory get tokenFactory; + + void set tokenFactory(TokenFactory input); +} diff --git a/runtime/Dart/lib/src/rule_context.dart b/runtime/Dart/lib/src/rule_context.dart new file mode 100644 index 000000000..36a3aeb68 --- /dev/null +++ b/runtime/Dart/lib/src/rule_context.dart @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'atn/atn.dart'; +import 'interval_set.dart'; +import 'parser.dart'; +import 'parser_rule_context.dart'; +import 'recognizer.dart'; +import 'tree/tree.dart'; + +/** A rule context is a record of a single rule invocation. + * + * We form a stack of these context objects using the parent + * pointer. A parent pointer of null indicates that the current + * context is the bottom of the stack. The ParserRuleContext subclass + * as a children list so that we can turn this data structure into a + * tree. + * + * The root node always has a null pointer and invokingState of -1. + * + * Upon entry to parsing, the first invoked rule function creates a + * context object (a subclass specialized for that rule such as + * SContext) and makes it the root of a parse tree, recorded by field + * Parser._ctx. + * + * public final SContext s() throws RecognitionException { + * SContext _localctx = new SContext(_ctx, getState()); <-- create new node + * enterRule(_localctx, 0, RULE_s); <-- push it + * ... + * exitRule(); <-- pop back to _localctx + * return _localctx; + * } + * + * A subsequent rule invocation of r from the start rule s pushes a + * new context object for r whose parent points at s and use invoking + * state is the state with r emanating as edge label. + * + * The invokingState fields from a context object to the root + * together form a stack of rule indication states where the root + * (bottom of the stack) has a -1 sentinel value. If we invoke start + * symbol s then call r1, which calls r2, the would look like + * this: + * + * SContext[-1] <- root node (bottom of the stack) + * R1Context[p] <- p in rule s called r1 + * R2Context[q] <- q in rule r1 called r2 + * + * So the top of the stack, _ctx, represents a call to the current + * rule and it holds the return address from another rule that invoke + * to this rule. To invoke a rule, we must always have a current context. + * + * The parent contexts are useful for computing lookahead sets and + * getting error information. + * + * These objects are used during parsing and prediction. + * For the special case of parsers, we use the subclass + * ParserRuleContext. + * + * @see ParserRuleContext + */ +abstract class RuleContext extends RuleNode { + /// What context invoked this rule? + RuleContext parent = null; + + /// What state invoked the rule associated with this context? + /// The "return address" is the followState of invokingState + /// If parent is null, this should be -1. + int invokingState; + + RuleContext({this.parent, this.invokingState}) { + invokingState = invokingState ?? -1; + } + + int depth() { + var n = 0; + var p = this; + while (p != null) { + p = p.parent; + n++; + } + return n; + } + + /// A context is empty if there is no invoking state; meaning nobody call + /// current context. + bool get isEmpty => invokingState == -1; + + /// satisfy the ParseTree / SyntaxTree interface + Interval get sourceInterval => Interval.INVALID; + + RuleContext get ruleContext => this; + + RuleContext get payload => this; + + /** + * Return the combined text of all child nodes. This method only considers + * tokens which have been added to the parse tree. + *

+ * Since tokens on hidden channels (e.g. whitespace or comments) are not + * added to the parse trees, they will not appear in the output of this + * method. + */ + String get text { + if (childCount == 0) { + return ""; + } + + final builder = new StringBuffer(); + for (int i = 0; i < childCount; i++) { + builder.write(getChild(i).text); + } + + return builder.toString(); + } + + int get ruleIndex => -1; + + /// For rule associated with this parse tree internal node, return + /// the outer alternative number used to match the input. Default + /// implementation does not compute nor store this alt num. Create + /// a subclass of ParserRuleContext with backing field and set + /// option contextSuperClass. + /// to set it. + int get altNumber => ATN.INVALID_ALT_NUMBER; + + /// Set the outer alternative number for this context node. Default + /// implementation does nothing to avoid backing field overhead for + /// trees that don't need it. Create + /// a subclass of ParserRuleContext with backing field and set + /// option contextSuperClass. + set altNumber(int altNumber) {} + + ParseTree getChild(int i) { + return null; + } + + int get childCount => 0; + + T accept(ParseTreeVisitor visitor) { + return visitor.visitChildren(this); + } + + /// Print out a whole tree, not just a node, in LISP format + /// (root child1 .. childN). Print just a node if this is a leaf. + /// + String toStringTree({List ruleNames, Parser parser}) { + return Trees.toStringTree(this, ruleNames: ruleNames, recog: parser); + } + + String toString( + {List ruleNames, Recognizer recog, RuleContext stop}) { + ruleNames = ruleNames ?? recog?.ruleNames; + final buf = new StringBuffer(); + var p = this; + buf.write("["); + while (p != null && p != stop) { + if (ruleNames == null) { + if (!p.isEmpty) { + buf.write(p.invokingState); + } + } else { + int ruleIndex = p.ruleIndex; + String ruleName = ruleIndex >= 0 && ruleIndex < ruleNames.length + ? ruleNames[ruleIndex] + : ruleIndex.toString(); + buf.write(ruleName); + } + + if (p.parent != null && + (ruleNames != null || !p.parent.isEmpty)) { + buf.write(" "); + } + + p = p.parent; + } + + buf.write("]"); + return buf.toString(); + } + + static final EMPTY = new ParserRuleContext(); +} diff --git a/runtime/Dart/lib/src/runtime_meta_data.dart b/runtime/Dart/lib/src/runtime_meta_data.dart new file mode 100644 index 000000000..1d5f409be --- /dev/null +++ b/runtime/Dart/lib/src/runtime_meta_data.dart @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:developer'; +import 'dart:math' as math; + +import 'package:logging/logging.dart'; + +/** + * This class provides access to the current version of the ANTLR 4 runtime + * library as compile-time and runtime constants, along with methods for + * checking for matching version numbers and notifying listeners in the case + * where a version mismatch is detected. + * + *

+ * The runtime version information is provided by {@link #VERSION} and + * {@link #getRuntimeVersion()}. Detailed information about these values is + * provided in the documentation for each member.

+ * + *

+ * The runtime version check is implemented by {@link #checkVersion}. Detailed + * information about incorporating this call into user code, as well as its use + * in generated code, is provided in the documentation for the method.

+ * + *

+ * Version strings x.y and x.y.z are considered "compatible" and no error + * would be generated. Likewise, version strings x.y-SNAPSHOT and x.y.z are + * considered "compatible" because the major and minor components x.y + * are the same in each.

+ * + *

+ * To trap any error messages issued by this code, use System.setErr() + * in your main() startup code. + *

+ * + * @since 4.3 + */ +class RuntimeMetaData { + /** + * A compile-time constant containing the current version of the ANTLR 4 + * runtime library. + * + *

+ * This compile-time constant value allows generated parsers and other + * libraries to include a literal reference to the version of the ANTLR 4 + * runtime library the code was compiled against. At each release, we + * change this value.

+ * + *

Version numbers are assumed to have the form + * + * major.minor.patch.revision-suffix, + * + * with the individual components defined as follows.

+ * + *
    + *
  • major is a required non-negative integer, and is equal to + * {@code 4} for ANTLR 4.
    • + *
  • minor is a required non-negative integer.
    • + *
  • patch is an optional non-negative integer. When + * patch is omitted, the {@code .} (dot) appearing before it is + * also omitted.
    • + *
  • revision is an optional non-negative integer, and may only + * be included when patch is also included. When revision + * is omitted, the {@code .} (dot) appearing before it is also omitted.
    • + *
  • suffix is an optional string. When suffix is + * omitted, the {@code -} (hyphen-minus) appearing before it is also + * omitted.
    • + *
+ */ + static final String VERSION = "4.7.2"; + + /** + * Gets the currently executing version of the ANTLR 4 runtime library. + * + *

+ * This method provides runtime access to the {@link #VERSION} field, as + * opposed to directly referencing the field as a compile-time constant.

+ * + * @return The currently executing version of the ANTLR 4 library + */ + + static String get runtimeVersion { + return VERSION; + } + + /** + * This method provides the ability to detect mismatches between the version + * of ANTLR 4 used to generate a parser, the version of the ANTLR runtime a + * parser was compiled against, and the version of the ANTLR runtime which + * is currently executing. + * + *

+ * The version check is designed to detect the following two specific + * scenarios.

+ * + *
    + *
  • The ANTLR Tool version used for code generation does not match the + * currently executing runtime version.
    • + *
  • The ANTLR Runtime version referenced at the time a parser was + * compiled does not match the currently executing runtime version.
    • + *
+ * + *

+ * Starting with ANTLR 4.3, the code generator emits a call to this method + * using two constants in each generated lexer and parser: a hard-coded + * constant indicating the version of the tool used to generate the parser + * and a reference to the compile-time constant {@link #VERSION}. At + * runtime, this method is called during the initialization of the generated + * parser to detect mismatched versions, and notify the registered listeners + * prior to creating instances of the parser.

+ * + *

+ * This method does not perform any detection or filtering of semantic + * changes between tool and runtime versions. It simply checks for a + * version match and emits an error to stderr if a difference + * is detected.

+ * + *

+ * Note that some breaking changes between releases could result in other + * types of runtime exceptions, such as a [LinkageError], prior to + * calling this method. In these cases, the underlying version mismatch will + * not be reported here. This method is primarily intended to + * notify users of potential semantic changes between releases that do not + * result in binary compatibility problems which would be detected by the + * class loader. As with semantic changes, changes that break binary + * compatibility between releases are mentioned in the release notes + * accompanying the affected release.

+ * + *

+ * Additional note for target developers: The version check + * implemented by this class is designed to address specific compatibility + * concerns that may arise during the execution of Java applications. Other + * targets should consider the implementation of this method in the context + * of that target's known execution environment, which may or may not + * resemble the design provided for the Java target.

+ * + * @param generatingToolVersion The version of the tool used to generate a parser. + * This value may be null when called from user code that was not generated + * by, and does not reference, the ANTLR 4 Tool itself. + * @param compileTimeVersion The version of the runtime the parser was + * compiled against. This should always be passed using a direct reference + * to {@link #VERSION}. + */ + static void checkVersion( + String generatingToolVersion, String compileTimeVersion) { + String runtimeVersion = VERSION; + bool runtimeConflictsWithGeneratingTool = false; + bool runtimeConflictsWithCompileTimeTool = false; + + if (generatingToolVersion != null) { + runtimeConflictsWithGeneratingTool = + !(runtimeVersion == generatingToolVersion) && + !(getMajorMinorVersion(runtimeVersion) == + getMajorMinorVersion(generatingToolVersion)); + } + + runtimeConflictsWithCompileTimeTool = + !(runtimeVersion == compileTimeVersion) && + !(getMajorMinorVersion(runtimeVersion) == + getMajorMinorVersion(compileTimeVersion)); + + if (runtimeConflictsWithGeneratingTool) { + log("ANTLR Tool version $generatingToolVersion used for code generation does not match the current runtime version $runtimeVersion", + level: Level.SEVERE.value); + } + if (runtimeConflictsWithCompileTimeTool) { + log("ANTLR Runtime version $compileTimeVersion used for parser compilation does not match the current runtime version $runtimeVersion", + level: Level.SEVERE.value); + } + } + + /** + * Gets the major and minor version numbers from a version string. For + * details about the syntax of the input [version]. + * E.g., from x.y.z return x.y. + * + * @param version The complete version string. + * @return A string of the form major.minor containing + * only the major and minor components of the version string. + */ + static String getMajorMinorVersion(String version) { + int firstDot = version.indexOf('.'); + int secondDot = firstDot >= 0 ? version.indexOf('.', firstDot + 1) : -1; + int firstDash = version.indexOf('-'); + int referenceLength = version.length; + if (secondDot >= 0) { + referenceLength = math.min(referenceLength, secondDot); + } + + if (firstDash >= 0) { + referenceLength = math.min(referenceLength, firstDash); + } + + return version.substring(0, referenceLength); + } +} diff --git a/runtime/Dart/lib/src/token.dart b/runtime/Dart/lib/src/token.dart new file mode 100644 index 000000000..4667896b0 --- /dev/null +++ b/runtime/Dart/lib/src/token.dart @@ -0,0 +1,479 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'input_stream.dart'; +import 'interval_set.dart'; +import 'misc/pair.dart'; +import 'recognizer.dart'; +import 'token_source.dart'; + +/** A token has properties: text, type, line, character position in the line + * (so we can ignore tabs), token channel, index, and source from which + * we obtained this token. + */ +abstract class Token { + static const int INVALID_TYPE = 0; + + /** During lookahead operations, this "token" signifies we hit rule end ATN state + * and did not follow it despite needing to. + */ + static const int EPSILON = -2; + + static const int MIN_USER_TOKEN_TYPE = 1; + + static const int EOF = IntStream.EOF; + + /** All tokens go to the parser (unless skip() is called in that rule) + * on a particular "channel". The parser tunes to a particular channel + * so that whitespace etc... can go to the parser on a "hidden" channel. + */ + static const int DEFAULT_CHANNEL = 0; + + /** Anything on different channel than DEFAULT_CHANNEL is not parsed + * by parser. + */ + static const int HIDDEN_CHANNEL = 1; + + /** + * This is the minimum constant value which can be assigned to a + * user-defined token channel. + * + *

+ * The non-negative numbers less than {@link #MIN_USER_CHANNEL_VALUE} are + * assigned to the predefined channels {@link #DEFAULT_CHANNEL} and + * {@link #HIDDEN_CHANNEL}.

+ * + * @see Token#getChannel() + */ + static const int MIN_USER_CHANNEL_VALUE = 2; + + /** + * Get the text of the token. + */ + String get text; + + /** Get the token type of the token */ + int get type; + + /** The line number on which the 1st character of this token was matched, + * line=1..n + */ + int get line; + + /** The index of the first character of this token relative to the + * beginning of the line at which it occurs, 0..n-1 + */ + int get charPositionInLine; + + /** Return the channel this token. Each token can arrive at the parser + * on a different channel, but the parser only "tunes" to a single channel. + * The parser ignores everything not on DEFAULT_CHANNEL. + */ + int get channel; + + /** An index from 0..n-1 of the token object in the input stream. + * This must be valid in order to print token streams and + * use TokenRewriteStream. + * + * Return -1 to indicate that this token was conjured up since + * it doesn't have a valid index. + */ + int get tokenIndex; + + /** The starting character index of the token + * This method is optional; return -1 if not implemented. + */ + int get startIndex; + + /** The last character index of the token. + * This method is optional; return -1 if not implemented. + */ + int get stopIndex; + + /** Gets the [TokenSource] which created this token. + */ + TokenSource get tokenSource; + + /** + * Gets the [CharStream] from which this token was derived. + */ + CharStream get inputStream; +} + +abstract class WritableToken extends Token { + void set text(String text); + + void set type(int ttype); + + void set line(int line); + + void set charPositionInLine(int pos); + + void set channel(int channel); + + void set tokenIndex(int index); +} + +class CommonToken extends WritableToken { + /** + * An empty [Pair] which is used as the default value of + * {@link #source} for tokens that do not have a source. + */ + static const Pair EMPTY_SOURCE = + const Pair(null, null); + + int type; + + int line; + + int charPositionInLine = -1; // set to invalid position + + int channel = Token.DEFAULT_CHANNEL; + + /** + * These properties share a field to reduce the memory footprint of + * [CommonToken]. Tokens created by a [CommonTokenFactory] from + * the same source and input stream share a reference to the same + * [Pair] containing these values.

+ */ + Pair source; + + /** + * This is the backing field for {@link #getText} when the token text is + * explicitly set in the constructor or via {@link #setText}. + * + * @see #getText() + */ + String _text; + + int tokenIndex = -1; + + int startIndex; + + int stopIndex; + + /** + * Constructs a new [CommonToken] with the specified token type and + * text. + * + * @param type The token type. + * @param text The text of the token. + */ + CommonToken(this.type, + {this.source = EMPTY_SOURCE, + this.channel = Token.DEFAULT_CHANNEL, + this.startIndex, + this.stopIndex, + text}) { + this._text = text; + if (source.a != null) { + this.line = source.a.line; + this.charPositionInLine = source.a.charPositionInLine; + } + } + + /** + * Constructs a new [CommonToken] as a copy of another [Token]. + * + *

+ * If [oldToken] is also a [CommonToken] instance, the newly + * constructed token will share a reference to the {@link #text} field and + * the [Pair] stored in {@link #source}. Otherwise, {@link #text} will + * be assigned the result of calling {@link #getText}, and {@link #source} + * will be constructed from the result of {@link Token#getTokenSource} and + * {@link Token#getInputStream}.

+ * + * @param oldToken The token to copy. + */ + CommonToken.copy(Token oldToken) { + type = oldToken.type; + line = oldToken.line; + tokenIndex = oldToken.tokenIndex; + charPositionInLine = oldToken.charPositionInLine; + channel = oldToken.channel; + startIndex = oldToken.startIndex; + stopIndex = oldToken.stopIndex; + + if (oldToken is CommonToken) { + _text = oldToken.text; + source = oldToken.source; + } else { + _text = oldToken.text; + source = new Pair( + oldToken.tokenSource, oldToken.inputStream); + } + } + + String get text { + if (_text != null) { + return _text; + } + + CharStream input = inputStream; + if (input == null) return null; + int n = input.size; + if (startIndex < n && stopIndex < n) { + return input.getText(Interval.of(startIndex, stopIndex)); + } else { + return ""; + } + } + + /** + * Explicitly set the text for this token. If {code text} is not + * null, then {@link #getText} will return this value rather than + * extracting the text from the input. + * + * @param text The explicit text of the token, or null if the text + * should be obtained from the input along with the start and stop indexes + * of the token. + */ + void set text(String text) { + this._text = text; + } + + TokenSource get tokenSource { + return source.a; + } + + CharStream get inputStream { + return source.b; + } + + String toString([Recognizer r]) { + var txt = this.text; + if (txt != null) { + txt = txt + .replaceAll("\n", r"\n") + .replaceAll("\r", r"\r") + .replaceAll("\t", r"\t"); + } else { + txt = ""; + } + return "[@$tokenIndex,$startIndex:$stopIndex='$txt',<$type>" + + (this.channel > 0 ? ",channel=$channel" : "") + + ",$line:$charPositionInLine]"; + } +} + +/** + * A [Token] object representing an entire subtree matched by a parser + * rule; e.g., {@code }. These tokens are created for [TagChunk] + * chunks where the tag corresponds to a parser rule. + */ +class RuleTagToken implements Token { + /** + * Gets the name of the rule associated with this rule tag. + * + * @return The name of the parser rule associated with this rule tag. + */ + final String ruleName; + + /** + * The token type for the current token. This is the token type assigned to + * the bypass alternative for the rule during ATN deserialization. + */ + final int bypassTokenType; + + /** + * Gets the label associated with the rule tag. + * + * @return The name of the label associated with the rule tag, or + * null if this is an unlabeled rule tag. + */ + final String label; + + /** + * Constructs a new instance of [RuleTagToken] with the specified rule + * name, bypass token type, and label. + * + * @param ruleName The name of the parser rule this rule tag matches. + * @param bypassTokenType The bypass token type assigned to the parser rule. + * @param label The label associated with the rule tag, or null if + * the rule tag is unlabeled. + * + * @exception ArgumentError.value(value) if [ruleName] is null + * or empty. + */ + RuleTagToken(this.ruleName, this.bypassTokenType, [this.label]) { + if (ruleName == null || ruleName.isEmpty) { + throw new ArgumentError.value( + ruleName, "ruleName", "cannot be null or empty."); + } + } + + /** + * {@inheritDoc} + * + *

Rule tag tokens are always placed on the {@link #DEFAULT_CHANNEL}.

+ */ + + int get channel { + return Token.DEFAULT_CHANNEL; + } + + /** + * {@inheritDoc} + * + *

This method returns the rule tag formatted with {@code <} and {@code >} + * delimiters.

+ */ + + String get text { + if (label != null) { + return "<" + label + ":" + ruleName + ">"; + } + + return "<" + ruleName + ">"; + } + + /** + * {@inheritDoc} + * + *

Rule tag tokens have types assigned according to the rule bypass + * transitions created during ATN deserialization.

+ */ + + int get type { + return bypassTokenType; + } + + /** + * {@inheritDoc} + * + *

The implementation for [RuleTagToken] always returns 0.

+ */ + + int get line { + return 0; + } + + /** + * {@inheritDoc} + * + *

The implementation for [RuleTagToken] always returns -1.

+ */ + int get charPositionInLine { + return -1; + } + + /** + * {@inheritDoc} + * + *

The implementation for [RuleTagToken] always returns -1.

+ */ + int get tokenIndex { + return -1; + } + + /** + * {@inheritDoc} + * + *

The implementation for [RuleTagToken] always returns -1.

+ */ + int get startIndex { + return -1; + } + + /** + * {@inheritDoc} + * + *

The implementation for [RuleTagToken] always returns -1.

+ */ + + int get stopIndex { + return -1; + } + + /** + * {@inheritDoc} + * + *

The implementation for [RuleTagToken] always returns null.

+ */ + + TokenSource get tokenSource { + return null; + } + + /** + * {@inheritDoc} + * + *

The implementation for [RuleTagToken] always returns null.

+ */ + + CharStream get inputStream { + return null; + } + + /** + * {@inheritDoc} + * + *

The implementation for [RuleTagToken] returns a string of the form + * {@code ruleName:bypassTokenType}.

+ */ + + String toString() { + return ruleName + ":$bypassTokenType"; + } +} + +/** + * A [Token] object representing a token of a particular type; e.g., + * {@code }. These tokens are created for [TagChunk] chunks where the + * tag corresponds to a lexer rule or token type. + */ +class TokenTagToken extends CommonToken { + /** + * Gets the token name. + * @return The token name. + */ + final String tokenName; + + /** + * Gets the label associated with the rule tag. + * + * @return The name of the label associated with the rule tag, or + * null if this is an unlabeled rule tag. + */ + final String label; + + /** + * Constructs a new instance of [TokenTagToken] with the specified + * token name, type, and label. + * + * @param tokenName The token name. + * @param type The token type. + * @param label The label associated with the token tag, or null if + * the token tag is unlabeled. + */ + TokenTagToken(this.tokenName, type, [this.label]) : super(type); + + /** + * {@inheritDoc} + * + *

The implementation for [TokenTagToken] returns the token tag + * formatted with {@code <} and {@code >} delimiters.

+ */ + + String get text { + if (label != null) { + return "<" + label + ":" + tokenName + ">"; + } + + return "<" + tokenName + ">"; + } + + /** + * {@inheritDoc} + * + *

The implementation for [TokenTagToken] returns a string of the form + * {@code tokenName:type}.

+ */ + + String toString([recognizer]) { + return tokenName + ":$type"; + } +} diff --git a/runtime/Dart/lib/src/token_factory.dart b/runtime/Dart/lib/src/token_factory.dart new file mode 100644 index 000000000..94f1b64ea --- /dev/null +++ b/runtime/Dart/lib/src/token_factory.dart @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'input_stream.dart'; +import 'interval_set.dart'; +import 'misc/pair.dart'; +import 'token.dart'; +import 'token_source.dart'; + +/** The default mechanism for creating tokens. It's used by default in Lexer and + * the error handling strategy (to create missing tokens). Notifying the parser + * of a new factory means that it notifies its token source and error strategy. + */ +abstract class TokenFactory { + /** This is the method used to create tokens in the lexer and in the + * error handling strategy. If text!=null, than the start and stop positions + * are wiped to -1 in the text override is set in the CommonToken. + */ + Symbol create(int type, String text, + [Pair source, + int channel, + int start, + int stop, + int line, + int charPositionInLine]); +} + +/** + * This default implementation of [TokenFactory] creates + * [CommonToken] objects. + */ +class CommonTokenFactory implements TokenFactory { + /** + * The default [CommonTokenFactory] instance. + * + *

+ * This token factory does not explicitly copy token text when constructing + * tokens.

+ */ + static final TokenFactory DEFAULT = new CommonTokenFactory(); + + /** + * Indicates whether {@link CommonToken#setText} should be called after + * constructing tokens to explicitly set the text. This is useful for cases + * where the input stream might not be able to provide arbitrary substrings + * of text from the input after the lexer creates a token (e.g. the + * implementation of {@link CharStream#getText} in + * [UnbufferedCharStream] throws an + * [UnsupportedOperationException]). Explicitly setting the token text + * allows {@link Token#getText} to be called at any time regardless of the + * input stream implementation. + * + *

+ * The default value is [false] to avoid the performance and memory + * overhead of copying text for every token unless explicitly requested.

+ */ + final bool copyText; + + /** + * Constructs a [CommonTokenFactory] with the specified value for + * {@link #copyText}. + * + *

+ * When [copyText] is [false], the {@link #DEFAULT} instance + * should be used instead of constructing a new instance.

+ * + * @param copyText The value for {@link #copyText}. + */ + CommonTokenFactory([this.copyText = false]); + + CommonToken create(int type, String text, + [Pair source, + int channel, + int start, + int stop, + int line, + int charPositionInLine]) { + if (source == null) { + return CommonToken(type, text: text); + } + + CommonToken t = new CommonToken(type, + source: source, channel: channel, startIndex: start, stopIndex: stop); + t.line = line; + t.charPositionInLine = charPositionInLine; + if (text != null) { + t.text = text; + } else if (copyText && source.b != null) { + t.text = source.b.getText(Interval.of(start, stop)); + } + + return t; + } +} diff --git a/runtime/Dart/lib/src/token_source.dart b/runtime/Dart/lib/src/token_source.dart new file mode 100644 index 000000000..69fd6c573 --- /dev/null +++ b/runtime/Dart/lib/src/token_source.dart @@ -0,0 +1,273 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +import 'dart:math'; + +import 'input_stream.dart'; +import 'misc/pair.dart'; +import 'token.dart'; +import 'token_factory.dart'; + +/** + * A source of tokens must provide a sequence of tokens via {@link #nextToken()} + * and also must reveal it's source of characters; [CommonToken]'s text is + * computed from a [CharStream]; it only store indices into the char + * stream. + * + *

Errors from the lexer are never passed to the parser. Either you want to keep + * going or you do not upon token recognition error. If you do not want to + * continue lexing then you do not want to continue parsing. Just throw an + * exception not under [RecognitionException] and Java will naturally toss + * you all the way out of the recognizers. If you want to continue lexing then + * you should not throw an exception to the parser--it has already requested a + * token. Keep lexing until you get a valid one. Just report errors and keep + * going, looking for a valid token.

+ */ +abstract class TokenSource { + /** + * Return a [Token] object from your input stream (usually a + * [CharStream]). Do not fail/return upon lexing error; keep chewing + * on the characters until you get a good one; errors are not passed through + * to the parser. + */ + Token nextToken(); + + /** + * Get the line number for the current position in the input stream. The + * first line in the input is line 1. + * + * @return The line number for the current position in the input stream, or + * 0 if the current token source does not track line numbers. + */ + int get line; + + /** + * Get the index into the current line for the current position in the input + * stream. The first character on a line has position 0. + * + * @return The line number for the current position in the input stream, or + * -1 if the current token source does not track character positions. + */ + int get charPositionInLine; + + /** + * Get the [CharStream] from which this token source is currently + * providing tokens. + * + * @return The [CharStream] associated with the current position in + * the input, or null if no input stream is available for the token + * source. + */ + CharStream get inputStream; + + /** + * Gets the name of the underlying input source. This method returns a + * non-null, non-empty string. If such a name is not known, this method + * returns {@link IntStream#UNKNOWN_SOURCE_NAME}. + */ + String get sourceName; + + /** + * Set the [TokenFactory] this token source should use for creating + * [Token] objects from the input. + * + * @param factory The [TokenFactory] to use for creating tokens. + */ + void set tokenFactory(TokenFactory factory); + + /** + * Gets the [TokenFactory] this token source is currently using for + * creating [Token] objects from the input. + * + * @return The [TokenFactory] currently used by this token source. + */ + TokenFactory get tokenFactory; +} + +/** + * Provides an implementation of [TokenSource] as a wrapper around a list + * of [Token] objects. + * + *

If the final token in the list is an {@link Token#EOF} token, it will be used + * as the EOF token for every call to {@link #nextToken} after the end of the + * list is reached. Otherwise, an EOF token will be created.

+ */ +class ListTokenSource implements TokenSource { + /** + * The wrapped collection of [Token] objects to return. + */ + final List tokens; + + final String _sourceName; + + /** + * The index into {@link #tokens} of token to return by the next call to + * {@link #nextToken}. The end of the input is indicated by this value + * being greater than or equal to the number of items in {@link #tokens}. + */ + int i; + + /** + * This field caches the EOF token for the token source. + */ + Token eofToken; + + /** + * This is the backing field for {@link #getTokenFactory} and + * [setTokenFactory]. + */ + TokenFactory tokenFactory = CommonTokenFactory.DEFAULT; + + /** + * Constructs a new [ListTokenSource] instance from the specified + * collection of [Token] objects. + * + * @param tokens The collection of [Token] objects to provide as a + * [TokenSource]. + * @exception NullPointerException if [tokens] is null + */ + + /** + * Constructs a new [ListTokenSource] instance from the specified + * collection of [Token] objects and source name. + * + * @param tokens The collection of [Token] objects to provide as a + * [TokenSource]. + * @param sourceName The name of the [TokenSource]. If this value is + * null, {@link #getSourceName} will attempt to infer the name from + * the next [Token] (or the previous token if the end of the input has + * been reached). + * + * @exception NullPointerException if [tokens] is null + */ + ListTokenSource(this.tokens, [this._sourceName = null]) { + if (tokens == null) { + throw new ArgumentError.notNull("tokens"); + } + } + + /** + * {@inheritDoc} + */ + + int get charPositionInLine { + if (i < tokens.length) { + return tokens[i].charPositionInLine; + } else if (eofToken != null) { + return eofToken.charPositionInLine; + } else if (tokens.length > 0) { + // have to calculate the result from the line/column of the previous + // token, along with the text of the token. + Token lastToken = tokens[tokens.length - 1]; + String tokenText = lastToken.text; + if (tokenText != null) { + int lastNewLine = tokenText.lastIndexOf('\n'); + if (lastNewLine >= 0) { + return tokenText.length - lastNewLine - 1; + } + } + + return lastToken.charPositionInLine + + lastToken.stopIndex - + lastToken.startIndex + + 1; + } + + // only reach this if tokens is empty, meaning EOF occurs at the first + // position in the input + return 0; + } + + /** + * {@inheritDoc} + */ + + Token nextToken() { + if (i >= tokens.length) { + if (eofToken == null) { + int start = -1; + if (tokens.length > 0) { + int previousStop = tokens[tokens.length - 1].stopIndex; + if (previousStop != -1) { + start = previousStop + 1; + } + } + + int stop = max(-1, start - 1); + eofToken = tokenFactory.create(Token.EOF, "EOF", Pair(this, inputStream), + Token.DEFAULT_CHANNEL, start, stop, line, charPositionInLine); + } + + return eofToken; + } + + Token t = tokens[i]; + if (i == tokens.length - 1 && t.type == Token.EOF) { + eofToken = t; + } + + i++; + return t; + } + + /** + * {@inheritDoc} + */ + + int get line { + if (i < tokens.length) { + return tokens[i].line; + } else if (eofToken != null) { + return eofToken.line; + } else if (tokens.length > 0) { + // have to calculate the result from the line/column of the previous + // token, along with the text of the token. + Token lastToken = tokens[tokens.length - 1]; + int line = lastToken.line; + + String tokenText = lastToken.text; + if (tokenText != null) { + for (int i = 0; i < tokenText.length; i++) { + if (tokenText[i] == '\n') { + line++; + } + } + } + + // if no text is available, assume the token did not contain any newline characters. + return line; + } + + // only reach this if tokens is empty, meaning EOF occurs at the first + // position in the input + return 1; + } + + /** + * {@inheritDoc} + */ + + CharStream get inputStream { + if (i < tokens.length) { + return tokens[i].inputStream; + } else if (eofToken != null) { + return eofToken.inputStream; + } else if (tokens.length > 0) { + return tokens[tokens.length - 1].inputStream; + } + + // no input stream information is available + return null; + } + + /** + * The name of the input source. If this value is null, a call to + * {@link #getSourceName} should return the source name used to create the + * the next token in {@link #tokens} (or the previous token if the end of + * the input has been reached). + */ + String get sourceName =>_sourceName ?? inputStream?.sourceName ?? "List"; +} diff --git a/runtime/Dart/lib/src/token_stream.dart b/runtime/Dart/lib/src/token_stream.dart new file mode 100644 index 000000000..9e6dc9b72 --- /dev/null +++ b/runtime/Dart/lib/src/token_stream.dart @@ -0,0 +1,650 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +import 'input_stream.dart'; +import 'interval_set.dart'; +import 'lexer.dart'; +import 'rule_context.dart'; +import 'token.dart'; +import 'token_source.dart'; + +/** + * An [IntStream] whose symbols are [Token] instances. + */ +abstract class TokenStream extends IntStream { + /** + * Get the [Token] instance associated with the value returned by + * {@link #LA LA(k)}. This method has the same pre- and post-conditions as + * {@link IntStream#LA}. In addition, when the preconditions of this method + * are met, the return value is non-null and the value of + * {@code LT(k).getType()==LA(k)}. + * + * @see IntStream#LA + */ + Token LT(int k); + + /** + * Gets the [Token] at the specified [index] in the stream. When + * the preconditions of this method are met, the return value is non-null. + * + *

The preconditions for this method are the same as the preconditions of + * {@link IntStream#seek}. If the behavior of {@code seek(index)} is + * unspecified for the current state and given [index], then the + * behavior of this method is also unspecified.

+ * + *

The symbol referred to by [index] differs from {@code seek()} only + * in the case of filtering streams where [index] lies before the end + * of the stream. Unlike {@code seek()}, this method does not adjust + * [index] to point to a non-ignored symbol.

+ * + * @throws IllegalArgumentException if {code index} is less than 0 + * @throws UnsupportedOperationException if the stream does not support + * retrieving the token at the specified index + */ + Token get(int index); + + /** + * Gets the underlying [TokenSource] which provides tokens for this + * stream. + */ + TokenSource get tokenSource; + + /** + * Return the text of all tokens within the specified [interval]. This + * method behaves like the following code (including potential exceptions + * for violating preconditions of {@link #get}, but may be optimized by the + * specific implementation. + * + * 
+   * TokenStream stream = ...;
+   * String text = "";
+   * for (int i = interval.a; i <= interval.b; i++) {
+   *   text += stream.get(i).getText();
+   * }
+   *
+ * + * 
+   * TokenStream stream = ...;
+   * String text = stream.getText(new Interval(0, stream.length));
+   *
+ * + * 
+   * TokenStream stream = ...;
+   * String text = stream.getText(ctx.getSourceInterval());
+   *
+ * + * @param interval The interval of tokens within this stream to get text + * for. + * @return The text of all tokens / within the specified interval in this + * stream. + */ + String getText([Interval interval]); + + String get text; + + /** + * Return the text of all tokens in the source interval of the specified + * context. This method behaves like the following code, including potential + * exceptions from the call to {@link #getText(Interval)}, but may be + * optimized by the specific implementation. + * + *

If {@code ctx.getSourceInterval()} does not return a valid interval of + * tokens provided by this stream, the behavior is unspecified.

+ * + * @param ctx The context providing the source interval of tokens to get + * text for. + * @return The text of all tokens within the source interval of [ctx]. + */ + String getTextFromCtx(RuleContext ctx); + + /** + * Return the text of all tokens in this stream between [start] and + * [stop] (inclusive). + * + *

If the specified [start] or [stop] token was not provided by + * this stream, or if the [stop] occurred before the [start] + * token, the behavior is unspecified.

+ * + *

For streams which ensure that the {@link Token#getTokenIndex} method is + * accurate for all of its provided tokens, this method behaves like the + * following code. Other streams may implement this method in other ways + * provided the behavior is consistent with this at a high level.

+ * + * 
+   * TokenStream stream = ...;
+   * String text = "";
+   * for (int i = start.getTokenIndex(); i <= stop.getTokenIndex(); i++) {
+   *   text += stream.get(i).getText();
+   * }
+   *
+ * + * @param start The first token in the interval to get text for. + * @param stop The last token in the interval to get text for (inclusive). + * @return The text of all tokens lying between the specified [start] + * and [stop] tokens. + * + * @throws UnsupportedOperationException if this stream does not support + * this method for the specified tokens + */ + String getTextRange(Token start, Token stop); +} + +/** + * This implementation of [TokenStream] loads tokens from a + * [TokenSource] on-demand, and places the tokens in a buffer to provide + * access to any previous token by index. + * + *

+ * This token stream ignores the value of {@link Token#getChannel}. If your + * parser requires the token stream filter tokens to only those on a particular + * channel, such as {@link Token#DEFAULT_CHANNEL} or + * {@link Token#HIDDEN_CHANNEL}, use a filtering token stream such a + * [CommonTokenStream].

+ */ +class BufferedTokenStream implements TokenStream { + /** + * The [TokenSource] from which tokens for this stream are fetched. + */ + TokenSource _tokenSource; + + /** + * A collection of all tokens fetched from the token source. The list is + * considered a complete view of the input once {@link #fetchedEOF} is set + * to [true]. + */ + List tokens = List(); + + /** + * The index into [tokens] of the current token (next token to [consume]). + * [tokens][p] should be [LT(1)]. + * + *

This field is set to -1 when the stream is first constructed or when + * [tokenSource] is set, indicating that the first token has + * not yet been fetched from the token source. For additional information, + * see the documentation of [IntStream] for a description of + * Initializing Methods.

+ */ + int p = -1; + + /** + * Indicates whether the [Token.EOF] token has been fetched from + * [tokenSource] and added to [tokens]. This field improves + * performance for the following cases: + * + *

." +PositionAdjustingLexerDef() ::= "" + PositionAdjustingLexer() ::= << public override IToken NextToken() { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Chrome.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Chrome.test.stg index e02457a90..070409c21 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Chrome.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Chrome.test.stg @@ -14,6 +14,8 @@ Cast(t,v) ::= "" Append(a,b) ::= " + " +AppendStr(a,b) ::= <%%> + Concat(a,b) ::= "" DeclareLocal(s,v) ::= "var = ;" @@ -26,6 +28,12 @@ InitIntMember(n,v) ::= <%this. = ;%> InitBooleanMember(n,v) ::= <%this. = ;%> +InitIntVar(n,v) ::= <%%> + +IntArg(n) ::= "" + +VarRef(n) ::= "" + GetMember(n) ::= <%this.%> SetMember(n,v) ::= <%this. = ;%> @@ -92,6 +100,8 @@ this.Property = function() { } >> +PositionAdjustingLexerDef() ::= "" + PositionAdjustingLexer() ::= << PositionAdjustingLexer.prototype.resetAcceptPosition = function(index, line, column) { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Cpp.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Cpp.test.stg index c2b1a7f84..bd3f0dd79 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Cpp.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Cpp.test.stg @@ -9,6 +9,7 @@ Assert(s) ::= "" Cast(t,v) ::= "dynamic_cast\< *>()" // Should actually use a more specific name. We may have to use other casts as well. Append(a,b) ::= " + ->toString()" Concat(a,b) ::= "" +AppendStr(a,b) ::= " + " DeclareLocal(s,v) ::= " = " @@ -17,6 +18,9 @@ AssignLocal(s,v) ::= " = ;" InitIntMember(n,v) ::= "int = ;" InitBooleanMember(n,v) ::= "bool = ;" +InitIntVar(n,v) ::= <%%> +IntArg(n) ::= "int " +VarRef(n) ::= "" GetMember(n) ::= "" SetMember(n,v) ::= " = ;" @@ -68,6 +72,8 @@ bool Property() { ParserPropertyCall(p, call) ::= "" +PositionAdjustingLexerDef() ::= "" + PositionAdjustingLexer() ::= << protected: class PositionAdjustingLexerATNSimulator : public antlr4::atn::LexerATNSimulator { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Explorer.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Explorer.test.stg index 6cfe9ba27..a3d4cb026 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Explorer.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Explorer.test.stg @@ -14,6 +14,8 @@ Cast(t,v) ::= "" Append(a,b) ::= " + " +AppendStr(a,b) ::= <%%> + Concat(a,b) ::= "" DeclareLocal(s,v) ::= "var = ;" @@ -26,6 +28,12 @@ InitIntMember(n,v) ::= <%this. = ;%> InitBooleanMember(n,v) ::= <%this. = ;%> +InitIntVar(n,v) ::= <%%> + +IntArg(n) ::= "" + +VarRef(n) ::= "" + GetMember(n) ::= <%this.%> SetMember(n,v) ::= <%this. = ;%> @@ -98,6 +106,8 @@ this.Property = function() { ParserPropertyCall(p, call) ::= "