From 4bcbbe23a175b68f1f8e2b8af9e7d0f89f73bdb3 Mon Sep 17 00:00:00 2001 From: Larry Li Date: Wed, 25 Nov 2020 17:38:41 +1100 Subject: [PATCH 01/42] Include prerequisite to publish Dart target add link to download dart sdk --- doc/releasing-antlr.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index 53db4cdc3..df34d88bd 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -449,6 +449,8 @@ popd ### Dart +Install Dart SDK from https://dart.dev/get-dart + Push to pub.dev ```bash From 5dce78c87a9e65b938f069d01f29a233605bd3c3 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Thu, 21 Jan 2021 17:23:44 +0800 Subject: [PATCH 02/42] Fix typo --- runtime/JavaScript/webpack.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/JavaScript/webpack.config.js b/runtime/JavaScript/webpack.config.js index b3b728b9b..7b2fead63 100644 --- a/runtime/JavaScript/webpack.config.js +++ b/runtime/JavaScript/webpack.config.js @@ -1,4 +1,4 @@ -const path = require('path') +const path = require('path'); module.exports = { mode: "production", From 10958043880b0f36d6ef056770a16f0128162406 Mon Sep 17 00:00:00 2001 From: ericvergnaud Date: Fri, 22 Jan 2021 11:33:29 +0800 Subject: [PATCH 03/42] Fix swift warnings (#3043) * fix warnings when compiling swift runtime * more warnings gone * more warnings gone * Revert "more warnings gone" This reverts commit a09e221c41417fca652a2225e80845c0b0cbb832. --- runtime/Swift/Sources/Antlr4/Lexer.swift | 2 +- .../Swift/Sources/Antlr4/atn/LexerATNSimulator.swift | 10 +++++----- .../Sources/Antlr4/atn/LexerActionExecutor.swift | 2 +- .../Sources/Antlr4/atn/ProfilingATNSimulator.swift | 12 ++++++------ .../antlr/v4/tool/templates/codegen/Swift/Swift.stg | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/runtime/Swift/Sources/Antlr4/Lexer.swift b/runtime/Swift/Sources/Antlr4/Lexer.swift index e6c26296b..4a2676cd3 100644 --- a/runtime/Swift/Sources/Antlr4/Lexer.swift +++ b/runtime/Swift/Sources/Antlr4/Lexer.swift @@ -132,7 +132,7 @@ open class Lexer: Recognizer, TokenSource { // Mark start location in char stream so unbuffered streams are // guaranteed at least have text of current token - var tokenStartMarker = _input.mark() + let tokenStartMarker = _input.mark() defer { // make sure we release marker after match or // unbuffered char stream will keep buffering diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerATNSimulator.swift b/runtime/Swift/Sources/Antlr4/atn/LexerATNSimulator.swift index c40b0d33e..dee758b53 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerATNSimulator.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerATNSimulator.swift @@ -116,7 +116,7 @@ open class LexerATNSimulator: ATNSimulator { LexerATNSimulator.match_calls += 1 self.mode = mode - var mark = input.mark() + let mark = input.mark() defer { try! input.release(mark) } @@ -609,10 +609,10 @@ open class LexerATNSimulator: ATNSimulator { return try recog.sempred(nil, ruleIndex, predIndex) } - var savedCharPositionInLine = charPositionInLine - var savedLine = line - var index = input.index() - var marker = input.mark() + let savedCharPositionInLine = charPositionInLine + let savedLine = line + let index = input.index() + let marker = input.mark() do { try consume(input) defer diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerActionExecutor.swift b/runtime/Swift/Sources/Antlr4/atn/LexerActionExecutor.swift index 365f41a58..4b7df45c5 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerActionExecutor.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerActionExecutor.swift @@ -149,7 +149,7 @@ public class LexerActionExecutor: Hashable { /// public func execute(_ lexer: Lexer, _ input: CharStream, _ startIndex: Int) throws { var requiresSeek: Bool = false - var stopIndex: Int = input.index() + let stopIndex: Int = input.index() defer { if requiresSeek { try! input.seek(stopIndex) diff --git a/runtime/Swift/Sources/Antlr4/atn/ProfilingATNSimulator.swift b/runtime/Swift/Sources/Antlr4/atn/ProfilingATNSimulator.swift index 51ffd2f20..12f673f57 100644 --- a/runtime/Swift/Sources/Antlr4/atn/ProfilingATNSimulator.swift +++ b/runtime/Swift/Sources/Antlr4/atn/ProfilingATNSimulator.swift @@ -53,17 +53,17 @@ public class ProfilingATNSimulator: ParserATNSimulator { override public func adaptivePredict(_ input: TokenStream, _ decision: Int,_ outerContext: ParserRuleContext?) throws -> Int { - var outerContext = outerContext + let outerContext = outerContext self._sllStopIndex = -1 self._llStopIndex = -1 self.currentDecision = decision - var start: Int64 = Int64(Date().timeIntervalSince1970) //System.nanoTime(); // expensive but useful info - var alt: Int = try super.adaptivePredict(input, decision, outerContext) - var stop: Int64 = Int64(Date().timeIntervalSince1970) //System.nanoTime(); + let start: Int64 = Int64(Date().timeIntervalSince1970) //System.nanoTime(); // expensive but useful info + let alt: Int = try super.adaptivePredict(input, decision, outerContext) + let stop: Int64 = Int64(Date().timeIntervalSince1970) //System.nanoTime(); decisions[decision].timeInPrediction += (stop - start) decisions[decision].invocations += 1 - var SLL_k: Int64 = Int64(_sllStopIndex - _startIndex + 1) + let SLL_k: Int64 = Int64(_sllStopIndex - _startIndex + 1) decisions[decision].SLL_TotalLook += SLL_k decisions[decision].SLL_MinLook = decisions[decision].SLL_MinLook == 0 ? SLL_k : min(decisions[decision].SLL_MinLook, SLL_k) if SLL_k > decisions[decision].SLL_MaxLook { @@ -73,7 +73,7 @@ public class ProfilingATNSimulator: ParserATNSimulator { } if _llStopIndex >= 0 { - var LL_k: Int64 = Int64(_llStopIndex - _startIndex + 1) + let LL_k: Int64 = Int64(_llStopIndex - _startIndex + 1) decisions[decision].LL_TotalLook += LL_k decisions[decision].LL_MinLook = decisions[decision].LL_MinLook == 0 ? LL_k : min(decisions[decision].LL_MinLook, LL_k) if LL_k > decisions[decision].LL_MaxLook { diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg index 3e19be51f..2c25e6948 100755 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg @@ -397,7 +397,7 @@ RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,fina }; separator="\n"> @discardableResult }> func (_ ) throws -> { - var _localctx: = (_ctx, getState()}>) + let _localctx: = (_ctx, getState()}>) try enterRule(_localctx, , .RULE_) From 107f40c63cad4dfc7972f89c886df0e643915f76 Mon Sep 17 00:00:00 2001 From: ericvergnaud Date: Fri, 22 Jan 2021 17:04:17 +0800 Subject: [PATCH 04/42] Use self hosted GitHub CI for Swift (#3036) * Create swift-lexer.yml Exploring self-hosted GitHub CI * Create run-tests-swift.sh * Update swift-lexer.yml * Update swift-lexer.yml * Update swift-lexer.yml * chmod script * chmod script * fix typo * Update swift-lexer.yml * chmod script * Update swift-lexer.yml * Update swift-lexer.yml * Update run-tests-swift.sh * Update run-tests-swift.sh * Update run-tests-swift.sh * Update run-tests-swift.sh * Update swift-lexer.yml * Update swift-lexer.yml * Update run-tests-swift.sh * Update swift-lexer.yml * Update swift-lexer.yml * Update run-tests-swift.sh * Update run-tests-swift.sh * Update run-tests-swift.sh * Update run-tests-swift.sh * Update run-tests-swift.sh * Update run-tests-swift.sh * try java tests * Update java-all.yml * Update java-all.yml * only run heartbeat when necessary (otherwise build hangs) * kill hung process * Update run-tests-swift.sh * Update run-tests-swift.sh * Update run-tests-swift.sh * Create mac-action-runner-readme.txt * try prefixing swift cmd with arch * try again * try locate generated binaries * fix crash * try from script * keep trying * keep trying * in progress! * let's see how it goes... * fix failing build * more logs * more logs * more logs * finally ? * grrrr * let's see if maven is the culprit * fix warnings when compiling swift runtime * more warnings gone * works with 1 class, how about many? * more warnings gone * less noise * Revert "more warnings gone" This reverts commit a09e221c41417fca652a2225e80845c0b0cbb832. * try matrix * fix failing build * don't fail fast --- .github/scripts/mac-action-runner-readme.txt | 23 ++++ .github/scripts/run-tests-java.sh | 25 ++++ .github/scripts/run-tests-swift.sh | 50 ++++++++ .github/workflows/java-all.yml | 28 +++++ .github/workflows/swift-macosx.yml | 35 ++++++ .../v4/test/runtime/BaseRuntimeTest.java | 46 ++++++-- .../v4/test/runtime/RuntimeTestSupport.java | 3 + .../v4/test/runtime/cpp/BaseCppTest.java | 9 ++ .../test/runtime/csharp/BaseCSharpTest.java | 13 +- .../v4/test/runtime/dart/BaseDartTest.java | 13 +- .../antlr/v4/test/runtime/go/BaseGoTest.java | 9 ++ .../v4/test/runtime/java/BaseJavaTest.java | 13 +- .../test/runtime/javascript/BaseNodeTest.java | 52 ++------ .../v4/test/runtime/php/BasePHPTest.java | 9 ++ .../test/runtime/python/BasePythonTest.java | 13 +- .../v4/test/runtime/swift/BaseSwiftTest.java | 111 +++++++++++++++--- .../v4/tool/templates/codegen/Swift/Swift.stg | 2 +- 17 files changed, 368 insertions(+), 86 deletions(-) create mode 100644 .github/scripts/mac-action-runner-readme.txt create mode 100755 .github/scripts/run-tests-java.sh create mode 100755 .github/scripts/run-tests-swift.sh create mode 100644 .github/workflows/java-all.yml create mode 100644 .github/workflows/swift-macosx.yml diff --git a/.github/scripts/mac-action-runner-readme.txt b/.github/scripts/mac-action-runner-readme.txt new file mode 100644 index 000000000..ce774da3f --- /dev/null +++ b/.github/scripts/mac-action-runner-readme.txt @@ -0,0 +1,23 @@ +Sharing the trouble of getting github action runners to work on a mac coming straight out of the factory, running Big Sur + +XCode (you need XCode to build the Swift runtime): + - install XCode from the Mac App Store + - Launch it, this will force installation of components + - Go to Preferences -> Locations and select XCode as Command Line Tools + +Brew (you need Brew to install maven): + - get the script from https://brew.sh + - once installed, run the following: + echo 'eval $(/opt/homebrew/bin/brew shellenv)' >> /Users/{user-account}/.zprofile + eval $(/opt/homebrew/bin/brew shellenv) + (you need to repeat these last steps for each user account) + +Maven (supposedly installed by the github workflow, but it's convenient to have a global install for troubleshooting): + - brew install maven + +JDK (we need a specific JDK): + - download openjdk8 from Oracle (later versions break the build due to some packages having disappeared) + - install it -> this will mess up your JAVA_HOME completely, pointing to /Library/Internet... + - fix the JAVA_HOME mess as follows: + sudo rm -fr /Library/Internet\ Plug-Ins/JavaAppletPlugin.plugin + sudo rm -fr /Library/PreferencePanes/JavaControlPanel.prefpane diff --git a/.github/scripts/run-tests-java.sh b/.github/scripts/run-tests-java.sh new file mode 100755 index 000000000..678581bf1 --- /dev/null +++ b/.github/scripts/run-tests-java.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -euo pipefail + +if [ -z "${JAVA_HOME-}" ] +then + export JAVA_HOME="$(java -XshowSettings:properties -version 2>&1 | + grep 'java\.home' | awk '{ print $3 }')" + echo "export JAVA_HOME=$JAVA_HOME" +fi + +# run java tests +cd runtime-testsuite/ +if [ $GROUP == "LEXER" ]; then + mvn -X -e -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest="swift.*" test +elif [ $GROUP == "PARSER" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest="swift.*" test +elif [ $GROUP == "RECURSION" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest="swift.*" test +else + mvn -q -Dtest="java.*" test +fi +rc=$? +cat target/surefire-reports/*.dumpstream || true +exit $rc diff --git a/.github/scripts/run-tests-swift.sh b/.github/scripts/run-tests-swift.sh new file mode 100755 index 000000000..7e40e9376 --- /dev/null +++ b/.github/scripts/run-tests-swift.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -euo pipefail + +# linux specific setup, those setup have to be +# here since environment variables doesn't pass +# across scripts +if [ $RUNNER_OS == "Linux" ]; then + export SWIFT_VERSION=swift-5.0.1 + export SWIFT_HOME=$(pwd)/swift/$SWIFT_VERSION-RELEASE-ubuntu16.04/usr/bin/ + export PATH=$SWIFT_HOME:$PATH + + # download swift + mkdir swift + curl https://swift.org/builds/$SWIFT_VERSION-release/ubuntu1604/$SWIFT_VERSION-RELEASE/$SWIFT_VERSION-RELEASE-ubuntu16.04.tar.gz -s | tar xz -C swift &> /dev/null +fi + +if [ -z "${JAVA_HOME}" ] +then + export JAVA_HOME="$(java -XshowSettings:properties -version 2>&1 | + grep 'java\.home' | awk '{ print $3 }')" +fi +echo "export JAVA_HOME=$JAVA_HOME" + +# check swift +swift --version +swift build --version + +# run swift tests +pushd runtime/Swift +./boot.py --test +rc=$? +popd + +if [ $rc == 0 ]; then + # run java tests + cd runtime-testsuite/ + if [ $GROUP == "LEXER" ]; then + mvn -e -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest="swift.*" test + elif [ $GROUP == "PARSER" ]; then + mvn -e -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest="swift.*" test + elif [ $GROUP == "RECURSION" ]; then + mvn -e -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest="swift.*" test + else + mvn -e -q -Dtest=swift.* test + fi + rc=$? + cat target/surefire-reports/*.dumpstream || true +fi +exit $rc diff --git a/.github/workflows/java-all.yml b/.github/workflows/java-all.yml new file mode 100644 index 000000000..34cdb8b46 --- /dev/null +++ b/.github/workflows/java-all.yml @@ -0,0 +1,28 @@ +name: Java/MacOSX + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + runs-on: [self-hosted, macOS, x64] + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Set up Maven + uses: stCarolas/setup-maven@v4 + with: + maven-version: 3.5.4 + - name: Build tool with Maven + run: mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V + - name: Test with Maven + run: .github/scripts/run-tests-java.sh + env: + TARGET: swift + GROUP: ALL diff --git a/.github/workflows/swift-macosx.yml b/.github/workflows/swift-macosx.yml new file mode 100644 index 000000000..37fa14128 --- /dev/null +++ b/.github/workflows/swift-macosx.yml @@ -0,0 +1,35 @@ +name: Swift/MacOSX + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + runs-on: [self-hosted, macOS, x64] + strategy: + fail-fast: false + matrix: + include: + - GROUP: LEXER + - GROUP: PARSER + - GROUP: RECURSION + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Set up Maven + uses: stCarolas/setup-maven@v4 + with: + maven-version: 3.5.4 + - name: Build tool with Maven + run: mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V + - name: Test with Maven + run: arch -x86_64 .github/scripts/run-tests-swift.sh + env: + TARGET: swift + GROUP: ${{ matrix.GROUP }} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java index ebed54349..17c0d5e96 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java @@ -11,9 +11,7 @@ import org.antlr.v4.runtime.misc.Pair; import org.antlr.v4.runtime.misc.Utils; import org.antlr.v4.tool.ANTLRMessage; import org.antlr.v4.tool.DefaultToolListener; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; +import org.junit.*; import org.junit.rules.TestRule; import org.junit.rules.TestWatcher; import org.junit.runner.Description; @@ -44,6 +42,7 @@ import static org.junit.Assume.assumeFalse; * @since 4.6. */ public abstract class BaseRuntimeTest { + public final static String[] Targets = { "Cpp", "CSharp", @@ -56,18 +55,43 @@ public abstract class BaseRuntimeTest { "Swift" }; - static { - // Add heartbeat thread to gen minimal output for travis, appveyor to - // avoid timeout. + @BeforeClass + public static void startHeartbeatToAvoidTimeout() { + if (isTravisCI() || isAppVeyorCI()) + startHeartbeat(); + } + + @AfterClass + public static void stopHeartbeat() { + heartbeat = false; + } + + private static boolean isAppVeyorCI() { + // see https://www.appveyor.com/docs/environment-variables/ + String s = System.getenv("APPVEYOR"); + return s!=null && "true".equals(s.toLowerCase()); + } + + private static boolean isTravisCI() { + // see https://docs.travis-ci.com/user/environment-variables/#default-environment-variables + String s = System.getenv("TRAVIS"); + return s!=null && "true".equals(s.toLowerCase()); + } + + static boolean heartbeat = false; + + private static void startHeartbeat() { + // Add heartbeat thread to gen minimal output for travis, appveyor to avoid timeout. Thread t = new Thread("heartbeat") { @Override public void run() { - while (true) { + heartbeat = true; + while (heartbeat) { System.out.print('.'); try { + //noinspection BusyWait Thread.sleep(5000); - } - catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); } } @@ -118,18 +142,20 @@ public abstract class BaseRuntimeTest { @Test public void testOne() throws Exception { + // System.out.println(descriptor.getTestName()); // System.out.println(delegate.getTmpDir()); if (descriptor.ignore(descriptor.getTarget()) ) { System.out.println("Ignore " + descriptor); return; } - + delegate.beforeTest(descriptor); if (descriptor.getTestType().contains("Parser") ) { testParser(descriptor); } else { testLexer(descriptor); } + delegate.afterTest(descriptor); } public void testParser(RuntimeTestDescriptor descriptor) throws Exception { diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestSupport.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestSupport.java index ad94e2907..ca0950986 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestSupport.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestSupport.java @@ -39,4 +39,7 @@ public interface RuntimeTestSupport { String startRuleName, String input, boolean showDiagnosticErrors); + + void beforeTest(RuntimeTestDescriptor descriptor); + void afterTest(RuntimeTestDescriptor descriptor); } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java index fa8ae24cd..03895f55a 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java @@ -35,6 +35,7 @@ import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.semantics.SemanticPipeline; import org.antlr.v4.test.runtime.ErrorQueue; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; import org.antlr.v4.test.runtime.RuntimeTestSupport; import org.antlr.v4.test.runtime.StreamVacuum; import org.antlr.v4.tool.ANTLRMessage; @@ -111,6 +112,14 @@ public class BaseCppTest implements RuntimeTestSupport { public void testTearDown() throws Exception { } + @Override + public void beforeTest(RuntimeTestDescriptor descriptor) { + } + + @Override + public void afterTest(RuntimeTestDescriptor descriptor) { + } + @Override public String getTmpDir() { return tmpdir; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java index 79b87f4d0..5fbd97ecf 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java @@ -11,10 +11,7 @@ import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.TokenSource; import org.antlr.v4.runtime.WritableToken; import org.antlr.v4.runtime.misc.Utils; -import org.antlr.v4.test.runtime.ErrorQueue; -import org.antlr.v4.test.runtime.RuntimeTestSupport; -import org.antlr.v4.test.runtime.StreamVacuum; -import org.antlr.v4.test.runtime.TestOutputReading; +import org.antlr.v4.test.runtime.*; import org.antlr.v4.tool.ANTLRMessage; import org.antlr.v4.tool.GrammarSemanticsMessage; import org.junit.rules.TestRule; @@ -143,6 +140,14 @@ public class BaseCSharpTest implements RuntimeTestSupport { public void testTearDown() throws Exception { } + @Override + public void beforeTest(RuntimeTestDescriptor descriptor) { + } + + @Override + public void afterTest(RuntimeTestDescriptor descriptor) { + } + @Override public String getTmpDir() { return tmpdir; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java index 88c69d755..6ce699ae9 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java @@ -22,10 +22,7 @@ import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.misc.Pair; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.semantics.SemanticPipeline; -import org.antlr.v4.test.runtime.BaseRuntimeTest; -import org.antlr.v4.test.runtime.ErrorQueue; -import org.antlr.v4.test.runtime.RuntimeTestSupport; -import org.antlr.v4.test.runtime.StreamVacuum; +import org.antlr.v4.test.runtime.*; import org.antlr.v4.test.runtime.descriptors.LexerExecDescriptors; import org.antlr.v4.test.runtime.descriptors.PerformanceDescriptors; import org.antlr.v4.tool.*; @@ -153,6 +150,14 @@ public class BaseDartTest implements RuntimeTestSupport { public void testTearDown() throws Exception { } + @Override + public void beforeTest(RuntimeTestDescriptor descriptor) { + } + + @Override + public void afterTest(RuntimeTestDescriptor descriptor) { + } + @Override public String getTmpDir() { return tmpdir; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java index 48791461f..f5596db3e 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java @@ -32,6 +32,7 @@ import org.antlr.v4.runtime.misc.IntegerList; import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.semantics.SemanticPipeline; import org.antlr.v4.test.runtime.ErrorQueue; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; import org.antlr.v4.test.runtime.RuntimeTestSupport; import org.antlr.v4.test.runtime.StreamVacuum; import org.antlr.v4.tool.ANTLRMessage; @@ -120,6 +121,14 @@ public class BaseGoTest implements RuntimeTestSupport { public void testTearDown() throws Exception { } + @Override + public void beforeTest(RuntimeTestDescriptor descriptor) { + } + + @Override + public void afterTest(RuntimeTestDescriptor descriptor) { + } + @Override public String getTmpDir() { return tmpdir.getPath(); diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java index c94e8e7a6..765ab9e9f 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java @@ -36,10 +36,7 @@ import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.misc.Pair; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.semantics.SemanticPipeline; -import org.antlr.v4.test.runtime.BaseRuntimeTest; -import org.antlr.v4.test.runtime.ErrorQueue; -import org.antlr.v4.test.runtime.RuntimeTestSupport; -import org.antlr.v4.test.runtime.StreamVacuum; +import org.antlr.v4.test.runtime.*; import org.antlr.v4.tool.ANTLRMessage; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.GrammarSemanticsMessage; @@ -196,6 +193,14 @@ public class BaseJavaTest implements RuntimeTestSupport { public void testTearDown() throws Exception { } + @Override + public void beforeTest(RuntimeTestDescriptor descriptor) { + } + + @Override + public void afterTest(RuntimeTestDescriptor descriptor) { + } + @Override public String getTmpDir() { return tmpdir; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java index 3f5c3e378..bfe98a21c 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java @@ -6,65 +6,25 @@ package org.antlr.v4.test.runtime.javascript; import org.antlr.v4.Tool; -import org.antlr.v4.automata.ATNFactory; -import org.antlr.v4.automata.ATNPrinter; import org.antlr.v4.automata.LexerATNFactory; import org.antlr.v4.automata.ParserATNFactory; -import org.antlr.v4.codegen.CodeGenerator; -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CommonToken; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.IntStream; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.RuleContext; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenSource; -import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.WritableToken; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNDeserializer; import org.antlr.v4.runtime.atn.ATNSerializer; -import org.antlr.v4.runtime.atn.ATNState; -import org.antlr.v4.runtime.atn.LexerATNSimulator; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.misc.IntegerList; -import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.semantics.SemanticPipeline; -import org.antlr.v4.test.runtime.ErrorQueue; -import org.antlr.v4.test.runtime.RuntimeTestSupport; -import org.antlr.v4.test.runtime.StreamVacuum; -import org.antlr.v4.test.runtime.TestContext; -import org.antlr.v4.tool.ANTLRMessage; -import org.antlr.v4.tool.DOTGenerator; +import org.antlr.v4.test.runtime.*; import org.antlr.v4.tool.Grammar; -import org.antlr.v4.tool.GrammarSemanticsMessage; import org.antlr.v4.tool.LexerGrammar; -import org.antlr.v4.tool.Rule; import org.stringtemplate.v4.ST; -import org.stringtemplate.v4.STGroup; -import org.stringtemplate.v4.STGroupString; import java.io.File; import java.io.IOException; import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.TimeUnit; +import java.util.*; import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString; import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; public class BaseNodeTest implements RuntimeTestSupport { @@ -108,6 +68,14 @@ public class BaseNodeTest implements RuntimeTestSupport { public void testTearDown() throws Exception { } + @Override + public void beforeTest(RuntimeTestDescriptor descriptor) { + } + + @Override + public void afterTest(RuntimeTestDescriptor descriptor) { + } + @Override public String getTmpDir() { return tmpdir; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java index 8f2cc5bcc..f164ffec8 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java @@ -25,6 +25,7 @@ import org.antlr.v4.runtime.atn.ATNDeserializer; import org.antlr.v4.runtime.atn.ATNSerializer; import org.antlr.v4.semantics.SemanticPipeline; import org.antlr.v4.test.runtime.ErrorQueue; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; import org.antlr.v4.test.runtime.RuntimeTestSupport; import org.antlr.v4.test.runtime.StreamVacuum; import org.antlr.v4.tool.Grammar; @@ -78,6 +79,14 @@ public class BasePHPTest implements RuntimeTestSupport { public void testTearDown() throws Exception { } + @Override + public void beforeTest(RuntimeTestDescriptor descriptor) { + } + + @Override + public void afterTest(RuntimeTestDescriptor descriptor) { + } + @Override public String getTmpDir() { return tmpdir; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java index c7c39857e..c8461ac9c 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java @@ -34,10 +34,7 @@ import org.antlr.v4.runtime.misc.IntegerList; import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.semantics.SemanticPipeline; -import org.antlr.v4.test.runtime.ErrorQueue; -import org.antlr.v4.test.runtime.RuntimeTestSupport; -import org.antlr.v4.test.runtime.StreamVacuum; -import org.antlr.v4.test.runtime.TestOutputReading; +import org.antlr.v4.test.runtime.*; import org.antlr.v4.tool.ANTLRMessage; import org.antlr.v4.tool.DOTGenerator; import org.antlr.v4.tool.Grammar; @@ -125,6 +122,14 @@ public abstract class BasePythonTest implements RuntimeTestSupport { public void testTearDown() throws Exception { } + @Override + public void beforeTest(RuntimeTestDescriptor descriptor) { + } + + @Override + public void afterTest(RuntimeTestDescriptor descriptor) { + } + @Override public String getTmpDir() { return tmpdir; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java index 2e71deb9b..7f2158975 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java @@ -8,20 +8,17 @@ package org.antlr.v4.test.runtime.swift; import org.antlr.v4.runtime.misc.Pair; import org.antlr.v4.test.runtime.ErrorQueue; +import org.antlr.v4.test.runtime.RuntimeTestDescriptor; import org.antlr.v4.test.runtime.RuntimeTestSupport; import org.antlr.v4.test.runtime.StreamVacuum; import org.stringtemplate.v4.ST; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; +import java.io.InputStreamReader; import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString; import static org.antlr.v4.test.runtime.BaseRuntimeTest.mkdir; @@ -30,15 +27,18 @@ import static org.junit.Assert.assertTrue; public class BaseSwiftTest implements RuntimeTestSupport { + private static final boolean USE_ARCH_ARM64 = false; + private static final boolean VERBOSE = false; + /** * Path of the ANTLR runtime. */ - private static String ANTLR_RUNTIME_PATH; + private static final String ANTLR_RUNTIME_PATH; /** * Absolute path to swift command. */ - private static String SWIFT_CMD; + private static final String SWIFT_CMD; /** * Environment variable name for swift home. @@ -54,7 +54,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { // build swift runtime URL swiftRuntime = loader.getResource("Swift"); if (swiftRuntime == null) { - throw new RuntimeException("Swift runtime file not found at:" + swiftRuntime.getPath()); + throw new RuntimeException("Swift runtime file not found"); } ANTLR_RUNTIME_PATH = swiftRuntime.getPath(); try { @@ -94,7 +94,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { /** * Source files used in each small swift project. */ - private Set sourceFiles = new HashSet<>(); + private final Set sourceFiles = new HashSet<>(); @Override public void testSetUp() throws Exception { @@ -117,6 +117,15 @@ public class BaseSwiftTest implements RuntimeTestSupport { public void testTearDown() throws Exception { } + @Override + public void beforeTest(RuntimeTestDescriptor descriptor) { + System.out.println(descriptor.getTestName()); + } + + @Override + public void afterTest(RuntimeTestDescriptor descriptor) { + } + @Override public void eraseTempDir() { } @@ -221,31 +230,99 @@ public class BaseSwiftTest implements RuntimeTestSupport { } } + static Boolean IS_MAC_ARM_64 = null; + + private static boolean isMacOSArm64() { + if (IS_MAC_ARM_64 == null) { + IS_MAC_ARM_64 = computeIsMacOSArm64(); + System.err.println("IS_MAC_ARM_64 = " + IS_MAC_ARM_64); + } + return IS_MAC_ARM_64; + } + + private static boolean computeIsMacOSArm64() { + String os = System.getenv("RUNNER_OS"); + if(os==null || !os.equalsIgnoreCase("macos")) + return false; + try { + Process p = Runtime.getRuntime().exec("uname -a"); + BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream())); + String uname = in.readLine(); + return uname.contains("_ARM64_"); + } catch (IOException e) { + e.printStackTrace(); + return false; + } + } + private static Pair runProcess(String execPath, String... args) throws IOException, InterruptedException { - Process process = Runtime.getRuntime().exec(args, null, new File(execPath)); + List argsWithArch = new ArrayList<>(); + if(USE_ARCH_ARM64 && isMacOSArm64()) + argsWithArch.addAll(Arrays.asList("arch", "-arm64")); + argsWithArch.addAll(Arrays.asList(args)); + if(VERBOSE) + System.err.println("Executing " + argsWithArch.toString() + " " + execPath); + final Process process = Runtime.getRuntime().exec(argsWithArch.toArray(new String[0]), null, new File(execPath)); StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); stdoutVacuum.start(); stderrVacuum.start(); + Timer timer = new Timer(); + timer.schedule(new TimerTask() { + @Override + public void run() { + try { + process.destroy(); + } catch(Exception e) { + e.printStackTrace(System.err); + } + } + }, 120_000); int status = process.waitFor(); + timer.cancel(); stdoutVacuum.join(); stderrVacuum.join(); + if(VERBOSE) + System.err.println("Done executing " + argsWithArch.toString() + " " + execPath); if (status != 0) { + System.err.println("Process exited with status " + status); throw new IOException("Process exited with status " + status + ":\n" + stdoutVacuum.toString() + "\n" + stderrVacuum.toString()); } return new Pair<>(stdoutVacuum.toString(), stderrVacuum.toString()); } private static void fastFailRunProcess(String workingDir, String... command) throws IOException, InterruptedException { - ProcessBuilder builder = new ProcessBuilder(command); + List argsWithArch = new ArrayList<>(); + if(USE_ARCH_ARM64 && isMacOSArm64()) + argsWithArch.addAll(Arrays.asList("arch", "-arm64")); + argsWithArch.addAll(Arrays.asList(command)); + if(VERBOSE) + System.err.println("Executing " + argsWithArch.toString() + " " + workingDir); + ProcessBuilder builder = new ProcessBuilder(argsWithArch.toArray(new String[0])); builder.directory(new File(workingDir)); - Process p = builder.start(); - int status = p.waitFor(); + final Process process = builder.start(); + Timer timer = new Timer(); + timer.schedule(new TimerTask() { + @Override + public void run() { + try { + process.destroy(); + } catch(Exception e) { + e.printStackTrace(System.err); + } + } + }, 120_000); + int status = process.waitFor(); + timer.cancel(); + if(VERBOSE) + System.err.println("Done executing " + argsWithArch.toString() + " " + workingDir); if (status != 0) { + System.err.println("Process exited with status " + status); throw new IOException("Process exited with status " + status); } } + @SuppressWarnings("SameParameterValue") private String execParser(String parserName, String lexerName, String parserStartRuleName, @@ -324,7 +401,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { "parser.setInterpreter(profiler)"); } else { - outputFileST.add("profile", new ArrayList()); + outputFileST.add("profile", new ArrayList<>()); } outputFileST.add("createParser", createParserST); outputFileST.add("parserName", parserName); @@ -387,6 +464,6 @@ public class BaseSwiftTest implements RuntimeTestSupport { files.add(grammarName + "BaseVisitor.swift"); } } - addSourceFiles(files.toArray(new String[files.size()])); + addSourceFiles(files.toArray(new String[0])); } } diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg index 2c25e6948..3e19be51f 100755 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg @@ -397,7 +397,7 @@ RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,fina }; separator="\n"> @discardableResult }> func (_ ) throws -> { - let _localctx: = (_ctx, getState()}>) + var _localctx: = (_ctx, getState()}>) try enterRule(_localctx, , .RULE_) From 53b65c015c9aa3c910c059334040aa6bf34fbd99 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 23 Jan 2021 17:04:01 +0800 Subject: [PATCH 05/42] fix #3042 in C# runtime --- .gitignore | 1 + runtime/CSharp/Atn/LL1Analyzer.cs | 401 +++++++++---------------- runtime/CSharp/DefaultErrorStrategy.cs | 31 +- 3 files changed, 170 insertions(+), 263 deletions(-) diff --git a/.gitignore b/.gitignore index 81e058979..00cfcd826 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,4 @@ javac-services.0.log.lck # Don't ignore python tests !runtime/Python3/test/ +Antlr4.sln diff --git a/runtime/CSharp/Atn/LL1Analyzer.cs b/runtime/CSharp/Atn/LL1Analyzer.cs index 06bcebb33..e67933412 100644 --- a/runtime/CSharp/Atn/LL1Analyzer.cs +++ b/runtime/CSharp/Atn/LL1Analyzer.cs @@ -3,7 +3,6 @@ * can be found in the LICENSE.txt file in the project root. */ using System.Collections.Generic; -using Antlr4.Runtime.Atn; using Antlr4.Runtime.Misc; using Antlr4.Runtime.Sharpen; @@ -11,12 +10,9 @@ namespace Antlr4.Runtime.Atn { public class LL1Analyzer { - /// - /// Special value added to the lookahead sets to indicate that we hit - /// a predicate during analysis if - /// seeThruPreds==false - /// . - /// + /** Special value added to the lookahead sets to indicate that we hit + * a predicate during analysis if {@code seeThruPreds==false}. + */ public const int HitPred = TokenConstants.InvalidType; [NotNull] @@ -27,25 +23,16 @@ namespace Antlr4.Runtime.Atn this.atn = atn; } - /// - /// Calculates the SLL(1) expected lookahead set for each outgoing transition - /// of an - /// - /// . The returned array has one element for each - /// outgoing transition in - /// - /// . If the closure from transition - /// i leads to a semantic predicate before matching a symbol, the - /// element at index i of the result will be - /// - /// . - /// - /// the ATN state - /// - /// the expected symbols for each outgoing transition of - /// - /// . - /// + /** + * Calculates the SLL(1) expected lookahead set for each outgoing transition + * of an {@link ATNState}. The returned array has one element for each + * outgoing transition in {@code s}. If the closure from transition + * i leads to a semantic predicate before matching a symbol, the + * element at index i of the result will be {@code null}. + * + * @param s the ATN state + * @return the expected symbols for each outgoing transition of {@code s}. + */ [return: Nullable] public virtual IntervalSet[] GetDecisionLookahead(ATNState s) { @@ -61,7 +48,7 @@ namespace Antlr4.Runtime.Atn HashSet lookBusy = new HashSet(); bool seeThruPreds = false; // fail to get lookahead upon pred - Look(s.Transition(alt).target, null, PredictionContext.EMPTY, look[alt], lookBusy, new BitSet(), seeThruPreds, false); + Look_(s.Transition(alt).target, null, PredictionContext.EMPTY, look[alt], lookBusy, new BitSet(), seeThruPreds, false); // Wipe out lookahead for this alternative if we found nothing // or we had a predicate when we !seeThruPreds if (look[alt].Count == 0 || look[alt].Contains(HitPred)) @@ -72,190 +59,88 @@ namespace Antlr4.Runtime.Atn return look; } - /// - /// Compute set of tokens that can follow - /// - /// in the ATN in the - /// specified - /// - /// . - ///

If - /// - /// is - /// - /// and the end of the rule containing - /// - /// is reached, - /// - /// is added to the result set. - /// If - /// - /// is not - /// - /// and the end of the outermost rule is - /// reached, - /// - /// is added to the result set.

- /// - /// the ATN state - /// - /// the complete parser context, or - /// - /// if the context - /// should be ignored - /// - /// - /// The set of tokens that can follow - /// - /// in the ATN in the - /// specified - /// - /// . - /// + /** + * Compute set of tokens that can follow {@code s} in the ATN in the + * specified {@code ctx}. + * + *

If {@code ctx} is {@code null} and the end of the rule containing + * {@code s} is reached, {@link Token#EPSILON} is added to the result set. + * If {@code ctx} is not {@code null} and the end of the outermost rule is + * reached, {@link Token#EOF} is added to the result set.

+ * + * @param s the ATN state + * @param ctx the complete parser context, or {@code null} if the context + * should be ignored + * + * @return The set of tokens that can follow {@code s} in the ATN in the + * specified {@code ctx}. + */ [return: NotNull] public virtual IntervalSet Look(ATNState s, RuleContext ctx) { return Look(s, null, ctx); } - /// - /// Compute set of tokens that can follow - /// - /// in the ATN in the - /// specified - /// - /// . - ///

If - /// - /// is - /// - /// and the end of the rule containing - /// - /// is reached, - /// - /// is added to the result set. - /// If - /// - /// is not - /// PredictionContext#EMPTY_LOCAL - /// and the end of the outermost rule is - /// reached, - /// - /// is added to the result set.

- /// - /// the ATN state - /// - /// the ATN state to stop at. This can be a - /// - /// to detect epsilon paths through a closure. - /// - /// - /// the complete parser context, or - /// - /// if the context - /// should be ignored - /// - /// - /// The set of tokens that can follow - /// - /// in the ATN in the - /// specified - /// - /// . - /// + /** + * Compute set of tokens that can follow {@code s} in the ATN in the + * specified {@code ctx}. + * + *

If {@code ctx} is {@code null} and the end of the rule containing + * {@code s} is reached, {@link Token#EPSILON} is added to the result set. + * If {@code ctx} is not {@code null} and the end of the outermost rule is + * reached, {@link Token#EOF} is added to the result set.

+ * + * @param s the ATN state + * @param stopState the ATN state to stop at. This can be a + * {@link BlockEndState} to detect epsilon paths through a closure. + * @param ctx the complete parser context, or {@code null} if the context + * should be ignored + * + * @return The set of tokens that can follow {@code s} in the ATN in the + * specified {@code ctx}. + */ [return: NotNull] public virtual IntervalSet Look(ATNState s, ATNState stopState, RuleContext ctx) { IntervalSet r = new IntervalSet(); bool seeThruPreds = true; - PredictionContext lookContext = ctx != null ? PredictionContext.FromRuleContext(s.atn, ctx) : null; - Look(s, stopState, lookContext, r, new HashSet(), new BitSet(), seeThruPreds, true); + PredictionContext lookContext = ctx != null ? PredictionContext.FromRuleContext(s.atn, ctx) : null; + Look_(s, stopState, lookContext, r, new HashSet(), new BitSet(), seeThruPreds, true); return r; } - /// - /// Compute set of tokens that can follow - /// - /// in the ATN in the - /// specified - /// - /// . - ///

- /// If - /// - /// is - /// - /// and - /// - /// or the end of the rule containing - /// - /// is reached, - /// - /// is added to the result set. If - /// - /// is not - /// - /// and - /// - /// is - /// - /// and - /// - /// or the end of the outermost rule is reached, - /// - /// is added to the result set. - ///

- /// the ATN state. - /// - /// the ATN state to stop at. This can be a - /// - /// to detect epsilon paths through a closure. - /// - /// - /// The outer context, or - /// - /// if - /// the outer context should not be used. - /// - /// The result lookahead set. - /// - /// A set used for preventing epsilon closures in the ATN - /// from causing a stack overflow. Outside code should pass - /// new HashSet<ATNConfig> - /// for this argument. - /// - /// - /// A set used for preventing left recursion in the - /// ATN from causing a stack overflow. Outside code should pass - /// new BitSet() - /// for this argument. - /// - /// - /// - /// - /// to true semantic predicates as - /// implicitly - /// - /// and "see through them", otherwise - /// - /// to treat semantic predicates as opaque and add - /// - /// to the - /// result if one is encountered. - /// - /// - /// Add - /// - /// to the result if the end of the - /// outermost context is reached. This parameter has no effect if - /// - /// is - /// - /// . - /// - protected internal virtual void Look(ATNState s, ATNState stopState, PredictionContext ctx, IntervalSet look, HashSet lookBusy, BitSet calledRuleStack, bool seeThruPreds, bool addEOF) + /** + * Compute set of tokens that can follow {@code s} in the ATN in the + * specified {@code ctx}. + * + *

If {@code ctx} is {@code null} and {@code stopState} or the end of the + * rule containing {@code s} is reached, {@link Token#EPSILON} is added to + * the result set. If {@code ctx} is not {@code null} and {@code addEOF} is + * {@code true} and {@code stopState} or the end of the outermost rule is + * reached, {@link Token#EOF} is added to the result set.

+ * + * @param s the ATN state. + * @param stopState the ATN state to stop at. This can be a + * {@link BlockEndState} to detect epsilon paths through a closure. + * @param ctx The outer context, or {@code null} if the outer context should + * not be used. + * @param look The result lookahead set. + * @param lookBusy A set used for preventing epsilon closures in the ATN + * from causing a stack overflow. Outside code should pass + * {@code new HashSet} for this argument. + * @param calledRuleStack A set used for preventing left recursion in the + * ATN from causing a stack overflow. Outside code should pass + * {@code new BitSet()} for this argument. + * @param seeThruPreds {@code true} to true semantic predicates as + * implicitly {@code true} and "see through them", otherwise {@code false} + * to treat semantic predicates as opaque and add {@link #HIT_PRED} to the + * result if one is encountered. + * @param addEOF Add {@link Token#EOF} to the result if the end of the + * outermost context is reached. This parameter has no effect if {@code ctx} + * is {@code null}. + */ + protected internal virtual void Look_(ATNState s, ATNState stopState, PredictionContext ctx, IntervalSet look, HashSet lookBusy, BitSet calledRuleStack, bool seeThruPreds, bool addEOF) { - // System.out.println("_LOOK("+s.stateNumber+", ctx="+ctx); ATNConfig c = new ATNConfig(s, 0, ctx); if (!lookBusy.Add(c)) { @@ -268,50 +153,51 @@ namespace Antlr4.Runtime.Atn look.Add(TokenConstants.EPSILON); return; } - else if (ctx.IsEmpty && addEOF) { - look.Add(TokenConstants.EOF); - return; - } - } - if (s is RuleStopState) - { - if (ctx == null) - { - look.Add(TokenConstants.EPSILON); - return; - } else if (ctx.IsEmpty && addEOF) { look.Add(TokenConstants.EOF); return; } - if (ctx != PredictionContext.EMPTY) - { - for (int i = 0; i < ctx.Size; i++) - { - ATNState returnState = atn.states[ctx.GetReturnState(i)]; - bool removed = calledRuleStack.Get(returnState.ruleIndex); - try - { - calledRuleStack.Clear(returnState.ruleIndex); - Look(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF); - } - finally - { - if (removed) - { - calledRuleStack.Set(returnState.ruleIndex); - } - } - } - return; - } + } + if (s is RuleStopState) + { + if (ctx == null) + { + look.Add(TokenConstants.EPSILON); + return; + } + else if (ctx.IsEmpty && addEOF) + { + look.Add(TokenConstants.EOF); + return; + } + if (ctx != PredictionContext.EMPTY) + { + bool removed = calledRuleStack.Get(s.ruleIndex); + try + { + calledRuleStack.Clear(s.ruleIndex); + for (int i = 0; i < ctx.Size; i++) + { + ATNState returnState = atn.states[ctx.GetReturnState(i)]; + Look_(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF); + } + } + finally + { + if (removed) + { + calledRuleStack.Set(s.ruleIndex); + } + } + return; + } } int n = s.NumberOfTransitions; for (int i_1 = 0; i_1 < n; i_1++) { Transition t = s.Transition(i_1); - if (t is RuleTransition) + if (t.GetType() == typeof(RuleTransition)) { RuleTransition ruleTransition = (RuleTransition)t; if (calledRuleStack.Get(ruleTransition.ruleIndex)) @@ -322,51 +208,42 @@ namespace Antlr4.Runtime.Atn try { calledRuleStack.Set(ruleTransition.target.ruleIndex); - Look(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); + Look_(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); } finally { calledRuleStack.Clear(ruleTransition.target.ruleIndex); } } - else + else if (t is AbstractPredicateTransition) { - if (t is AbstractPredicateTransition) + if (seeThruPreds) { - if (seeThruPreds) - { - Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); - } - else - { - look.Add(HitPred); - } + Look_(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); } else { - if (t.IsEpsilon) + look.Add(HitPred); + } + } + else if (t.IsEpsilon) + { + Look_(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); + } + else if (t.GetType() == typeof(WildcardTransition)) + { + look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); + } + else + { + IntervalSet set = t.Label; + if (set != null) + { + if (t is NotSetTransition) { - Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); - } - else - { - if (t is WildcardTransition) - { - look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); - } - else - { - IntervalSet set = t.Label; - if (set != null) - { - if (t is NotSetTransition) - { - set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); - } - look.AddAll(set); - } - } + set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); } + look.AddAll(set); } } } diff --git a/runtime/CSharp/DefaultErrorStrategy.cs b/runtime/CSharp/DefaultErrorStrategy.cs index 5dee39cdd..f208c610f 100644 --- a/runtime/CSharp/DefaultErrorStrategy.cs +++ b/runtime/CSharp/DefaultErrorStrategy.cs @@ -42,6 +42,21 @@ namespace Antlr4.Runtime protected internal IntervalSet lastErrorStates; + /** + * This field is used to propagate information about the lookahead following + * the previous match. Since prediction prefers completing the current rule + * to error recovery efforts, error reporting may occur later than the + * original point where it was discoverable. The original context is used to + * compute the true expected sets as though the reporting occurred as early + * as possible. + */ + protected ParserRuleContext nextTokensContext; + + /** + * @see #nextTokensContext + */ + protected int nextTokensState; + /// /// ///

The default implementation simply calls @@ -264,8 +279,22 @@ namespace Antlr4.Runtime int la = tokens.LA(1); // try cheaper subset first; might get lucky. seems to shave a wee bit off var nextTokens = recognizer.Atn.NextTokens(s); - if (nextTokens.Contains(TokenConstants.EPSILON) || nextTokens.Contains(la)) + if (nextTokens.Contains(la)) { + nextTokensContext = null; + nextTokensState = ATNState.InvalidStateNumber; + return; + } + + if (nextTokens.Contains(TokenConstants.EPSILON)) + { + if (nextTokensContext == null) + { + // It's possible the next token won't match; information tracked + // by sync is restricted for performance. + nextTokensContext = recognizer.Context; + nextTokensState = recognizer.State; + } return; } switch (s.StateType) From 7a9a26c7ecb9d30f48e18eb097440946f9dd084a Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 23 Jan 2021 17:22:17 +0800 Subject: [PATCH 06/42] fix #3042 in Python 3 runtime --- runtime/Python3/src/antlr4/LL1Analyzer.py | 18 +++++++++--------- .../Python3/src/antlr4/error/ErrorStrategy.py | 13 ++++++++++++- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/runtime/Python3/src/antlr4/LL1Analyzer.py b/runtime/Python3/src/antlr4/LL1Analyzer.py index a200a5d0e..ac149fa2f 100644 --- a/runtime/Python3/src/antlr4/LL1Analyzer.py +++ b/runtime/Python3/src/antlr4/LL1Analyzer.py @@ -132,16 +132,16 @@ class LL1Analyzer (object): return if ctx != PredictionContext.EMPTY: - # run thru all possible stack tops in ctx - for i in range(0, len(ctx)): - returnState = self.atn.states[ctx.getReturnState(i)] - removed = returnState.ruleIndex in calledRuleStack - try: - calledRuleStack.discard(returnState.ruleIndex) + removed = s.ruleIndex in calledRuleStack + try: + calledRuleStack.discard(s.ruleIndex) + # run thru all possible stack tops in ctx + for i in range(0, len(ctx)): + returnState = self.atn.states[ctx.getReturnState(i)] self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF) - finally: - if removed: - calledRuleStack.add(returnState.ruleIndex) + finally: + if removed: + calledRuleStack.add(s.ruleIndex) return for t in s.transitions: diff --git a/runtime/Python3/src/antlr4/error/ErrorStrategy.py b/runtime/Python3/src/antlr4/error/ErrorStrategy.py index d7538b607..0f7caadb2 100644 --- a/runtime/Python3/src/antlr4/error/ErrorStrategy.py +++ b/runtime/Python3/src/antlr4/error/ErrorStrategy.py @@ -58,6 +58,8 @@ class DefaultErrorStrategy(ErrorStrategy): # self.lastErrorIndex = -1 self.lastErrorStates = None + self.nextTokensContext = None + self.nextTokenState = 0 #

The default implementation simply calls {@link #endErrorCondition} to # ensure that the handler is not in error recovery mode.

@@ -208,7 +210,16 @@ class DefaultErrorStrategy(ErrorStrategy): la = recognizer.getTokenStream().LA(1) # try cheaper subset first; might get lucky. seems to shave a wee bit off nextTokens = recognizer.atn.nextTokens(s) - if Token.EPSILON in nextTokens or la in nextTokens: + if la in nextTokens: + self.nextTokensContext = None + self.nextTokenState = ATNState.INVALID_STATE_NUMBER + return + elif Token.EPSILON in nextTokens: + if self.nextTokensContext is None: + # It's possible the next token won't match information tracked + # by sync is restricted for performance. + self.nextTokensContext = recognizer._ctx + self.nextTokensState = recognizer._stateNumber return if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START, From a9f11612dd24f1857d33227ab1671720df14618e Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 23 Jan 2021 17:23:45 +0800 Subject: [PATCH 07/42] fix #3042 in Python 2 runtime --- runtime/Python2/src/antlr4/LL1Analyzer.py | 26 +++++++++---------- .../Python2/src/antlr4/error/ErrorStrategy.py | 13 +++++++++- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/runtime/Python2/src/antlr4/LL1Analyzer.py b/runtime/Python2/src/antlr4/LL1Analyzer.py index 7403f8c78..1c89c492e 100644 --- a/runtime/Python2/src/antlr4/LL1Analyzer.py +++ b/runtime/Python2/src/antlr4/LL1Analyzer.py @@ -129,16 +129,16 @@ class LL1Analyzer (object): return if ctx != PredictionContext.EMPTY: - # run thru all possible stack tops in ctx - for i in range(0, len(ctx)): - returnState = self.atn.states[ctx.getReturnState(i)] - removed = returnState.ruleIndex in calledRuleStack - try: - calledRuleStack.discard(returnState.ruleIndex) + removed = s.ruleIndex in calledRuleStack + try: + calledRuleStack.discard(s.ruleIndex) + # run thru all possible stack tops in ctx + for i in range(0, len(ctx)): + returnState = self.atn.states[ctx.getReturnState(i)] self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF) - finally: - if removed: - calledRuleStack.add(returnState.ruleIndex) + finally: + if removed: + calledRuleStack.add(s.ruleIndex) return for t in s.transitions: @@ -163,8 +163,8 @@ class LL1Analyzer (object): elif type(t) == WildcardTransition: look.addRange( Interval(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType + 1) ) else: - set = t.label - if set is not None: + set_ = t.label + if set_ is not None: if isinstance(t, NotSetTransition): - set = set.complement(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType) - look.addSet(set) + set_ = set_.complement(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType) + look.addSet(set_) diff --git a/runtime/Python2/src/antlr4/error/ErrorStrategy.py b/runtime/Python2/src/antlr4/error/ErrorStrategy.py index c0c4001b3..b70c7df8e 100644 --- a/runtime/Python2/src/antlr4/error/ErrorStrategy.py +++ b/runtime/Python2/src/antlr4/error/ErrorStrategy.py @@ -53,6 +53,8 @@ class DefaultErrorStrategy(ErrorStrategy): # self.lastErrorIndex = -1 self.lastErrorStates = None + self.nextTokensContext = None + self.nextTokenState = 0 #

The default implementation simply calls {@link #endErrorCondition} to # ensure that the handler is not in error recovery mode.

@@ -203,7 +205,16 @@ class DefaultErrorStrategy(ErrorStrategy): la = recognizer.getTokenStream().LA(1) # try cheaper subset first; might get lucky. seems to shave a wee bit off nextTokens = recognizer.atn.nextTokens(s) - if Token.EPSILON in nextTokens or la in nextTokens: + if la in nextTokens: + self.nextTokensContext = None + self.nextTokenState = ATNState.INVALID_STATE_NUMBER + return + elif Token.EPSILON in nextTokens: + if self.nextTokensContext is None: + # It's possible the next token won't match information tracked + # by sync is restricted for performance. + self.nextTokensContext = recognizer._ctx + self.nextTokensState = recognizer._stateNumber return if s.stateType in [ATNState.BLOCK_START, ATNState.STAR_BLOCK_START, From 329f1301ea6f413a4038f701086c248e45ab734e Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 23 Jan 2021 17:32:06 +0800 Subject: [PATCH 08/42] fix #3042 in JavaScript runtime --- runtime/JavaScript/src/antlr4/LL1Analyzer.js | 20 +++++++++---------- .../src/antlr4/error/ErrorStrategy.js | 20 +++++++++++++++---- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/runtime/JavaScript/src/antlr4/LL1Analyzer.js b/runtime/JavaScript/src/antlr4/LL1Analyzer.js index e76238dd7..18ed87899 100644 --- a/runtime/JavaScript/src/antlr4/LL1Analyzer.js +++ b/runtime/JavaScript/src/antlr4/LL1Analyzer.js @@ -128,17 +128,17 @@ class LL1Analyzer { return; } if (ctx !== PredictionContext.EMPTY) { - // run thru all possible stack tops in ctx - for(let i=0; i Date: Sat, 23 Jan 2021 18:18:30 +0800 Subject: [PATCH 09/42] fix typo --- runtime/JavaScript/src/antlr4/error/ErrorStrategy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/JavaScript/src/antlr4/error/ErrorStrategy.js b/runtime/JavaScript/src/antlr4/error/ErrorStrategy.js index 44a161014..3da80f28b 100644 --- a/runtime/JavaScript/src/antlr4/error/ErrorStrategy.js +++ b/runtime/JavaScript/src/antlr4/error/ErrorStrategy.js @@ -227,7 +227,7 @@ class DefaultErrorStrategy extends ErrorStrategy { this.nextTokenState = ATNState.INVALID_STATE_NUMBER; return; } else if (nextTokens.contains(Token.EPSILON)) { - if(self.nextTokensContext === null) { + if(this.nextTokensContext === null) { // It's possible the next token won't match information tracked // by sync is restricted for performance. this.nextTokensContext = recognizer._ctx; From 559d5266484c5565d8fe2b15a5a2ed359bd71b03 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 23 Jan 2021 18:47:54 +0800 Subject: [PATCH 10/42] fix #3042 in Cpp runtime --- runtime/Cpp/runtime/src/atn/LL1Analyzer.cpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/runtime/Cpp/runtime/src/atn/LL1Analyzer.cpp b/runtime/Cpp/runtime/src/atn/LL1Analyzer.cpp index d7949cd1e..ddca80088 100755 --- a/runtime/Cpp/runtime/src/atn/LL1Analyzer.cpp +++ b/runtime/Cpp/runtime/src/atn/LL1Analyzer.cpp @@ -100,18 +100,16 @@ void LL1Analyzer::_LOOK(ATNState *s, ATNState *stopState, Ref } if (ctx != PredictionContext::EMPTY) { - // run thru all possible stack tops in ctx + bool removed = calledRuleStack.test(s->ruleIndex); + calledRuleStack[s->ruleIndex] = false; + auto onExit = finally([removed, &calledRuleStack, s] { + if (removed) { + calledRuleStack.set(s->ruleIndex); + } + }); + // run thru all possible stack tops in ctx for (size_t i = 0; i < ctx->size(); i++) { ATNState *returnState = _atn.states[ctx->getReturnState(i)]; - - bool removed = calledRuleStack.test(returnState->ruleIndex); - auto onExit = finally([removed, &calledRuleStack, returnState] { - if (removed) { - calledRuleStack.set(returnState->ruleIndex); - } - }); - - calledRuleStack[returnState->ruleIndex] = false; _LOOK(returnState, stopState, ctx->getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF); } return; From 43226724350f73066de4498ab5f06c1fc56c22ff Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 23 Jan 2021 18:50:21 +0800 Subject: [PATCH 11/42] fix #3042 in Go runtime --- runtime/Go/antlr/ll1_analyzer.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/runtime/Go/antlr/ll1_analyzer.go b/runtime/Go/antlr/ll1_analyzer.go index f5afd09b3..3ebc40a76 100644 --- a/runtime/Go/antlr/ll1_analyzer.go +++ b/runtime/Go/antlr/ll1_analyzer.go @@ -112,16 +112,6 @@ func (la *LL1Analyzer) Look(s, stopState ATNState, ctx RuleContext) *IntervalSet func (la *LL1Analyzer) look2(s, stopState ATNState, ctx PredictionContext, look *IntervalSet, lookBusy *Set, calledRuleStack *BitSet, seeThruPreds, addEOF bool, i int) { returnState := la.atn.states[ctx.getReturnState(i)] - - removed := calledRuleStack.contains(returnState.GetRuleIndex()) - - defer func() { - if removed { - calledRuleStack.add(returnState.GetRuleIndex()) - } - }() - - calledRuleStack.remove(returnState.GetRuleIndex()) la.look1(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF) } @@ -158,6 +148,13 @@ func (la *LL1Analyzer) look1(s, stopState ATNState, ctx PredictionContext, look } if ctx != BasePredictionContextEMPTY { + removed := calledRuleStack.contains(s.GetRuleIndex()) + defer func() { + if removed { + calledRuleStack.add(s.GetRuleIndex()) + } + }() + calledRuleStack.remove(s.GetRuleIndex()) // run thru all possible stack tops in ctx for i := 0; i < ctx.length(); i++ { returnState := la.atn.states[ctx.getReturnState(i)] From 58da376bef2813973cbfdf61f1ac8a25f1db7619 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 23 Jan 2021 19:19:02 +0800 Subject: [PATCH 12/42] fix #3042 in Swift runtime --- runtime/Swift/Sources/Antlr4/atn/LL1Analyzer.swift | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/runtime/Swift/Sources/Antlr4/atn/LL1Analyzer.swift b/runtime/Swift/Sources/Antlr4/atn/LL1Analyzer.swift index 3f594f234..bc179a1d0 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LL1Analyzer.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LL1Analyzer.swift @@ -169,16 +169,18 @@ public class LL1Analyzer { } if ctx != PredictionContext.EMPTY { + let removed = try! calledRuleStack.get(s.ruleIndex!) + try! calledRuleStack.clear(s.ruleIndex!) + defer { + if removed { + try! calledRuleStack.set(s.ruleIndex!) + } + } // run thru all possible stack tops in ctx let length = ctx.size() for i in 0.. Date: Sat, 23 Jan 2021 20:39:53 +0800 Subject: [PATCH 13/42] Fix various javascript warnings and typos --- runtime/JavaScript/src/antlr4/CharStreams.js | 2 +- runtime/JavaScript/src/antlr4/IntervalSet.js | 4 +-- runtime/JavaScript/src/antlr4/Utils.js | 6 ++-- .../JavaScript/src/antlr4/atn/ATNConfig.js | 2 +- .../src/antlr4/atn/ParserATNSimulator.js | 26 +++++++------- .../src/antlr4/atn/PredictionMode.js | 4 +-- runtime/JavaScript/src/antlr4/dfa/DFA.js | 2 +- runtime/JavaScript/src/antlr4/error/Errors.js | 7 ++-- .../src/antlr4/polyfills/codepointat.js | 24 +++++++------ .../src/antlr4/polyfills/fromcodepoint.js | 35 ++++++++++--------- 10 files changed, 58 insertions(+), 54 deletions(-) diff --git a/runtime/JavaScript/src/antlr4/CharStreams.js b/runtime/JavaScript/src/antlr4/CharStreams.js index 045f93df5..fcf4ae8f2 100644 --- a/runtime/JavaScript/src/antlr4/CharStreams.js +++ b/runtime/JavaScript/src/antlr4/CharStreams.js @@ -73,4 +73,4 @@ const CharStreams = { } }; -module.exports = CharStreams +module.exports = CharStreams; diff --git a/runtime/JavaScript/src/antlr4/IntervalSet.js b/runtime/JavaScript/src/antlr4/IntervalSet.js index e301fa8f0..c802bc82f 100644 --- a/runtime/JavaScript/src/antlr4/IntervalSet.js +++ b/runtime/JavaScript/src/antlr4/IntervalSet.js @@ -99,11 +99,11 @@ class IntervalSet { const r = this.intervals[k + 1]; // if r contained in l if (l.stop >= r.stop) { - this.intervals.pop(k + 1); + this.intervals = this.intervals.splice(k + 1, 1); this.reduce(k); } else if (l.stop >= r.start) { this.intervals[k] = new Interval(l.start, r.stop); - this.intervals.pop(k + 1); + this.intervals = this.intervals.splice(k + 1, 1); } } } diff --git a/runtime/JavaScript/src/antlr4/Utils.js b/runtime/JavaScript/src/antlr4/Utils.js index 99595260f..8eed91a2c 100644 --- a/runtime/JavaScript/src/antlr4/Utils.js +++ b/runtime/JavaScript/src/antlr4/Utils.js @@ -423,12 +423,12 @@ function titleCase(str) { function equalArrays(a, b) { if (!Array.isArray(a) || !Array.isArray(b)) return false; - if (a == b) + if (a === b) return true; - if (a.length != b.length) + if (a.length !== b.length) return false; for (let i = 0; i < a.length; i++) { - if (a[i] == b[i]) + if (a[i] === b[i]) continue; if (!a[i].equals || !a[i].equals(b[i])) return false; diff --git a/runtime/JavaScript/src/antlr4/atn/ATNConfig.js b/runtime/JavaScript/src/antlr4/atn/ATNConfig.js index 73e4c86f6..b5c4f9927 100644 --- a/runtime/JavaScript/src/antlr4/atn/ATNConfig.js +++ b/runtime/JavaScript/src/antlr4/atn/ATNConfig.js @@ -156,7 +156,7 @@ class LexerATNConfig extends ATNConfig { equals(other) { return this === other || (other instanceof LexerATNConfig && - this.passedThroughNonGreedyDecision == other.passedThroughNonGreedyDecision && + this.passedThroughNonGreedyDecision === other.passedThroughNonGreedyDecision && (this.lexerActionExecutor ? this.lexerActionExecutor.equals(other.lexerActionExecutor) : !other.lexerActionExecutor) && super.equals(other)); } diff --git a/runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js b/runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js index 6eb24a171..2c4833771 100644 --- a/runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js +++ b/runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js @@ -589,7 +589,7 @@ class ParserATNSimulator extends ATNSimulator { } const fullCtx = true; let foundExactAmbig = false; - let reach = null; + let reach; let previous = s0; input.seek(startIndex); let t = input.LA(1); @@ -1253,7 +1253,7 @@ class ParserATNSimulator extends ATNSimulator { // both epsilon transitions and non-epsilon transitions. } for(let i = 0;i= 0 && this.startIndex < this.input.size) { - symbol = this.input.getText((this.startIndex,this.startIndex)); + symbol = this.input.getText(new Interval(this.startIndex,this.startIndex)); } return "LexerNoViableAltException" + symbol; } diff --git a/runtime/JavaScript/src/antlr4/polyfills/codepointat.js b/runtime/JavaScript/src/antlr4/polyfills/codepointat.js index f724c8926..616f254d4 100644 --- a/runtime/JavaScript/src/antlr4/polyfills/codepointat.js +++ b/runtime/JavaScript/src/antlr4/polyfills/codepointat.js @@ -4,22 +4,24 @@ if (!String.prototype.codePointAt) { 'use strict'; // needed to support `apply`/`call` with `undefined`/`null` var defineProperty = (function() { // IE 8 only supports `Object.defineProperty` on DOM elements + let result; try { - var object = {}; - var $defineProperty = Object.defineProperty; - var result = $defineProperty(object, object, object) && $defineProperty; - } catch(error) {} + const object = {}; + const $defineProperty = Object.defineProperty; + result = $defineProperty(object, object, object) && $defineProperty; + } catch(error) { + } return result; }()); - var codePointAt = function(position) { + const codePointAt = function(position) { if (this == null) { throw TypeError(); } - var string = String(this); - var size = string.length; + const string = String(this); + const size = string.length; // `ToInteger` - var index = position ? Number(position) : 0; - if (index != index) { // better `isNaN` + let index = position ? Number(position) : 0; + if (index !== index) { // better `isNaN` index = 0; } // Account for out-of-bounds indices: @@ -27,8 +29,8 @@ if (!String.prototype.codePointAt) { return undefined; } // Get the first code unit - var first = string.charCodeAt(index); - var second; + const first = string.charCodeAt(index); + let second; if ( // check if it’s the start of a surrogate pair first >= 0xD800 && first <= 0xDBFF && // high surrogate size > index + 1 // there is a next code unit diff --git a/runtime/JavaScript/src/antlr4/polyfills/fromcodepoint.js b/runtime/JavaScript/src/antlr4/polyfills/fromcodepoint.js index adae6909e..cf53f93f6 100644 --- a/runtime/JavaScript/src/antlr4/polyfills/fromcodepoint.js +++ b/runtime/JavaScript/src/antlr4/polyfills/fromcodepoint.js @@ -1,35 +1,36 @@ /*! https://mths.be/fromcodepoint v0.2.1 by @mathias */ if (!String.fromCodePoint) { (function() { - var defineProperty = (function() { + const defineProperty = (function() { // IE 8 only supports `Object.defineProperty` on DOM elements + let result; try { - var object = {}; - var $defineProperty = Object.defineProperty; - var result = $defineProperty(object, object, object) && $defineProperty; + const object = {}; + const $defineProperty = Object.defineProperty; + result = $defineProperty(object, object, object) && $defineProperty; } catch(error) {} return result; }()); - var stringFromCharCode = String.fromCharCode; - var floor = Math.floor; - var fromCodePoint = function(_) { - var MAX_SIZE = 0x4000; - var codeUnits = []; - var highSurrogate; - var lowSurrogate; - var index = -1; - var length = arguments.length; + const stringFromCharCode = String.fromCharCode; + const floor = Math.floor; + const fromCodePoint = function(_) { + const MAX_SIZE = 0x4000; + const codeUnits = []; + let highSurrogate; + let lowSurrogate; + let index = -1; + const length = arguments.length; if (!length) { return ''; } - var result = ''; + let result = ''; while (++index < length) { - var codePoint = Number(arguments[index]); + let codePoint = Number(arguments[index]); if ( !isFinite(codePoint) || // `NaN`, `+Infinity`, or `-Infinity` codePoint < 0 || // not a valid Unicode code point codePoint > 0x10FFFF || // not a valid Unicode code point - floor(codePoint) != codePoint // not an integer + floor(codePoint) !== codePoint // not an integer ) { throw RangeError('Invalid code point: ' + codePoint); } @@ -42,7 +43,7 @@ if (!String.fromCodePoint) { lowSurrogate = (codePoint % 0x400) + 0xDC00; codeUnits.push(highSurrogate, lowSurrogate); } - if (index + 1 == length || codeUnits.length > MAX_SIZE) { + if (index + 1 === length || codeUnits.length > MAX_SIZE) { result += stringFromCharCode.apply(null, codeUnits); codeUnits.length = 0; } From 020333deffc0d5181b046a8edad9f47d95b77b6d Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sun, 24 Jan 2021 13:46:58 +0800 Subject: [PATCH 14/42] work around Swift crash where reading hashValue on SemanticContext.AND instance would call the empty constructor (possibly a Swift bug?) --- .../Sources/Antlr4/atn/LookupDictionary.swift | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/runtime/Swift/Sources/Antlr4/atn/LookupDictionary.swift b/runtime/Swift/Sources/Antlr4/atn/LookupDictionary.swift index cd0af9f54..fca8ae536 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LookupDictionary.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LookupDictionary.swift @@ -28,13 +28,27 @@ public struct LookupDictionary { private func hash(_ config: ATNConfig) -> Int { if type == LookupDictionaryType.lookup { - - var hashCode: Int = 7 - hashCode = 31 * hashCode + config.state.stateNumber - hashCode = 31 * hashCode + config.alt - hashCode = 31 * hashCode + config.semanticContext.hashValue - return hashCode - + /* migrating to XCode 12.3/Swift 5.3 introduced a very weird bug + where reading hashValue from a SemanticContext.AND instance woul: + call the AND empty constructor + NOT call AND.hash(into) + Could it be a Swift compiler bug ? + All tests pass when using Hasher.combine() + Keeping the old code for reference: + + var hashCode: Int = 7 + hashCode = 31 * hashCode + config.state.stateNumber + hashCode = 31 * hashCode + config.alt + hashCode = 31 * hashCode + config.semanticContext.hashValue // <- the crash would occur here + return hashCode + + */ + var hasher = Hasher() + hasher.combine(7) + hasher.combine(config.state.stateNumber) + hasher.combine(config.alt) + hasher.combine(config.semanticContext) + return hasher.finalize() } else { //Ordered return config.hashValue From 4c2ccdb2c0f6a873b871beb99ec24f203fc4b703 Mon Sep 17 00:00:00 2001 From: ericvergnaud Date: Sun, 24 Jan 2021 15:18:42 +0800 Subject: [PATCH 15/42] Update and rename swift-macosx.yml to macosx.yml --- .github/workflows/{swift-macosx.yml => macosx.yml} | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) rename .github/workflows/{swift-macosx.yml => macosx.yml} (76%) diff --git a/.github/workflows/swift-macosx.yml b/.github/workflows/macosx.yml similarity index 76% rename from .github/workflows/swift-macosx.yml rename to .github/workflows/macosx.yml index 37fa14128..ff518868b 100644 --- a/.github/workflows/swift-macosx.yml +++ b/.github/workflows/macosx.yml @@ -12,10 +12,8 @@ jobs: strategy: fail-fast: false matrix: - include: - - GROUP: LEXER - - GROUP: PARSER - - GROUP: RECURSION + GROUP: [LEXER, PARSER, RECURSION] + TARGET: [swift, cpp, dotnet] steps: - uses: actions/checkout@v2 - name: Set up JDK 1.8 @@ -29,7 +27,7 @@ jobs: - name: Build tool with Maven run: mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V - name: Test with Maven - run: arch -x86_64 .github/scripts/run-tests-swift.sh + run: arch -x86_64 .github/scripts/run-tests-${{ matrix.TARGET }}.sh env: - TARGET: swift + TARGET: ${{ matrix.TARGET }} GROUP: ${{ matrix.GROUP }} From 5b34cc1ec3750f31aef34405489356d9ab51e536 Mon Sep 17 00:00:00 2001 From: ericvergnaud Date: Sun, 24 Jan 2021 15:18:59 +0800 Subject: [PATCH 16/42] Update macosx.yml --- .github/workflows/macosx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index ff518868b..cbce1a2ee 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -1,4 +1,4 @@ -name: Swift/MacOSX +name: MacOSX on: push: From 47fd7026b309be4967672350bc403cf7f7e7ca1f Mon Sep 17 00:00:00 2001 From: ericvergnaud Date: Sun, 24 Jan 2021 15:23:28 +0800 Subject: [PATCH 17/42] Update macosx.yml --- .github/workflows/macosx.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index cbce1a2ee..2d696fd16 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -12,8 +12,10 @@ jobs: strategy: fail-fast: false matrix: - GROUP: [LEXER, PARSER, RECURSION] - TARGET: [swift, cpp, dotnet] + # GROUP: [LEXER, PARSER, RECURSION] + # TARGET: [swift, cpp, dotnet] + GROUP: [LEXER] + TARGET: [cpp] steps: - uses: actions/checkout@v2 - name: Set up JDK 1.8 From 142a32ed5a72c6f4b11e86ab7999719fc2dee69f Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sun, 24 Jan 2021 16:12:55 +0800 Subject: [PATCH 18/42] enable self-hosted CI for cpp runtime --- .github/scripts/run-tests-cpp.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 .github/scripts/run-tests-cpp.sh diff --git a/.github/scripts/run-tests-cpp.sh b/.github/scripts/run-tests-cpp.sh new file mode 100755 index 000000000..e95dc8922 --- /dev/null +++ b/.github/scripts/run-tests-cpp.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euo pipefail + +if [ $GROUP == "LEXER" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=cpp.* test +elif [ $GROUP == "PARSER" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=cpp.* test +elif [ $GROUP == "RECURSION" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=cpp.* test +else + mvn -q -Dtest=cpp.* test +fi + From 4d0bc718dfbdb0d1156ccb8ced9b7fbd6370bf3c Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sun, 24 Jan 2021 16:16:45 +0800 Subject: [PATCH 19/42] no message --- .github/scripts/run-tests-cpp.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/run-tests-cpp.sh b/.github/scripts/run-tests-cpp.sh index e95dc8922..6fad9df27 100755 --- a/.github/scripts/run-tests-cpp.sh +++ b/.github/scripts/run-tests-cpp.sh @@ -2,6 +2,8 @@ set -euo pipefail +cd runtime-testsuite/ + if [ $GROUP == "LEXER" ]; then mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=cpp.* test elif [ $GROUP == "PARSER" ]; then @@ -11,4 +13,3 @@ elif [ $GROUP == "RECURSION" ]; then else mvn -q -Dtest=cpp.* test fi - From d3e2fdae79937049d9ad60fb08e6bfd635684085 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sun, 24 Jan 2021 16:46:59 +0800 Subject: [PATCH 20/42] try self-hosted CI for dotnet on mac OS --- .github/scripts/install-dotnet-on-osx.sh | 1112 ++++++++++++++++++ .github/scripts/mac-action-runner-readme.txt | 3 + .github/workflows/macosx.yml | 2 +- 3 files changed, 1116 insertions(+), 1 deletion(-) create mode 100755 .github/scripts/install-dotnet-on-osx.sh diff --git a/.github/scripts/install-dotnet-on-osx.sh b/.github/scripts/install-dotnet-on-osx.sh new file mode 100755 index 000000000..82f718e59 --- /dev/null +++ b/.github/scripts/install-dotnet-on-osx.sh @@ -0,0 +1,1112 @@ +#!/usr/bin/env bash + +# Copied from https://dot.net/v1/dotnet-install.sh (see also https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script#recommended-version) + +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +# Stop script on NZEC +set -e +# Stop script if unbound variable found (use ${var:-} if intentional) +set -u +# By default cmd1 | cmd2 returns exit code of cmd2 regardless of cmd1 success +# This is causing it to fail +set -o pipefail + +# Use in the the functions: eval $invocation +invocation='say_verbose "Calling: ${yellow:-}${FUNCNAME[0]} ${green:-}$*${normal:-}"' + +# standard output may be used as a return value in the functions +# we need a way to write text on the screen in the functions so that +# it won't interfere with the return value. +# Exposing stream 3 as a pipe to standard output of the script itself +exec 3>&1 + +# Setup some colors to use. These need to work in fairly limited shells, like the Ubuntu Docker container where there are only 8 colors. +# See if stdout is a terminal +if [ -t 1 ] && command -v tput > /dev/null; then + # see if it supports colors + ncolors=$(tput colors) + if [ -n "$ncolors" ] && [ $ncolors -ge 8 ]; then + bold="$(tput bold || echo)" + normal="$(tput sgr0 || echo)" + black="$(tput setaf 0 || echo)" + red="$(tput setaf 1 || echo)" + green="$(tput setaf 2 || echo)" + yellow="$(tput setaf 3 || echo)" + blue="$(tput setaf 4 || echo)" + magenta="$(tput setaf 5 || echo)" + cyan="$(tput setaf 6 || echo)" + white="$(tput setaf 7 || echo)" + fi +fi + +say_warning() { + printf "%b\n" "${yellow:-}dotnet_install: Warning: $1${normal:-}" +} + +say_err() { + printf "%b\n" "${red:-}dotnet_install: Error: $1${normal:-}" >&2 +} + +say() { + # using stream 3 (defined in the beginning) to not interfere with stdout of functions + # which may be used as return value + printf "%b\n" "${cyan:-}dotnet-install:${normal:-} $1" >&3 +} + +say_verbose() { + if [ "$verbose" = true ]; then + say "$1" + fi +} + +# This platform list is finite - if the SDK/Runtime has supported Linux distribution-specific assets, +# then and only then should the Linux distribution appear in this list. +# Adding a Linux distribution to this list does not imply distribution-specific support. +get_legacy_os_name_from_platform() { + eval $invocation + + platform="$1" + case "$platform" in + "centos.7") + echo "centos" + return 0 + ;; + "debian.8") + echo "debian" + return 0 + ;; + "debian.9") + echo "debian.9" + return 0 + ;; + "fedora.23") + echo "fedora.23" + return 0 + ;; + "fedora.24") + echo "fedora.24" + return 0 + ;; + "fedora.27") + echo "fedora.27" + return 0 + ;; + "fedora.28") + echo "fedora.28" + return 0 + ;; + "opensuse.13.2") + echo "opensuse.13.2" + return 0 + ;; + "opensuse.42.1") + echo "opensuse.42.1" + return 0 + ;; + "opensuse.42.3") + echo "opensuse.42.3" + return 0 + ;; + "rhel.7"*) + echo "rhel" + return 0 + ;; + "ubuntu.14.04") + echo "ubuntu" + return 0 + ;; + "ubuntu.16.04") + echo "ubuntu.16.04" + return 0 + ;; + "ubuntu.16.10") + echo "ubuntu.16.10" + return 0 + ;; + "ubuntu.18.04") + echo "ubuntu.18.04" + return 0 + ;; + "alpine.3.4.3") + echo "alpine" + return 0 + ;; + esac + return 1 +} + +get_linux_platform_name() { + eval $invocation + + if [ -n "$runtime_id" ]; then + echo "${runtime_id%-*}" + return 0 + else + if [ -e /etc/os-release ]; then + . /etc/os-release + echo "$ID${VERSION_ID:+.${VERSION_ID}}" + return 0 + elif [ -e /etc/redhat-release ]; then + local redhatRelease=$(&1 || true) | grep -q musl +} + +get_current_os_name() { + eval $invocation + + local uname=$(uname) + if [ "$uname" = "Darwin" ]; then + echo "osx" + return 0 + elif [ "$uname" = "FreeBSD" ]; then + echo "freebsd" + return 0 + elif [ "$uname" = "Linux" ]; then + local linux_platform_name + linux_platform_name="$(get_linux_platform_name)" || { echo "linux" && return 0 ; } + + if [ "$linux_platform_name" = "rhel.6" ]; then + echo $linux_platform_name + return 0 + elif is_musl_based_distro; then + echo "linux-musl" + return 0 + else + echo "linux" + return 0 + fi + fi + + say_err "OS name could not be detected: UName = $uname" + return 1 +} + +get_legacy_os_name() { + eval $invocation + + local uname=$(uname) + if [ "$uname" = "Darwin" ]; then + echo "osx" + return 0 + elif [ -n "$runtime_id" ]; then + echo $(get_legacy_os_name_from_platform "${runtime_id%-*}" || echo "${runtime_id%-*}") + return 0 + else + if [ -e /etc/os-release ]; then + . /etc/os-release + os=$(get_legacy_os_name_from_platform "$ID${VERSION_ID:+.${VERSION_ID}}" || echo "") + if [ -n "$os" ]; then + echo "$os" + return 0 + fi + fi + fi + + say_verbose "Distribution specific OS name and version could not be detected: UName = $uname" + return 1 +} + +machine_has() { + eval $invocation + + hash "$1" > /dev/null 2>&1 + return $? +} + + +check_min_reqs() { + local hasMinimum=false + if machine_has "curl"; then + hasMinimum=true + elif machine_has "wget"; then + hasMinimum=true + fi + + if [ "$hasMinimum" = "false" ]; then + say_err "curl (recommended) or wget are required to download dotnet. Install missing prerequisite to proceed." + return 1 + fi + return 0 +} + +# args: +# input - $1 +to_lowercase() { + #eval $invocation + + echo "$1" | tr '[:upper:]' '[:lower:]' + return 0 +} + +# args: +# input - $1 +remove_trailing_slash() { + #eval $invocation + + local input="${1:-}" + echo "${input%/}" + return 0 +} + +# args: +# input - $1 +remove_beginning_slash() { + #eval $invocation + + local input="${1:-}" + echo "${input#/}" + return 0 +} + +# args: +# root_path - $1 +# child_path - $2 - this parameter can be empty +combine_paths() { + eval $invocation + + # TODO: Consider making it work with any number of paths. For now: + if [ ! -z "${3:-}" ]; then + say_err "combine_paths: Function takes two parameters." + return 1 + fi + + local root_path="$(remove_trailing_slash "$1")" + local child_path="$(remove_beginning_slash "${2:-}")" + say_verbose "combine_paths: root_path=$root_path" + say_verbose "combine_paths: child_path=$child_path" + echo "$root_path/$child_path" + return 0 +} + +get_machine_architecture() { + eval $invocation + + if command -v uname > /dev/null; then + CPUName=$(uname -m) + case $CPUName in + armv7l) + echo "arm" + return 0 + ;; + aarch64) + echo "arm64" + return 0 + ;; + esac + fi + + # Always default to 'x64' + echo "x64" + return 0 +} + +# args: +# architecture - $1 +get_normalized_architecture_from_architecture() { + eval $invocation + + local architecture="$(to_lowercase "$1")" + case "$architecture" in + \) + echo "$(get_normalized_architecture_from_architecture "$(get_machine_architecture)")" + return 0 + ;; + amd64|x64) + echo "x64" + return 0 + ;; + arm) + echo "arm" + return 0 + ;; + arm64) + echo "arm64" + return 0 + ;; + esac + + say_err "Architecture \`$architecture\` not supported. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues" + return 1 +} + +# The version text returned from the feeds is a 1-line or 2-line string: +# For the SDK and the dotnet runtime (2 lines): +# Line 1: # commit_hash +# Line 2: # 4-part version +# For the aspnetcore runtime (1 line): +# Line 1: # 4-part version + +# args: +# version_text - stdin +get_version_from_version_info() { + eval $invocation + + cat | tail -n 1 | sed 's/\r$//' + return 0 +} + +# args: +# install_root - $1 +# relative_path_to_package - $2 +# specific_version - $3 +is_dotnet_package_installed() { + eval $invocation + + local install_root="$1" + local relative_path_to_package="$2" + local specific_version="${3//[$'\t\r\n']}" + + local dotnet_package_path="$(combine_paths "$(combine_paths "$install_root" "$relative_path_to_package")" "$specific_version")" + say_verbose "is_dotnet_package_installed: dotnet_package_path=$dotnet_package_path" + + if [ -d "$dotnet_package_path" ]; then + return 0 + else + return 1 + fi +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +# coherent - $4 +get_latest_version_info() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + local coherent="$4" + + local version_file_url=null + if [[ "$runtime" == "dotnet" ]]; then + version_file_url="$uncached_feed/Runtime/$channel/latest.version" + elif [[ "$runtime" == "aspnetcore" ]]; then + version_file_url="$uncached_feed/aspnetcore/Runtime/$channel/latest.version" + elif [ -z "$runtime" ]; then + if [ "$coherent" = true ]; then + version_file_url="$uncached_feed/Sdk/$channel/latest.coherent.version" + else + version_file_url="$uncached_feed/Sdk/$channel/latest.version" + fi + else + say_err "Invalid value for \$runtime" + return 1 + fi + say_verbose "get_latest_version_info: latest url: $version_file_url" + + download "$version_file_url" + return $? +} + +# args: +# json_file - $1 +parse_jsonfile_for_version() { + eval $invocation + + local json_file="$1" + if [ ! -f "$json_file" ]; then + say_err "Unable to find \`$json_file\`" + return 1 + fi + + sdk_section=$(cat $json_file | awk '/"sdk"/,/}/') + if [ -z "$sdk_section" ]; then + say_err "Unable to parse the SDK node in \`$json_file\`" + return 1 + fi + + sdk_list=$(echo $sdk_section | awk -F"[{}]" '{print $2}') + sdk_list=${sdk_list//[\" ]/} + sdk_list=${sdk_list//,/$'\n'} + + local version_info="" + while read -r line; do + IFS=: + while read -r key value; do + if [[ "$key" == "version" ]]; then + version_info=$value + fi + done <<< "$line" + done <<< "$sdk_list" + if [ -z "$version_info" ]; then + say_err "Unable to find the SDK:version node in \`$json_file\`" + return 1 + fi + + unset IFS; + echo "$version_info" + return 0 +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +# version - $4 +# json_file - $5 +get_specific_version_from_version() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + local version="$(to_lowercase "$4")" + local json_file="$5" + + if [ -z "$json_file" ]; then + case "$version" in + latest) + local version_info + version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" false)" || return 1 + say_verbose "get_specific_version_from_version: version_info=$version_info" + echo "$version_info" | get_version_from_version_info + return 0 + ;; + coherent) + local version_info + version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" true)" || return 1 + say_verbose "get_specific_version_from_version: version_info=$version_info" + echo "$version_info" | get_version_from_version_info + return 0 + ;; + *) + echo "$version" + return 0 + ;; + esac + else + local version_info + version_info="$(parse_jsonfile_for_version "$json_file")" || return 1 + echo "$version_info" + return 0 + fi +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +# specific_version - $4 +construct_download_link() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + local specific_version="${4//[$'\t\r\n']}" + local specific_product_version="$(get_specific_product_version "$1" "$4")" + + local osname + osname="$(get_current_os_name)" || return 1 + + local download_link=null + if [[ "$runtime" == "dotnet" ]]; then + download_link="$azure_feed/Runtime/$specific_version/dotnet-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz" + elif [[ "$runtime" == "aspnetcore" ]]; then + download_link="$azure_feed/aspnetcore/Runtime/$specific_version/aspnetcore-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz" + elif [ -z "$runtime" ]; then + download_link="$azure_feed/Sdk/$specific_version/dotnet-sdk-$specific_product_version-$osname-$normalized_architecture.tar.gz" + else + return 1 + fi + + echo "$download_link" + return 0 +} + +# args: +# azure_feed - $1 +# specific_version - $2 +get_specific_product_version() { + # If we find a 'productVersion.txt' at the root of any folder, we'll use its contents + # to resolve the version of what's in the folder, superseding the specified version. + eval $invocation + + local azure_feed="$1" + local specific_version="${2//[$'\t\r\n']}" + local specific_product_version=$specific_version + + local download_link=null + if [[ "$runtime" == "dotnet" ]]; then + download_link="$azure_feed/Runtime/$specific_version/productVersion.txt${feed_credential}" + elif [[ "$runtime" == "aspnetcore" ]]; then + download_link="$azure_feed/aspnetcore/Runtime/$specific_version/productVersion.txt${feed_credential}" + elif [ -z "$runtime" ]; then + download_link="$azure_feed/Sdk/$specific_version/productVersion.txt${feed_credential}" + else + return 1 + fi + + if machine_has "curl" + then + specific_product_version=$(curl -s --fail "$download_link") + if [ $? -ne 0 ] + then + specific_product_version=$specific_version + fi + elif machine_has "wget" + then + specific_product_version=$(wget -qO- "$download_link") + if [ $? -ne 0 ] + then + specific_product_version=$specific_version + fi + fi + specific_product_version="${specific_product_version//[$'\t\r\n']}" + + echo "$specific_product_version" + return 0 +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +# specific_version - $4 +construct_legacy_download_link() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + local specific_version="${4//[$'\t\r\n']}" + + local distro_specific_osname + distro_specific_osname="$(get_legacy_os_name)" || return 1 + + local legacy_download_link=null + if [[ "$runtime" == "dotnet" ]]; then + legacy_download_link="$azure_feed/Runtime/$specific_version/dotnet-$distro_specific_osname-$normalized_architecture.$specific_version.tar.gz" + elif [ -z "$runtime" ]; then + legacy_download_link="$azure_feed/Sdk/$specific_version/dotnet-dev-$distro_specific_osname-$normalized_architecture.$specific_version.tar.gz" + else + return 1 + fi + + echo "$legacy_download_link" + return 0 +} + +get_user_install_path() { + eval $invocation + + if [ ! -z "${DOTNET_INSTALL_DIR:-}" ]; then + echo "$DOTNET_INSTALL_DIR" + else + echo "$HOME/.dotnet" + fi + return 0 +} + +# args: +# install_dir - $1 +resolve_installation_path() { + eval $invocation + + local install_dir=$1 + if [ "$install_dir" = "" ]; then + local user_install_path="$(get_user_install_path)" + say_verbose "resolve_installation_path: user_install_path=$user_install_path" + echo "$user_install_path" + return 0 + fi + + echo "$install_dir" + return 0 +} + +# args: +# relative_or_absolute_path - $1 +get_absolute_path() { + eval $invocation + + local relative_or_absolute_path=$1 + echo "$(cd "$(dirname "$1")" && pwd -P)/$(basename "$1")" + return 0 +} + +# args: +# input_files - stdin +# root_path - $1 +# out_path - $2 +# override - $3 +copy_files_or_dirs_from_list() { + eval $invocation + + local root_path="$(remove_trailing_slash "$1")" + local out_path="$(remove_trailing_slash "$2")" + local override="$3" + local osname="$(get_current_os_name)" + local override_switch=$( + if [ "$override" = false ]; then + if [ "$osname" = "linux-musl" ]; then + printf -- "-u"; + else + printf -- "-n"; + fi + fi) + + cat | uniq | while read -r file_path; do + local path="$(remove_beginning_slash "${file_path#$root_path}")" + local target="$out_path/$path" + if [ "$override" = true ] || (! ([ -d "$target" ] || [ -e "$target" ])); then + mkdir -p "$out_path/$(dirname "$path")" + if [ -d "$target" ]; then + rm -rf "$target" + fi + cp -R $override_switch "$root_path/$path" "$target" + fi + done +} + +# args: +# zip_path - $1 +# out_path - $2 +extract_dotnet_package() { + eval $invocation + + local zip_path="$1" + local out_path="$2" + + local temp_out_path="$(mktemp -d "$temporary_file_template")" + + local failed=false + tar -xzf "$zip_path" -C "$temp_out_path" > /dev/null || failed=true + + local folders_with_version_regex='^.*/[0-9]+\.[0-9]+[^/]+/' + find "$temp_out_path" -type f | grep -Eo "$folders_with_version_regex" | sort | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" false + find "$temp_out_path" -type f | grep -Ev "$folders_with_version_regex" | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" "$override_non_versioned_files" + + rm -rf "$temp_out_path" + + if [ "$failed" = true ]; then + say_err "Extraction failed" + return 1 + fi +} + +# args: +# remote_path - $1 +# [out_path] - $2 - stdout if not provided +download() { + eval $invocation + + local remote_path="$1" + local out_path="${2:-}" + + if [[ "$remote_path" != "http"* ]]; then + cp "$remote_path" "$out_path" + return $? + fi + + local failed=false + if machine_has "curl"; then + downloadcurl "$remote_path" "$out_path" || failed=true + elif machine_has "wget"; then + downloadwget "$remote_path" "$out_path" || failed=true + else + failed=true + fi + if [ "$failed" = true ]; then + say_verbose "Download failed: $remote_path" + return 1 + fi + return 0 +} + +downloadcurl() { + eval $invocation + local remote_path="$1" + local out_path="${2:-}" + + # Append feed_credential as late as possible before calling curl to avoid logging feed_credential + remote_path="${remote_path}${feed_credential}" + + local curl_options="--retry 20 --retry-delay 2 --connect-timeout 15 -sSL -f --create-dirs " + local failed=false + if [ -z "$out_path" ]; then + curl $curl_options "$remote_path" || failed=true + else + curl $curl_options -o "$out_path" "$remote_path" || failed=true + fi + if [ "$failed" = true ]; then + say_verbose "Curl download failed" + return 1 + fi + return 0 +} + +downloadwget() { + eval $invocation + local remote_path="$1" + local out_path="${2:-}" + + # Append feed_credential as late as possible before calling wget to avoid logging feed_credential + remote_path="${remote_path}${feed_credential}" + local wget_options="--tries 20 --waitretry 2 --connect-timeout 15 " + local failed=false + if [ -z "$out_path" ]; then + wget -q $wget_options -O - "$remote_path" || failed=true + else + wget $wget_options -O "$out_path" "$remote_path" || failed=true + fi + if [ "$failed" = true ]; then + say_verbose "Wget download failed" + return 1 + fi + return 0 +} + +calculate_vars() { + eval $invocation + valid_legacy_download_link=true + + normalized_architecture="$(get_normalized_architecture_from_architecture "$architecture")" + say_verbose "normalized_architecture=$normalized_architecture" + + specific_version="$(get_specific_version_from_version "$azure_feed" "$channel" "$normalized_architecture" "$version" "$json_file")" + specific_product_version="$(get_specific_product_version "$azure_feed" "$specific_version")" + say_verbose "specific_version=$specific_version" + if [ -z "$specific_version" ]; then + say_err "Could not resolve version information." + return 1 + fi + + download_link="$(construct_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version")" + say_verbose "Constructed primary named payload URL: $download_link" + + legacy_download_link="$(construct_legacy_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version")" || valid_legacy_download_link=false + + if [ "$valid_legacy_download_link" = true ]; then + say_verbose "Constructed legacy named payload URL: $legacy_download_link" + else + say_verbose "Cound not construct a legacy_download_link; omitting..." + fi + + install_root="$(resolve_installation_path "$install_dir")" + say_verbose "InstallRoot: $install_root" +} + +install_dotnet() { + eval $invocation + local download_failed=false + local asset_name='' + local asset_relative_path='' + + if [[ "$runtime" == "dotnet" ]]; then + asset_relative_path="shared/Microsoft.NETCore.App" + asset_name=".NET Core Runtime" + elif [[ "$runtime" == "aspnetcore" ]]; then + asset_relative_path="shared/Microsoft.AspNetCore.App" + asset_name="ASP.NET Core Runtime" + elif [ -z "$runtime" ]; then + asset_relative_path="sdk" + asset_name=".NET Core SDK" + else + say_err "Invalid value for \$runtime" + return 1 + fi + + # Check if the SDK version is already installed. + if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$specific_version"; then + say "$asset_name version $specific_version is already installed." + return 0 + fi + + mkdir -p "$install_root" + zip_path="$(mktemp "$temporary_file_template")" + say_verbose "Zip path: $zip_path" + + say "Downloading link: $download_link" + + # Failures are normal in the non-legacy case for ultimately legacy downloads. + # Do not output to stderr, since output to stderr is considered an error. + download "$download_link" "$zip_path" 2>&1 || download_failed=true + + # if the download fails, download the legacy_download_link + if [ "$download_failed" = true ]; then + say "Cannot download: $download_link" + + if [ "$valid_legacy_download_link" = true ]; then + download_failed=false + download_link="$legacy_download_link" + zip_path="$(mktemp "$temporary_file_template")" + say_verbose "Legacy zip path: $zip_path" + say "Downloading legacy link: $download_link" + download "$download_link" "$zip_path" 2>&1 || download_failed=true + + if [ "$download_failed" = true ]; then + say "Cannot download: $download_link" + fi + fi + fi + + if [ "$download_failed" = true ]; then + say_err "Could not find/download: \`$asset_name\` with version = $specific_version" + say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" + return 1 + fi + + say "Extracting zip from $download_link" + extract_dotnet_package "$zip_path" "$install_root" + + # Check if the SDK version is installed; if not, fail the installation. + # if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed. + if [[ $specific_version == *"rtm"* || $specific_version == *"servicing"* ]]; then + IFS='-' + read -ra verArr <<< "$specific_version" + release_version="${verArr[0]}" + unset IFS; + say_verbose "Checking installation: version = $release_version" + if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$release_version"; then + return 0 + fi + fi + + # Check if the standard SDK version is installed. + say_verbose "Checking installation: version = $specific_product_version" + if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$specific_product_version"; then + return 0 + fi + + say_err "\`$asset_name\` with version = $specific_product_version failed to install with an unknown error." + return 1 +} + +args=("$@") + +local_version_file_relative_path="/.version" +bin_folder_relative_path="" +temporary_file_template="${TMPDIR:-/tmp}/dotnet.XXXXXXXXX" + +channel="LTS" +version="Latest" +json_file="" +install_dir="" +architecture="" +dry_run=false +no_path=false +no_cdn=false +azure_feed="https://dotnetcli.azureedge.net/dotnet" +uncached_feed="https://dotnetcli.blob.core.windows.net/dotnet" +feed_credential="" +verbose=false +runtime="" +runtime_id="" +override_non_versioned_files=true +non_dynamic_parameters="" + +while [ $# -ne 0 ] +do + name="$1" + case "$name" in + -c|--channel|-[Cc]hannel) + shift + channel="$1" + ;; + -v|--version|-[Vv]ersion) + shift + version="$1" + ;; + -i|--install-dir|-[Ii]nstall[Dd]ir) + shift + install_dir="$1" + ;; + --arch|--architecture|-[Aa]rch|-[Aa]rchitecture) + shift + architecture="$1" + ;; + --shared-runtime|-[Ss]hared[Rr]untime) + say_warning "The --shared-runtime flag is obsolete and may be removed in a future version of this script. The recommended usage is to specify '--runtime dotnet'." + if [ -z "$runtime" ]; then + runtime="dotnet" + fi + ;; + --runtime|-[Rr]untime) + shift + runtime="$1" + if [[ "$runtime" != "dotnet" ]] && [[ "$runtime" != "aspnetcore" ]]; then + say_err "Unsupported value for --runtime: '$1'. Valid values are 'dotnet' and 'aspnetcore'." + if [[ "$runtime" == "windowsdesktop" ]]; then + say_err "WindowsDesktop archives are manufactured for Windows platforms only." + fi + exit 1 + fi + ;; + --dry-run|-[Dd]ry[Rr]un) + dry_run=true + ;; + --no-path|-[Nn]o[Pp]ath) + no_path=true + non_dynamic_parameters+=" $name" + ;; + --verbose|-[Vv]erbose) + verbose=true + non_dynamic_parameters+=" $name" + ;; + --no-cdn|-[Nn]o[Cc]dn) + no_cdn=true + non_dynamic_parameters+=" $name" + ;; + --azure-feed|-[Aa]zure[Ff]eed) + shift + azure_feed="$1" + non_dynamic_parameters+=" $name "\""$1"\""" + ;; + --uncached-feed|-[Uu]ncached[Ff]eed) + shift + uncached_feed="$1" + non_dynamic_parameters+=" $name "\""$1"\""" + ;; + --feed-credential|-[Ff]eed[Cc]redential) + shift + feed_credential="$1" + non_dynamic_parameters+=" $name "\""$1"\""" + ;; + --runtime-id|-[Rr]untime[Ii]d) + shift + runtime_id="$1" + non_dynamic_parameters+=" $name "\""$1"\""" + ;; + --jsonfile|-[Jj][Ss]on[Ff]ile) + shift + json_file="$1" + ;; + --skip-non-versioned-files|-[Ss]kip[Nn]on[Vv]ersioned[Ff]iles) + override_non_versioned_files=false + non_dynamic_parameters+=" $name" + ;; + -?|--?|-h|--help|-[Hh]elp) + script_name="$(basename "$0")" + echo ".NET Tools Installer" + echo "Usage: $script_name [-c|--channel ] [-v|--version ] [-p|--prefix ]" + echo " $script_name -h|-?|--help" + echo "" + echo "$script_name is a simple command line interface for obtaining dotnet cli." + echo "" + echo "Options:" + echo " -c,--channel Download from the channel specified, Defaults to \`$channel\`." + echo " -Channel" + echo " Possible values:" + echo " - Current - most current release" + echo " - LTS - most current supported release" + echo " - 2-part version in a format A.B - represents a specific release" + echo " examples: 2.0; 1.0" + echo " - Branch name" + echo " examples: release/2.0.0; Master" + echo " Note: The version parameter overrides the channel parameter." + echo " -v,--version Use specific VERSION, Defaults to \`$version\`." + echo " -Version" + echo " Possible values:" + echo " - latest - most latest build on specific channel" + echo " - coherent - most latest coherent build on specific channel" + echo " coherent applies only to SDK downloads" + echo " - 3-part version in a format A.B.C - represents specific version of build" + echo " examples: 2.0.0-preview2-006120; 1.1.0" + echo " -i,--install-dir Install under specified location (see Install Location below)" + echo " -InstallDir" + echo " --architecture Architecture of dotnet binaries to be installed, Defaults to \`$architecture\`." + echo " --arch,-Architecture,-Arch" + echo " Possible values: x64, arm, and arm64" + echo " --runtime Installs a shared runtime only, without the SDK." + echo " -Runtime" + echo " Possible values:" + echo " - dotnet - the Microsoft.NETCore.App shared runtime" + echo " - aspnetcore - the Microsoft.AspNetCore.App shared runtime" + echo " --dry-run,-DryRun Do not perform installation. Display download link." + echo " --no-path, -NoPath Do not set PATH for the current process." + echo " --verbose,-Verbose Display diagnostics information." + echo " --azure-feed,-AzureFeed Azure feed location. Defaults to $azure_feed, This parameter typically is not changed by the user." + echo " --uncached-feed,-UncachedFeed Uncached feed location. This parameter typically is not changed by the user." + echo " --feed-credential,-FeedCredential Azure feed shared access token. This parameter typically is not specified." + echo " --skip-non-versioned-files Skips non-versioned files if they already exist, such as the dotnet executable." + echo " -SkipNonVersionedFiles" + echo " --no-cdn,-NoCdn Disable downloading from the Azure CDN, and use the uncached feed directly." + echo " --jsonfile Determines the SDK version from a user specified global.json file." + echo " Note: global.json must have a value for 'SDK:Version'" + echo " --runtime-id Installs the .NET Tools for the given platform (use linux-x64 for portable linux)." + echo " -RuntimeId" + echo " -?,--?,-h,--help,-Help Shows this help message" + echo "" + echo "Obsolete parameters:" + echo " --shared-runtime The recommended alternative is '--runtime dotnet'." + echo " This parameter is obsolete and may be removed in a future version of this script." + echo " Installs just the shared runtime bits, not the entire SDK." + echo "" + echo "Install Location:" + echo " Location is chosen in following order:" + echo " - --install-dir option" + echo " - Environmental variable DOTNET_INSTALL_DIR" + echo " - $HOME/.dotnet" + exit 0 + ;; + *) + say_err "Unknown argument \`$name\`" + exit 1 + ;; + esac + + shift +done + +if [ "$no_cdn" = true ]; then + azure_feed="$uncached_feed" +fi + +say "Note that the intended use of this script is for Continuous Integration (CI) scenarios, where:" +say "- The SDK needs to be installed without user interaction and without admin rights." +say "- The SDK installation doesn't need to persist across multiple CI runs." +say "To set up a development environment or to run apps, use installers rather than this script. Visit https://dotnet.microsoft.com/download to get the installer.\n" + +check_min_reqs +calculate_vars +script_name=$(basename "$0") + +if [ "$dry_run" = true ]; then + say "Payload URLs:" + say "Primary named payload URL: $download_link" + if [ "$valid_legacy_download_link" = true ]; then + say "Legacy named payload URL: $legacy_download_link" + fi + repeatable_command="./$script_name --version "\""$specific_version"\"" --install-dir "\""$install_root"\"" --architecture "\""$normalized_architecture"\""" + if [[ "$runtime" == "dotnet" ]]; then + repeatable_command+=" --runtime "\""dotnet"\""" + elif [[ "$runtime" == "aspnetcore" ]]; then + repeatable_command+=" --runtime "\""aspnetcore"\""" + fi + repeatable_command+="$non_dynamic_parameters" + say "Repeatable invocation: $repeatable_command" + exit 0 +fi + +install_dotnet + +bin_path="$(get_absolute_path "$(combine_paths "$install_root" "$bin_folder_relative_path")")" +if [ "$no_path" = false ]; then + say "Adding to current process PATH: \`$bin_path\`. Note: This change will be visible only when sourcing script." + export PATH="$bin_path":"$PATH" +else + say "Binaries of dotnet can be found in $bin_path" +fi + +say "Note that the script does not resolve dependencies during installation." +say "To check the list of dependencies, go to https://docs.microsoft.com/dotnet/core/install, select your operating system and check the \"Dependencies\" section." +say "Installation finished successfully." + + diff --git a/.github/scripts/mac-action-runner-readme.txt b/.github/scripts/mac-action-runner-readme.txt index ce774da3f..1cf46a42e 100644 --- a/.github/scripts/mac-action-runner-readme.txt +++ b/.github/scripts/mac-action-runner-readme.txt @@ -21,3 +21,6 @@ JDK (we need a specific JDK): - fix the JAVA_HOME mess as follows: sudo rm -fr /Library/Internet\ Plug-Ins/JavaAppletPlugin.plugin sudo rm -fr /Library/PreferencePanes/JavaControlPanel.prefpane + +C++: + - brew install cmake \ No newline at end of file diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 2d696fd16..5bae5b723 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -15,7 +15,7 @@ jobs: # GROUP: [LEXER, PARSER, RECURSION] # TARGET: [swift, cpp, dotnet] GROUP: [LEXER] - TARGET: [cpp] + TARGET: [dotnet] steps: - uses: actions/checkout@v2 - name: Set up JDK 1.8 From daf633643003969bd921c6081baee6628d3363ff Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sun, 24 Jan 2021 16:50:50 +0800 Subject: [PATCH 21/42] add script --- .github/scripts/run-tests-dotnet.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100755 .github/scripts/run-tests-dotnet.sh diff --git a/.github/scripts/run-tests-dotnet.sh b/.github/scripts/run-tests-dotnet.sh new file mode 100755 index 000000000..de8c046e0 --- /dev/null +++ b/.github/scripts/run-tests-dotnet.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -euo pipefail + +# we need to build the runtime before test run, since we used "--no-dependencies" +# when we call dotnet cli for restore and build, in order to speed up + +dotnet build -c Release -f netstandard2.0 ../runtime/CSharp/Antlr4.csproj + +# call test + +if [ $GROUP == "LEXER" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dparallel=classes -DthreadCount=4 -Dtest=csharp.* test +elif [ $GROUP == "PARSER" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dparallel=classes -DthreadCount=4 -Dtest=csharp.* test +elif [ $GROUP == "RECURSION" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dparallel=classes -DthreadCount=4 -Dtest=csharp.* test +else + mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=csharp.* test +fi From 0caddd86342c3a8d98ae3bc506ee273997c34a13 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sun, 24 Jan 2021 17:29:12 +0800 Subject: [PATCH 22/42] add dotnet path --- .github/scripts/mac-action-runner-readme.txt | 6 +++++- .github/scripts/run-tests-dotnet.sh | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/scripts/mac-action-runner-readme.txt b/.github/scripts/mac-action-runner-readme.txt index 1cf46a42e..f6aa39d77 100644 --- a/.github/scripts/mac-action-runner-readme.txt +++ b/.github/scripts/mac-action-runner-readme.txt @@ -23,4 +23,8 @@ JDK (we need a specific JDK): sudo rm -fr /Library/PreferencePanes/JavaControlPanel.prefpane C++: - - brew install cmake \ No newline at end of file + - brew install cmake + +C#: + - .github/scripts/install-dotnet-on-osx.sh + (you need to repeat this step for each user account) \ No newline at end of file diff --git a/.github/scripts/run-tests-dotnet.sh b/.github/scripts/run-tests-dotnet.sh index de8c046e0..17f1e6d65 100755 --- a/.github/scripts/run-tests-dotnet.sh +++ b/.github/scripts/run-tests-dotnet.sh @@ -2,6 +2,8 @@ set -euo pipefail +export PATH=$PATH:~/.dotnet + # we need to build the runtime before test run, since we used "--no-dependencies" # when we call dotnet cli for restore and build, in order to speed up From 92f4c0707b98fac6c2dc462a0125bae835d12667 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sun, 24 Jan 2021 17:32:24 +0800 Subject: [PATCH 23/42] fix cd --- .github/scripts/run-tests-dotnet.sh | 3 ++- .github/scripts/run-tests-java.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/run-tests-dotnet.sh b/.github/scripts/run-tests-dotnet.sh index 17f1e6d65..ae4efae8f 100755 --- a/.github/scripts/run-tests-dotnet.sh +++ b/.github/scripts/run-tests-dotnet.sh @@ -9,7 +9,8 @@ export PATH=$PATH:~/.dotnet dotnet build -c Release -f netstandard2.0 ../runtime/CSharp/Antlr4.csproj -# call test +# run tests +cd runtime-testsuite/ if [ $GROUP == "LEXER" ]; then mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dparallel=classes -DthreadCount=4 -Dtest=csharp.* test diff --git a/.github/scripts/run-tests-java.sh b/.github/scripts/run-tests-java.sh index 678581bf1..f076fc3dc 100755 --- a/.github/scripts/run-tests-java.sh +++ b/.github/scripts/run-tests-java.sh @@ -11,6 +11,7 @@ fi # run java tests cd runtime-testsuite/ + if [ $GROUP == "LEXER" ]; then mvn -X -e -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest="swift.*" test elif [ $GROUP == "PARSER" ]; then From a5589d0dabe0dac61834dc92046bcf2f71a8e405 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sun, 24 Jan 2021 17:34:59 +0800 Subject: [PATCH 24/42] fix cd --- .github/scripts/run-tests-dotnet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/run-tests-dotnet.sh b/.github/scripts/run-tests-dotnet.sh index ae4efae8f..aa6f196a7 100755 --- a/.github/scripts/run-tests-dotnet.sh +++ b/.github/scripts/run-tests-dotnet.sh @@ -7,7 +7,7 @@ export PATH=$PATH:~/.dotnet # we need to build the runtime before test run, since we used "--no-dependencies" # when we call dotnet cli for restore and build, in order to speed up -dotnet build -c Release -f netstandard2.0 ../runtime/CSharp/Antlr4.csproj +dotnet build -c Release -f netstandard2.0 runtime/CSharp/Antlr4.csproj # run tests cd runtime-testsuite/ From 0fa457a36ea818cab2ce7736fb330fca3adcd6b0 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sun, 24 Jan 2021 17:45:41 +0800 Subject: [PATCH 25/42] avoid concurrency --- .github/scripts/run-tests-dotnet.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/scripts/run-tests-dotnet.sh b/.github/scripts/run-tests-dotnet.sh index aa6f196a7..39055c3ac 100755 --- a/.github/scripts/run-tests-dotnet.sh +++ b/.github/scripts/run-tests-dotnet.sh @@ -13,11 +13,11 @@ dotnet build -c Release -f netstandard2.0 runtime/CSharp/Antlr4.csproj cd runtime-testsuite/ if [ $GROUP == "LEXER" ]; then - mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dparallel=classes -DthreadCount=4 -Dtest=csharp.* test + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=csharp.* test elif [ $GROUP == "PARSER" ]; then - mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dparallel=classes -DthreadCount=4 -Dtest=csharp.* test + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=csharp.* test elif [ $GROUP == "RECURSION" ]; then - mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dparallel=classes -DthreadCount=4 -Dtest=csharp.* test + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=csharp.* test else - mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=csharp.* test + mvn -q -Dtest=csharp.* test fi From b3542834a5309c154976478a208ea2484b3923cd Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sun, 24 Jan 2021 17:55:49 +0800 Subject: [PATCH 26/42] add Github action badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 44cbf62a8..cd9d7c2ee 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ANTLR v4 -[![Build Travis-CI Status](https://travis-ci.org/antlr/antlr4.svg?branch=master)](https://travis-ci.org/antlr/antlr4) [![Build AppVeyor Status](https://ci.appveyor.com/api/projects/status/5acpbx1pg7bhgh8v/branch/master?svg=true)](https://ci.appveyor.com/project/parrt/antlr4) [![Java 7+](https://img.shields.io/badge/java-7+-4c7e9f.svg)](http://java.oracle.com) [![License](https://img.shields.io/badge/license-BSD-blue.svg)](https://raw.githubusercontent.com/antlr/antlr4/master/LICENSE.txt) +[![Github Build Status (MacOSX)](https://github.com/antlr/antlr4/workflows/MacOSX/badge.svg)](https://github.com/antlr/antlr4/actions) [![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/5acpbx1pg7bhgh8v/branch/master?svg=true)](https://ci.appveyor.com/project/parrt/antlr4) [![Java 7+](https://img.shields.io/badge/java-7+-4c7e9f.svg)](http://java.oracle.com) [![License](https://img.shields.io/badge/license-BSD-blue.svg)](https://raw.githubusercontent.com/antlr/antlr4/master/LICENSE.txt) **ANTLR** (ANother Tool for Language Recognition) is a powerful parser generator for reading, processing, executing, or translating structured text or binary files. It's widely used to build languages, tools, and frameworks. From a grammar, ANTLR generates a parser that can build parse trees and also generates a listener interface (or visitor) that makes it easy to respond to the recognition of phrases of interest. From 6a7c3ad256e6f7486a4857c678b311d1d3f8a375 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sun, 24 Jan 2021 17:57:40 +0800 Subject: [PATCH 27/42] more feedback --- .../test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java | 1 + .../test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java | 1 + 2 files changed, 2 insertions(+) diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java index 03895f55a..2d9579a82 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java @@ -114,6 +114,7 @@ public class BaseCppTest implements RuntimeTestSupport { @Override public void beforeTest(RuntimeTestDescriptor descriptor) { + System.out.println(descriptor.getTestName()); } @Override diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java index 5fbd97ecf..a14067ee4 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java @@ -142,6 +142,7 @@ public class BaseCSharpTest implements RuntimeTestSupport { @Override public void beforeTest(RuntimeTestDescriptor descriptor) { + System.out.println(descriptor.getTestName()); } @Override From 4830925ab0021ba362920bdd0eacfb6b76db74f1 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sun, 24 Jan 2021 17:59:35 +0800 Subject: [PATCH 28/42] reinstate all builds --- .github/workflows/java-all.yml | 28 ---------------------------- .github/workflows/macosx.yml | 6 ++---- 2 files changed, 2 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/java-all.yml diff --git a/.github/workflows/java-all.yml b/.github/workflows/java-all.yml deleted file mode 100644 index 34cdb8b46..000000000 --- a/.github/workflows/java-all.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Java/MacOSX - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - runs-on: [self-hosted, macOS, x64] - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Set up Maven - uses: stCarolas/setup-maven@v4 - with: - maven-version: 3.5.4 - - name: Build tool with Maven - run: mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V - - name: Test with Maven - run: .github/scripts/run-tests-java.sh - env: - TARGET: swift - GROUP: ALL diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 5bae5b723..cbce1a2ee 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -12,10 +12,8 @@ jobs: strategy: fail-fast: false matrix: - # GROUP: [LEXER, PARSER, RECURSION] - # TARGET: [swift, cpp, dotnet] - GROUP: [LEXER] - TARGET: [dotnet] + GROUP: [LEXER, PARSER, RECURSION] + TARGET: [swift, cpp, dotnet] steps: - uses: actions/checkout@v2 - name: Set up JDK 1.8 From 05f422268a611949e1d809f335fb36acbf67ddc1 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sun, 24 Jan 2021 18:08:34 +0800 Subject: [PATCH 29/42] drop unused script --- .github/scripts/run-tests-java.sh | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100755 .github/scripts/run-tests-java.sh diff --git a/.github/scripts/run-tests-java.sh b/.github/scripts/run-tests-java.sh deleted file mode 100755 index f076fc3dc..000000000 --- a/.github/scripts/run-tests-java.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -if [ -z "${JAVA_HOME-}" ] -then - export JAVA_HOME="$(java -XshowSettings:properties -version 2>&1 | - grep 'java\.home' | awk '{ print $3 }')" - echo "export JAVA_HOME=$JAVA_HOME" -fi - -# run java tests -cd runtime-testsuite/ - -if [ $GROUP == "LEXER" ]; then - mvn -X -e -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest="swift.*" test -elif [ $GROUP == "PARSER" ]; then - mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest="swift.*" test -elif [ $GROUP == "RECURSION" ]; then - mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest="swift.*" test -else - mvn -q -Dtest="java.*" test -fi -rc=$? -cat target/surefire-reports/*.dumpstream || true -exit $rc From 84d8348dc1633e50bbb78e681622627bdf52e845 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sun, 24 Jan 2021 19:04:42 +0800 Subject: [PATCH 30/42] fix disk space issue on MacOSX CI --- .../org/antlr/v4/test/runtime/cpp/BaseCppTest.java | 14 ++++++++++++++ .../v4/test/runtime/csharp/BaseCSharpTest.java | 11 +++++++++++ .../antlr/v4/test/runtime/dart/BaseDartTest.java | 14 ++++++++++++++ .../org/antlr/v4/test/runtime/go/BaseGoTest.java | 14 ++++++++++++++ .../antlr/v4/test/runtime/java/BaseJavaTest.java | 14 ++++++++++++++ .../v4/test/runtime/javascript/BaseNodeTest.java | 14 ++++++++++++++ .../org/antlr/v4/test/runtime/php/BasePHPTest.java | 14 ++++++++++++++ .../antlr/v4/test/runtime/swift/BaseSwiftTest.java | 14 ++++++++++++++ 8 files changed, 109 insertions(+) diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java index 2d9579a82..d8c70ca23 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java @@ -44,6 +44,9 @@ import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.GrammarSemanticsMessage; import org.antlr.v4.tool.LexerGrammar; import org.antlr.v4.tool.Rule; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; import org.stringtemplate.v4.ST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.STGroupString; @@ -89,6 +92,17 @@ public class BaseCppTest implements RuntimeTestSupport { /** Errors found while running antlr */ protected StringBuilder antlrToolErrors; + @org.junit.Rule + public final TestRule testWatcher = new TestWatcher() { + + @Override + protected void succeeded(Description description) { + // remove tmpdir if no error. + eraseTempDir(); + } + + }; + private String getPropertyPrefix() { return "antlr-" + getLanguage().toLowerCase(); } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java index a14067ee4..923b34298 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java @@ -121,6 +121,17 @@ public class BaseCSharpTest implements RuntimeTestSupport { */ protected StringBuilder antlrToolErrors; + @org.junit.Rule + public final TestRule testWatcher = new TestWatcher() { + + @Override + protected void succeeded(Description description) { + // remove tmpdir if no error. + eraseTempDir(); + } + + }; + @Override public void testSetUp() throws Exception { if (CREATE_PER_TEST_DIRECTORIES) { diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java index 6ce699ae9..fd4da1cfb 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java @@ -26,6 +26,9 @@ import org.antlr.v4.test.runtime.*; import org.antlr.v4.test.runtime.descriptors.LexerExecDescriptors; import org.antlr.v4.test.runtime.descriptors.PerformanceDescriptors; import org.antlr.v4.tool.*; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; import org.stringtemplate.v4.ST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.STGroupString; @@ -126,6 +129,17 @@ public class BaseDartTest implements RuntimeTestSupport { private static String cacheDartPackages; + @org.junit.Rule + public final TestRule testWatcher = new TestWatcher() { + + @Override + protected void succeeded(Description description) { + // remove tmpdir if no error. + eraseTempDir(); + } + + }; + private String getPropertyPrefix() { return "antlr-dart"; } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java index f5596db3e..0226d5023 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java @@ -41,6 +41,9 @@ import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.GrammarSemanticsMessage; import org.antlr.v4.tool.LexerGrammar; import org.antlr.v4.tool.Rule; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; import org.stringtemplate.v4.ST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.STGroupString; @@ -86,6 +89,17 @@ public class BaseGoTest implements RuntimeTestSupport { /** Errors found while running antlr */ protected StringBuilder antlrToolErrors; + @org.junit.Rule + public final TestRule testWatcher = new TestWatcher() { + + @Override + protected void succeeded(Description description) { + // remove tmpdir if no error. + eraseTempDir(); + } + + }; + /** * Copies all files from go runtime to a temporary folder that is inside a valid GOPATH project structure. */ diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java index 765ab9e9f..d142d1df3 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java @@ -42,6 +42,9 @@ import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.GrammarSemanticsMessage; import org.antlr.v4.tool.LexerGrammar; import org.antlr.v4.tool.Rule; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; import org.stringtemplate.v4.ST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.STGroupString; @@ -170,6 +173,17 @@ public class BaseJavaTest implements RuntimeTestSupport { */ protected StringBuilder antlrToolErrors; + @org.junit.Rule + public final TestRule testWatcher = new TestWatcher() { + + @Override + protected void succeeded(Description description) { + // remove tmpdir if no error. + eraseTempDir(); + } + + }; + @Override public void testSetUp() throws Exception { // STGroup.verbose = true; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java index bfe98a21c..65fb05aba 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java @@ -15,6 +15,9 @@ import org.antlr.v4.semantics.SemanticPipeline; import org.antlr.v4.test.runtime.*; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.LexerGrammar; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; import org.stringtemplate.v4.ST; import java.io.File; @@ -46,6 +49,17 @@ public class BaseNodeTest implements RuntimeTestSupport { /** Errors found while running antlr */ protected StringBuilder antlrToolErrors; + @org.junit.Rule + public final TestRule testWatcher = new TestWatcher() { + + @Override + protected void succeeded(Description description) { + // remove tmpdir if no error. + eraseTempDir(); + } + + }; + @Override public void testSetUp() throws Exception { // new output dir for each test diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java index f164ffec8..89d9d081f 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java @@ -30,6 +30,9 @@ import org.antlr.v4.test.runtime.RuntimeTestSupport; import org.antlr.v4.test.runtime.StreamVacuum; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.LexerGrammar; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; import org.stringtemplate.v4.ST; import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString; @@ -53,6 +56,17 @@ public class BasePHPTest implements RuntimeTestSupport { */ protected StringBuilder antlrToolErrors; + @org.junit.Rule + public final TestRule testWatcher = new TestWatcher() { + + @Override + protected void succeeded(Description description) { + // remove tmpdir if no error. + eraseTempDir(); + } + + }; + private String getPropertyPrefix() { return "antlr-php"; } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java index 7f2158975..2efd08a64 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java @@ -11,6 +11,9 @@ import org.antlr.v4.test.runtime.ErrorQueue; import org.antlr.v4.test.runtime.RuntimeTestDescriptor; import org.antlr.v4.test.runtime.RuntimeTestSupport; import org.antlr.v4.test.runtime.StreamVacuum; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; import org.stringtemplate.v4.ST; import java.io.BufferedReader; @@ -96,6 +99,17 @@ public class BaseSwiftTest implements RuntimeTestSupport { */ private final Set sourceFiles = new HashSet<>(); + @org.junit.Rule + public final TestRule testWatcher = new TestWatcher() { + + @Override + protected void succeeded(Description description) { + // remove tmpdir if no error. + eraseTempDir(); + } + + }; + @Override public void testSetUp() throws Exception { // new output dir for each test From ce479dc228cf474071e5cfdf933c1db7600c3e13 Mon Sep 17 00:00:00 2001 From: kaby76 Date: Sun, 24 Jan 2021 10:46:39 -0500 Subject: [PATCH 31/42] Fix for https://github.com/antlr/antlr4/issues/3049 --- runtime/CSharp/Antlr4.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/CSharp/Antlr4.csproj b/runtime/CSharp/Antlr4.csproj index 431c87330..0a7b1e006 100644 --- a/runtime/CSharp/Antlr4.csproj +++ b/runtime/CSharp/Antlr4.csproj @@ -32,6 +32,7 @@ false false Antlr4.Runtime + true true From cf16ec791ea32c1ccc7e2bace84c9d79f1788f84 Mon Sep 17 00:00:00 2001 From: Ivan Kochurkin Date: Mon, 25 Jan 2021 21:14:18 +0300 Subject: [PATCH 32/42] Fix missed version, https://github.com/antlr/antlr4/pull/2987#pullrequestreview-575417430 --- runtime/CSharp/Properties/AssemblyInfo.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtime/CSharp/Properties/AssemblyInfo.cs b/runtime/CSharp/Properties/AssemblyInfo.cs index d846533bb..4e719d09c 100644 --- a/runtime/CSharp/Properties/AssemblyInfo.cs +++ b/runtime/CSharp/Properties/AssemblyInfo.cs @@ -3,5 +3,7 @@ * can be found in the LICENSE.txt file in the project root. */ using System; +using System.Reflection; [assembly: CLSCompliant(true)] +[assembly: AssemblyVersion("4.9.1")] \ No newline at end of file From bca2536f3fb689e6e06b8351c71571f8e837f71e Mon Sep 17 00:00:00 2001 From: ericvergnaud Date: Tue, 26 Jan 2021 12:56:16 +0800 Subject: [PATCH 33/42] Circleci project setup (#3055) * Add .circleci/config.yml * setup config * use jdk8 image * Updated config.yml * Updated config.yml * Updated config.yml * more targets * Updated config.yml * update config * update config * update config * update config * update config * Updated config.yml * Updated config.yml * Updated config.yml * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update runtime version * update config * update config * python3 config * update config * align Python2 and Python3 test hierarchy * update runtime version * update config * update config * update config * update config * explore dart issue * update dart installer * update config * update dart installer * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * use sudo on circle ci * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * Fix NPE * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * update config * more consistent badges * do not parallelize tests on circle-ci --- .circleci/config.yml | 66 +++++++ .circleci/scripts/install-linux-cpp.sh | 35 ++++ .circleci/scripts/install-linux-dart.sh | 14 ++ .circleci/scripts/install-linux-dotnet.sh | 19 ++ .circleci/scripts/install-linux-go.sh | 9 + .circleci/scripts/install-linux-javascript.sh | 17 ++ .circleci/scripts/install-linux-libcurl3.sh | 24 +++ .circleci/scripts/install-linux-php.sh | 11 ++ .circleci/scripts/install-linux-python2.sh | 8 + .circleci/scripts/install-linux-python3.sh | 8 + .circleci/scripts/install-linux-swift.sh | 36 ++++ .circleci/scripts/run-tests-cpp.sh | 17 ++ .circleci/scripts/run-tests-dart.sh | 11 ++ .circleci/scripts/run-tests-dotnet.sh | 16 ++ .circleci/scripts/run-tests-go.sh | 10 + .circleci/scripts/run-tests-javascript.sh | 8 + .circleci/scripts/run-tests-php.sh | 9 + .circleci/scripts/run-tests-python2.sh | 24 +++ .circleci/scripts/run-tests-python3.sh | 24 +++ .circleci/scripts/run-tests-swift.sh | 27 +++ .travis.yml | 186 +----------------- README.md | 7 +- .../antlr/v4/test/runtime/TestContext.java | 5 + .../v4/test/runtime/cpp/BaseCppTest.java | 25 ++- .../test/runtime/csharp/BaseCSharpTest.java | 6 +- .../v4/test/runtime/dart/BaseDartTest.java | 56 +++++- .../antlr/v4/test/runtime/go/BaseGoTest.java | 18 +- .../v4/test/runtime/java/BaseJavaTest.java | 21 +- .../test/runtime/javascript/BaseNodeTest.java | 25 ++- .../v4/test/runtime/php/BasePHPTest.java | 19 +- .../test/runtime/python/BasePythonTest.java | 18 +- runtime/Python2/tests/mocks/TestLexer.py | 4 +- .../Python3/{test => tests}/TestFileStream.py | 0 .../{test => tests}/TestInputStream.py | 0 .../{test => tests}/TestIntervalSet.py | 0 .../Python3/{test => tests}/TestRecognizer.py | 0 .../TestTokenStreamRewriter.py | 0 runtime/Python3/{test => tests}/__init__.py | 0 runtime/Python3/{test => tests}/c.c | 0 runtime/Python3/{test => tests}/ctest.py | 0 runtime/Python3/{test => tests}/expr/Expr.g4 | 0 .../Python3/{test => tests}/expr/ExprLexer.py | 0 .../{test => tests}/expr/ExprParser.py | 0 .../{test => tests}/mocks/TestLexer.py | 2 +- .../Python3/{test => tests}/mocks/__init__.py | 0 .../{test => tests}/parser/__init__.py | 0 .../Python3/{test => tests}/parser/clexer.py | 0 .../Python3/{test => tests}/parser/cparser.py | 0 runtime/Python3/{test => tests}/run.py | 0 runtime/Python3/{test => tests}/xpathtest.py | 0 50 files changed, 547 insertions(+), 238 deletions(-) create mode 100644 .circleci/config.yml create mode 100755 .circleci/scripts/install-linux-cpp.sh create mode 100755 .circleci/scripts/install-linux-dart.sh create mode 100755 .circleci/scripts/install-linux-dotnet.sh create mode 100755 .circleci/scripts/install-linux-go.sh create mode 100755 .circleci/scripts/install-linux-javascript.sh create mode 100755 .circleci/scripts/install-linux-libcurl3.sh create mode 100755 .circleci/scripts/install-linux-php.sh create mode 100755 .circleci/scripts/install-linux-python2.sh create mode 100755 .circleci/scripts/install-linux-python3.sh create mode 100755 .circleci/scripts/install-linux-swift.sh create mode 100755 .circleci/scripts/run-tests-cpp.sh create mode 100755 .circleci/scripts/run-tests-dart.sh create mode 100755 .circleci/scripts/run-tests-dotnet.sh create mode 100755 .circleci/scripts/run-tests-go.sh create mode 100755 .circleci/scripts/run-tests-javascript.sh create mode 100755 .circleci/scripts/run-tests-php.sh create mode 100755 .circleci/scripts/run-tests-python2.sh create mode 100755 .circleci/scripts/run-tests-python3.sh create mode 100755 .circleci/scripts/run-tests-swift.sh rename runtime/Python3/{test => tests}/TestFileStream.py (100%) rename runtime/Python3/{test => tests}/TestInputStream.py (100%) rename runtime/Python3/{test => tests}/TestIntervalSet.py (100%) rename runtime/Python3/{test => tests}/TestRecognizer.py (100%) rename runtime/Python3/{test => tests}/TestTokenStreamRewriter.py (100%) rename runtime/Python3/{test => tests}/__init__.py (100%) rename runtime/Python3/{test => tests}/c.c (100%) rename runtime/Python3/{test => tests}/ctest.py (100%) rename runtime/Python3/{test => tests}/expr/Expr.g4 (100%) rename runtime/Python3/{test => tests}/expr/ExprLexer.py (100%) rename runtime/Python3/{test => tests}/expr/ExprParser.py (100%) rename runtime/Python3/{test => tests}/mocks/TestLexer.py (99%) rename runtime/Python3/{test => tests}/mocks/__init__.py (100%) rename runtime/Python3/{test => tests}/parser/__init__.py (100%) rename runtime/Python3/{test => tests}/parser/clexer.py (100%) rename runtime/Python3/{test => tests}/parser/cparser.py (100%) rename runtime/Python3/{test => tests}/run.py (100%) rename runtime/Python3/{test => tests}/xpathtest.py (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..0fdffd99b --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,66 @@ +version: 2.1 + +jobs: + test_tool_and_runtime_java: + docker: + - image: cimg/openjdk:8.0 + steps: + - checkout + - run: + name: build tool + command: mvn -B -V -DskipTests=true -Dmaven.javadoc.skip=true install + - run: + name: test runtime + command: | + cd runtime-testsuite + mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=java.* test + cd .. + - run: + name: test tool + command: | + cd tool-testsuite + mvn -q -Dparallel=methods -DthreadCount=4 test + cd .. + test_runtime: + parameters: + test-group: + description: The section + type: string + default: ALL + target: + description: The target + type: string + default: java + docker: + - image: cimg/openjdk:8.0 + environment: + TARGET: << parameters.target >> + GROUP: << parameters.test-group >> + steps: + - checkout + - run: + name: Install << parameters.target >> pre-requisites + command: | + f=".circleci/scripts/install-linux-<< parameters.target >>.sh"; ! [ -x "$f" ] || "$f" + - run: + name: Build ANTLR4 tool + command: mvn -B -V -DskipTests=true -Dmaven.javadoc.skip=true install + - run: + name: Test << parameters.target >> runtime + command: | + .circleci/scripts/run-tests-<< parameters.target >>.sh + +workflows: + build: + jobs: + - test_tool_and_runtime_java + - test_runtime: + matrix: + parameters: + target: [ dart, go, python2, python3, javascript ] + - test_runtime: + matrix: + parameters: +# target: [ cpp, dotnet, swift ] + target: [ cpp, dotnet ] + test-group: [ LEXER, PARSER, RECURSION ] diff --git a/.circleci/scripts/install-linux-cpp.sh b/.circleci/scripts/install-linux-cpp.sh new file mode 100755 index 000000000..8e3b003a0 --- /dev/null +++ b/.circleci/scripts/install-linux-cpp.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -euo pipefail + +echo "installing cpp SDK..." + +sudo apt-get update -y +sudo apt-get install -y clang +sudo apt-get install -y cmake +sudo apt-get install -y pkg-config +sudo apt-get install -y uuid-dev + +echo "done installing cpp SDK" + +clang++ --version +cmake --version + +echo "building cpp runtime..." + +pushd "runtime/Cpp/" + echo $PWD + rc=0 + if [ $rc == 0 ]; then + cmake . -DCMAKE_BUILD_TYPE=release + rc=$? + fi + if [ $rc == 0 ]; then + make -j 8 + rc=$? + fi +popd + + +echo "done building cpp runtime" + diff --git a/.circleci/scripts/install-linux-dart.sh b/.circleci/scripts/install-linux-dart.sh new file mode 100755 index 000000000..f15e29f5b --- /dev/null +++ b/.circleci/scripts/install-linux-dart.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euo pipefail + +echo "installing dart SDK..." +sudo apt-get update +sudo apt-get install apt-transport-https +sudo sh -c 'wget -qO- https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -' +sudo sh -c 'wget -qO- https://storage.googleapis.com/download.dartlang.org/linux/debian/dart_stable.list > /etc/apt/sources.list.d/dart_stable.list' +sudo apt-get update +sudo apt-get install dart=2.8.4-1 +export PATH="$PATH:/usr/lib/dart/bin" +echo "done installing dart SDK" +sudo apt-get install -f diff --git a/.circleci/scripts/install-linux-dotnet.sh b/.circleci/scripts/install-linux-dotnet.sh new file mode 100755 index 000000000..035ffdaf6 --- /dev/null +++ b/.circleci/scripts/install-linux-dotnet.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -euo pipefail + +echo "installing .Net SDK..." +wget https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb +sudo dpkg -i packages-microsoft-prod.deb +sudo apt-get update; \ + sudo apt-get install -y apt-transport-https && \ + sudo apt-get update && \ + sudo apt-get install -y dotnet-sdk-3.1 +export PATH=$PATH:~/.dotnet +echo "done installing .Net SDK" + +# we need to build the runtime before test run, since we used "--no-dependencies" +# when we call dotnet cli for restore and build, in order to speed up +echo "building runtime..." +dotnet build -c Release -f netstandard2.0 runtime/CSharp/Antlr4.csproj +echo "done building runtime" diff --git a/.circleci/scripts/install-linux-go.sh b/.circleci/scripts/install-linux-go.sh new file mode 100755 index 000000000..363e09095 --- /dev/null +++ b/.circleci/scripts/install-linux-go.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail + +echo "installing go SDK..." +sudo apt update +sudo apt install golang-go +go version +echo "done installing go SDK" \ No newline at end of file diff --git a/.circleci/scripts/install-linux-javascript.sh b/.circleci/scripts/install-linux-javascript.sh new file mode 100755 index 000000000..1ecc3a437 --- /dev/null +++ b/.circleci/scripts/install-linux-javascript.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -euo pipefail + +# use v14 and check +echo "installing nodejs..." +curl -sL https://deb.nodesource.com/setup_14.x | sudo -E bash - +sudo apt-get install -y nodejs +echo node version: $(node --version) +echo "done installing nodejs..." + +echo "packaging javascript runtime..." +pushd runtime/JavaScript + sudo npm install + sudo npm link +popd +echo "done packaging javascript runtime" diff --git a/.circleci/scripts/install-linux-libcurl3.sh b/.circleci/scripts/install-linux-libcurl3.sh new file mode 100755 index 000000000..6fe59bafa --- /dev/null +++ b/.circleci/scripts/install-linux-libcurl3.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +echo "before patching" +ls -all /lib/x86_64-linux-gnu/ | grep libcurl + +# This would fix missing CURL_OPENSSL_3 +# use a dedicated temp dir in the user space +mkdir ~/libcurl3 +cd ~/libcurl3 +# fetch latest libcurl3 +wget http://archive.ubuntu.com/ubuntu/pool/main/c/curl/libcurl3_7.47.0-1ubuntu2_amd64.deb +# extract data.tar.xz +ar x libcurl3* data.tar.xz +# extract all from data.tar.xz +tar xf data.tar.xz +# copy libcurl.so.3 where required +sudo cp -L ~/libcurl3/usr/lib/x86_64-linux-gnu/libcurl.so.4.4.0 /lib/x86_64-linux-gnu/libcurl.so.4.4.0 +sudo ln -sf libcurl.so.4.4.0 /lib/x86_64-linux-gnu/libcurl.so.4 +cd .. +# drop dedicated temp dir +sudo rm -rf ~/libcurl3 + +echo "after patching" +ls -all /lib/x86_64-linux-gnu/ | grep libcurl diff --git a/.circleci/scripts/install-linux-php.sh b/.circleci/scripts/install-linux-php.sh new file mode 100755 index 000000000..09028e5b8 --- /dev/null +++ b/.circleci/scripts/install-linux-php.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -euo pipefail + +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +sudo apt-get update -qq + +php -v + +git clone https://github.com/antlr/antlr-php-runtime.git +mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V \ No newline at end of file diff --git a/.circleci/scripts/install-linux-python2.sh b/.circleci/scripts/install-linux-python2.sh new file mode 100755 index 000000000..5549c1b9c --- /dev/null +++ b/.circleci/scripts/install-linux-python2.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -euo pipefail + +echo "installing python 2..." +sudo apt-get update -y +sudo apt-get install python2 +echo "done installing python 2" diff --git a/.circleci/scripts/install-linux-python3.sh b/.circleci/scripts/install-linux-python3.sh new file mode 100755 index 000000000..4c90f519c --- /dev/null +++ b/.circleci/scripts/install-linux-python3.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -euo pipefail + +echo "installing python 3..." +sudo apt-get update -y +sudo apt-get install python3 +echo "done installing python 3" diff --git a/.circleci/scripts/install-linux-swift.sh b/.circleci/scripts/install-linux-swift.sh new file mode 100755 index 000000000..739017565 --- /dev/null +++ b/.circleci/scripts/install-linux-swift.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -euo pipefail + +echo "installing swift SDK..." + +.circleci/scripts/install-linux-libcurl3.sh + +# see https://tecadmin.net/install-swift-ubuntu-1604-xenial/ +sudo apt-get update -y +sudo apt-get install clang libicu-dev +sudo apt-get install libpython2.7 libpython2.7-dev + +export SWIFT_VERSION=swift-5.3.2 +echo "installing gpg key..." +wget -q -O - https://swift.org/keys/all-keys.asc | sudo gpg --import - +echo "downloading SDK gpg key..." +SWIFT_SDK=https://swift.org/builds/$SWIFT_VERSION-release/ubuntu1604/$SWIFT_VERSION-RELEASE/$SWIFT_VERSION-RELEASE-ubuntu16.04.tar.gz +echo $SWIFT_SDK +wget -q $SWIFT_SDK +sudo tar xzf $SWIFT_VERSION-RELEASE-ubuntu16.04.tar.gz +mv $SWIFT_VERSION-RELEASE-ubuntu16.04 $PWD/swift + +export SWIFT_HOME=$PWD/swift/$SWIFT_VERSION-RELEASE-ubuntu16.04/usr/bin/ +export PATH=$PWD/swift/usr/bin:$PATH + +# This would fix a know linker issue mentioned in: # https://bugs.swift.org/browse/SR-2299 +sudo ln -sf ld.gold /usr/bin/ld +# This would fix missing libtinfo.so.5 +sudo apt install libncurses5 + +echo "done installing swift SDK..." + +# check swift +swift --version +swift build --version diff --git a/.circleci/scripts/run-tests-cpp.sh b/.circleci/scripts/run-tests-cpp.sh new file mode 100755 index 000000000..0913a82d8 --- /dev/null +++ b/.circleci/scripts/run-tests-cpp.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -euo pipefail + +pushd runtime-testsuite + echo "running maven tests..." + if [ $GROUP == "LEXER" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=cpp.* test + elif [ $GROUP == "PARSER" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=cpp.* test + elif [ $GROUP == "RECURSION" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=cpp.* test + else + mvn -q -Dtest=cpp.* test + fi +popd + diff --git a/.circleci/scripts/run-tests-dart.sh b/.circleci/scripts/run-tests-dart.sh new file mode 100755 index 000000000..89ccc37af --- /dev/null +++ b/.circleci/scripts/run-tests-dart.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -euo pipefail + +dart --version + +pushd runtime-testsuite + echo "running maven tests..." +# mvn -q -Dparallel=classes -DthreadCount=4 -Dtest=dart.* test + mvn -q -Dtest=dart.* test +popd diff --git a/.circleci/scripts/run-tests-dotnet.sh b/.circleci/scripts/run-tests-dotnet.sh new file mode 100755 index 000000000..5af345c42 --- /dev/null +++ b/.circleci/scripts/run-tests-dotnet.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euo pipefail + +pushd runtime-testsuite/ + echo "running maven tests..." + if [ $GROUP == "LEXER" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=csharp.* test + elif [ $GROUP == "PARSER" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=csharp.* test + elif [ $GROUP == "RECURSION" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=csharp.* test + else + mvn -q -Dtest=csharp.* test + fi +popd diff --git a/.circleci/scripts/run-tests-go.sh b/.circleci/scripts/run-tests-go.sh new file mode 100755 index 000000000..3c29def4d --- /dev/null +++ b/.circleci/scripts/run-tests-go.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -euo pipefail + +go version + +pushd runtime-testsuite + echo "running maven tests..." + mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=go.* test +popd \ No newline at end of file diff --git a/.circleci/scripts/run-tests-javascript.sh b/.circleci/scripts/run-tests-javascript.sh new file mode 100755 index 000000000..b2390d468 --- /dev/null +++ b/.circleci/scripts/run-tests-javascript.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -euo pipefail + +pushd runtime-testsuite + echo "running maven tests..." + mvn -q -Dtest=javascript.* test +popd \ No newline at end of file diff --git a/.circleci/scripts/run-tests-php.sh b/.circleci/scripts/run-tests-php.sh new file mode 100755 index 000000000..853efbd86 --- /dev/null +++ b/.circleci/scripts/run-tests-php.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail + +php_path=$(which php) + +composer install -d ../runtime/PHP + +mvn -q -DPHP_PATH="${php_path}" -Dparallel=methods -DthreadCount=4 -Dtest=php.* test diff --git a/.circleci/scripts/run-tests-python2.sh b/.circleci/scripts/run-tests-python2.sh new file mode 100755 index 000000000..9b65b302e --- /dev/null +++ b/.circleci/scripts/run-tests-python2.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -euo pipefail + +python2 --version + +pushd runtime/Python2/tests + echo "running native tests..." + python2 run.py + rc=$? + if [ $rc != 0 ]; then + echo "failed running native tests" + fi +popd + +if [ $rc == 0 ]; then + pushd runtime-testsuite + echo "running maven tests..." + mvn -q -Dtest=python2.* test + rc=$? + popd +fi + +# return $rc \ No newline at end of file diff --git a/.circleci/scripts/run-tests-python3.sh b/.circleci/scripts/run-tests-python3.sh new file mode 100755 index 000000000..11290cccb --- /dev/null +++ b/.circleci/scripts/run-tests-python3.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -euo pipefail + +python3 --version + +pushd runtime/Python3/tests + echo "running native tests..." + python3 run.py + rc=$? + if [ $rc != 0 ]; then + echo "failed running native tests" + fi +popd + +if [ $rc == 0 ]; then + pushd runtime-testsuite + echo "running maven tests..." + mvn -q -Dtest=python3.* test + rc=$? + popd +fi + +# return $rc \ No newline at end of file diff --git a/.circleci/scripts/run-tests-swift.sh b/.circleci/scripts/run-tests-swift.sh new file mode 100755 index 000000000..c773b6492 --- /dev/null +++ b/.circleci/scripts/run-tests-swift.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -euo pipefail + +pushd runtime/Swift + echo "running native tests..." + ./boot.py --test + rc=$? + if [ $rc != 0 ]; then + echo "failed running native tests" + fi +popd + +if [ $rc == 0 ]; then + pushd runtime-testsuite + echo "running maven tests..." + if [ $GROUP == "LEXER" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=swift.* test + elif [ $GROUP == "PARSER" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=swift.* test + elif [ $GROUP == "RECURSION" ]; then + mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=swift.* test + else + mvn -q -Dtest=swift.* test + fi + popd +fi diff --git a/.travis.yml b/.travis.yml index ca525bd4c..33db1610d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,105 +12,12 @@ cache: - $HOME/Library/Caches/Homebrew stages: - - smoke-test - - main-test +# - smoke-test +# - main-test - extended-test matrix: include: - - os: linux - dist: focal - compiler: clang - jdk: openjdk11 - env: - - TARGET=cpp - - CXX=g++-10 - - GROUP=LEXER - stage: main-test - addons: - apt: - sources: - - sourceline: 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main' - packages: - - g++-10 - - uuid-dev - - clang-10 - - os: linux - dist: focal - compiler: clang - jdk: openjdk11 - env: - - TARGET=cpp - - CXX=g++-10 - - GROUP=PARSER - stage: main-test - addons: - apt: - sources: - - sourceline: 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main' - packages: - - g++-10 - - uuid-dev - - clang-10 - - os: linux - dist: focal - compiler: clang - jdk: openjdk11 - env: - - TARGET=cpp - - CXX=g++-10 - - GROUP=RECURSION - stage: main-test - addons: - apt: - sources: - - sourceline: 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main' - packages: - - g++-10 - - uuid-dev - - clang-10 - - os: osx - compiler: clang - osx_image: xcode10.2 - env: - - TARGET=cpp - - GROUP=LEXER - stage: extended-test - - os: osx - compiler: clang - osx_image: xcode10.2 - env: - - TARGET=cpp - - GROUP=PARSER - stage: extended-test - - os: osx - compiler: clang - osx_image: xcode10.2 - env: - - TARGET=cpp - - GROUP=RECURSION - stage: extended-test - - os: osx - compiler: clang - osx_image: xcode10.2 - env: - - TARGET=swift - - GROUP=LEXER - stage: main-test - - os: osx - compiler: clang - osx_image: xcode10.2 - env: - - TARGET=swift - - GROUP=PARSER - stage: main-test - - os: osx - compiler: clang - osx_image: xcode10.2 - env: - - TARGET=swift - - GROUP=RECURSION - stage: main-test - os: linux dist: xenial compiler: clang @@ -118,95 +25,6 @@ matrix: - TARGET=swift - GROUP=ALL stage: extended-test - - os: osx - osx_image: xcode10.2 - env: - - TARGET=dotnet - - GROUP=LEXER - stage: extended-test - - os: osx - osx_image: xcode10.2 - env: - - TARGET=dotnet - - GROUP=PARSER - stage: extended-test - - os: osx - osx_image: xcode10.2 - env: - - TARGET=dotnet - - GROUP=RECURSION - stage: extended-test - - os: linux - dist: trusty - jdk: openjdk7 - env: TARGET=java - stage: extended-test - - os: linux - jdk: openjdk8 - env: TARGET=java - stage: smoke-test - - os: linux - jdk: openjdk8 - env: - - TARGET=dotnet - - GROUP=MAIN - stage: main-test - - os: linux - jdk: openjdk8 - env: TARGET=dart - stage: main-test - - os: linux - language: php - php: - - 7.2 - jdk: openjdk8 - env: TARGET=php - stage: main-test - - os: linux - jdk: openjdk8 - env: - - TARGET=dotnet - - GROUP=LEXER - stage: extended-test - - os: linux - jdk: openjdk8 - env: - - TARGET=dotnet - - GROUP=PARSER - stage: extended-test - - os: linux - jdk: openjdk8 - env: - - TARGET=dotnet - - GROUP=RECURSION - stage: extended-test - - os: linux - jdk: openjdk8 - env: TARGET=python2 - stage: main-test - - os: linux - jdk: openjdk8 - env: TARGET=python3 - addons: - apt: - sources: - - deadsnakes # source required so it finds the package definition below - packages: - - python3.7 - stage: main-test - - os: linux - dist: trusty - jdk: openjdk8 - env: TARGET=javascript - stage: main-test - before_install: - - nvm install 14 # otherwise it runs by default on node 8 - - f="./.travis/before-install-linux-javascript.sh"; ! [ -x "$f" ] || "$f" - - os: linux - dist: trusty - jdk: openjdk8 - env: TARGET=go - stage: main-test before_install: - f="./.travis/before-install-$TRAVIS_OS_NAME-$TARGET.sh"; ! [ -x "$f" ] || "$f" diff --git a/README.md b/README.md index cd9d7c2ee..72462f557 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,11 @@ # ANTLR v4 -[![Github Build Status (MacOSX)](https://github.com/antlr/antlr4/workflows/MacOSX/badge.svg)](https://github.com/antlr/antlr4/actions) [![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/5acpbx1pg7bhgh8v/branch/master?svg=true)](https://ci.appveyor.com/project/parrt/antlr4) [![Java 7+](https://img.shields.io/badge/java-7+-4c7e9f.svg)](http://java.oracle.com) [![License](https://img.shields.io/badge/license-BSD-blue.svg)](https://raw.githubusercontent.com/antlr/antlr4/master/LICENSE.txt) +[![Github CI Build Status (MacOSX)](https://img.shields.io/github/workflow/status/antlr/antlr4/MacOSX?label=MacOSX)](https://github.com/antlr/antlr4/actions) +[![AppVeyor CI Build Status (Windows)](https://img.shields.io/appveyor/build/parrt/antlr4?label=Windows)](https://ci.appveyor.com/project/parrt/antlr4) +[![Circle CI Build Status (Linux)](https://img.shields.io/circleci/build/gh/antlr/antlr4/master?label=Linux)](https://app.circleci.com/pipelines/github/antlr/antlr4) +[![Travis-CI Build Status (Swift-Linux)](https://img.shields.io/travis/antlr/antlr4.svg?label=Linux-Swift&branch=master)](https://travis-ci.org/antlr/antlr4) +[![Java 7+](https://img.shields.io/badge/java-7+-4c7e9f.svg)](http://java.oracle.com) +[![License](https://img.shields.io/badge/license-BSD-blue.svg)](https://raw.githubusercontent.com/antlr/antlr4/master/LICENSE.txt) **ANTLR** (ANother Tool for Language Recognition) is a powerful parser generator for reading, processing, executing, or translating structured text or binary files. It's widely used to build languages, tools, and frameworks. From a grammar, ANTLR generates a parser that can build parse trees and also generates a listener interface (or visitor) that makes it easy to respond to the recognition of phrases of interest. diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/TestContext.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/TestContext.java index bb1cb78f6..50a3d97b5 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/TestContext.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/TestContext.java @@ -10,10 +10,15 @@ public abstract class TestContext { return "true".equals(String.valueOf(System.getenv("APPVEYOR")).toLowerCase()); } + public static boolean isCircleCI() { + return "true".equals(String.valueOf(System.getenv("CIRCLECI")).toLowerCase()); + } + public static boolean isSupportedTarget(String target) { if(isAppVeyorCI()) return !target.matches("Swift|Node"); else return true; } + } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java index d8c70ca23..780c64839 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java @@ -534,7 +534,7 @@ public class BaseCppTest implements RuntimeTestSupport { } private String runProcess(ProcessBuilder builder, String description, boolean showStderr) throws Exception { -// System.out.println("BUILDER: "+builder.command()); + System.out.println("BUILDER: " + builder.command() + " @ " + builder.directory().toString()); Process process = builder.start(); StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); @@ -712,7 +712,7 @@ public class BaseCppTest implements RuntimeTestSupport { p = Paths.get(runtimeURL.toURI()).toFile().toString(); } catch (URISyntaxException use) { - p = "Can't find runtime"; + p = "Can't find runtime at " + runtimeURL.toString(); } return p; } @@ -987,20 +987,26 @@ public class BaseCppTest implements RuntimeTestSupport { @Override public void eraseTempDir() { - boolean doErase = true; - String propName = getPropertyPrefix() + "-erase-test-dir"; - String prop = System.getProperty(propName); - if(prop!=null && prop.length()>0) - doErase = Boolean.getBoolean(prop); - if(doErase) { + if (shouldEraseTempDir()) { File tmpdirF = new File(tmpdir); - if ( tmpdirF.exists() ) { + if (tmpdirF.exists()) { eraseFiles(tmpdirF); tmpdirF.delete(); } } } + private boolean shouldEraseTempDir() { + if(tmpdir==null) + return false; + String propName = getPropertyPrefix() + "-erase-test-dir"; + String prop = System.getProperty(propName); + if (prop != null && prop.length() > 0) + return Boolean.getBoolean(prop); + else + return true; + } + public String getFirstLineOfException() { if ( this.stderrDuringParse ==null ) { return null; @@ -1152,3 +1158,4 @@ public class BaseCppTest implements RuntimeTestSupport { return dup; } } + diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java index 923b34298..2cd12d015 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java @@ -581,7 +581,7 @@ public class BaseCSharpTest implements RuntimeTestSupport { @Override public void eraseTempDir() { - if (!PRESERVE_TEST_DIR) { + if (shouldEraseTempDir()) { File tmpdirF = new File(tmpdir); if (tmpdirF.exists()) { eraseDirectory(tmpdirF); @@ -590,6 +590,10 @@ public class BaseCSharpTest implements RuntimeTestSupport { } } + private boolean shouldEraseTempDir() { + return tmpdir!=null && !PRESERVE_TEST_DIR; + } + /** * Return map sorted by key */ diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java index fd4da1cfb..5bedebde2 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java @@ -560,10 +560,22 @@ public class BaseDartTest implements RuntimeTestSupport { " path: " + runtime + "\n"); if (cacheDartPackages == null) { try { - Process process = Runtime.getRuntime().exec(new String[]{locatePub(), "get"}, null, new File(tmpdir)); + final Process process = Runtime.getRuntime().exec(new String[]{locatePub(), "get"}, null, new File(tmpdir)); StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); stderrVacuum.start(); + Timer timer = new Timer(); + timer.schedule(new TimerTask() { + @Override + public void run() { + try { + process.destroy(); + } catch(Exception e) { + e.printStackTrace(System.err); + } + } + }, 30_000); process.waitFor(); + timer.cancel(); stderrVacuum.join(); String stderrDuringPubGet = stderrVacuum.toString(); if (!stderrDuringPubGet.isEmpty()) { @@ -609,11 +621,23 @@ public class BaseDartTest implements RuntimeTestSupport { }; String cmdLine = Utils.join(args, " "); System.err.println("Compile: " + cmdLine); - Process process = + final Process process = Runtime.getRuntime().exec(args, null, new File(tmpdir)); StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); stderrVacuum.start(); + Timer timer = new Timer(); + timer.schedule(new TimerTask() { + @Override + public void run() { + try { + process.destroy(); + } catch(Exception e) { + e.printStackTrace(System.err); + } + } + }, 30_000); int result = process.waitFor(); + timer.cancel(); if (result != 0) { stderrVacuum.join(); System.err.print("Error compiling dart file: " + stderrVacuum.toString()); @@ -633,13 +657,25 @@ public class BaseDartTest implements RuntimeTestSupport { } //String cmdLine = Utils.join(args, " "); //System.err.println("execParser: " + cmdLine); - Process process = + final Process process = Runtime.getRuntime().exec(args, null, new File(tmpdir)); StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); stdoutVacuum.start(); stderrVacuum.start(); + Timer timer = new Timer(); + timer.schedule(new TimerTask() { + @Override + public void run() { + try { + process.destroy(); + } catch(Exception e) { + e.printStackTrace(System.err); + } + } + }, 30_000); process.waitFor(); + timer.cancel(); stdoutVacuum.join(); stderrVacuum.join(); String output = stdoutVacuum.toString(); @@ -1031,13 +1067,19 @@ public class BaseDartTest implements RuntimeTestSupport { @Override public void eraseTempDir() { - File tmpdirF = new File(tmpdir); - if (tmpdirF.exists()) { - eraseFiles(); - tmpdirF.delete(); + if(shouldEraseTempDir()) { + File tmpdirF = new File(tmpdir); + if (tmpdirF.exists()) { + eraseFiles(); + tmpdirF.delete(); + } } } + private boolean shouldEraseTempDir() { + return tmpdir!=null; + } + public String getFirstLineOfException() { if (this.stderrDuringParse == null) { return null; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java index 0226d5023..1696d5307 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java @@ -821,18 +821,24 @@ public class BaseGoTest implements RuntimeTestSupport { } public void eraseTempDir() { - boolean doErase = true; - String propName = "antlr-go-erase-test-dir"; - String prop = System.getProperty(propName); - if (prop != null && prop.length() > 0) - doErase = Boolean.getBoolean(prop); - if (doErase) { + if (shouldEraseTempDir()) { if ( overall_tmpdir.exists()) { eraseDirectory(overall_tmpdir); } } } + private boolean shouldEraseTempDir() { + if(overall_tmpdir==null) + return false; + String propName = "antlr-go-erase-test-dir"; + String prop = System.getProperty(propName); + if (prop != null && prop.length() > 0) + return Boolean.getBoolean(prop); + else + return true; + } + public String getFirstLineOfException() { if (this.stderrDuringParse == null) { return null; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java index d142d1df3..606cdede2 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java @@ -1032,6 +1032,9 @@ public class BaseJavaTest implements RuntimeTestSupport { protected void eraseFiles(final String filesEndingWith) { + if (tmpdir == null) { + return; + } File tmpdirF = new File(tmpdir); String[] files = tmpdirF.list(); for(int i = 0; files!=null && i < files.length; i++) { @@ -1045,7 +1048,6 @@ public class BaseJavaTest implements RuntimeTestSupport { if (tmpdir == null) { return; } - File tmpdirF = new File(tmpdir); String[] files = tmpdirF.list(); for(int i = 0; files!=null && i < files.length; i++) { @@ -1054,13 +1056,20 @@ public class BaseJavaTest implements RuntimeTestSupport { } public void eraseTempDir() { - File tmpdirF = new File(tmpdir); - if ( tmpdirF.exists() ) { - eraseFiles(); - tmpdirF.delete(); - } + if (shouldEraseTempDir()) { + File tmpdirF = new File(tmpdir); + if (tmpdirF.exists()) { + eraseFiles(); + tmpdirF.delete(); + } + } } + private boolean shouldEraseTempDir() { + return tmpdir != null; + } + + public String getFirstLineOfException() { if ( this.stderrDuringParse ==null ) { return null; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java index 65fb05aba..99b063158 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java @@ -251,7 +251,7 @@ public class BaseNodeTest implements RuntimeTestSupport { public String execModule(String fileName) { try { String npmPath = locateNpm(); - if(!TestContext.isTravisCI()) { + if(!TestContext.isTravisCI() && !TestContext.isCircleCI()) { installRuntime(npmPath); registerRuntime(npmPath); } @@ -327,7 +327,11 @@ public class BaseNodeTest implements RuntimeTestSupport { } private void linkRuntime(String npmPath) throws IOException, InterruptedException { - ProcessBuilder builder = new ProcessBuilder(npmPath, "link", "antlr4"); + List args = new ArrayList<>(); + if(TestContext.isCircleCI()) + args.add("sudo"); + args.addAll(Arrays.asList(npmPath, "link", "antlr4")); + ProcessBuilder builder = new ProcessBuilder(args.toArray(new String[0])); builder.directory(new File(tmpdir)); builder.redirectError(new File(tmpdir, "error.txt")); builder.redirectOutput(new File(tmpdir, "output.txt")); @@ -479,12 +483,7 @@ public class BaseNodeTest implements RuntimeTestSupport { @Override public void eraseTempDir() { - boolean doErase = true; - String propName = "antlr-javascript-erase-test-dir"; - String prop = System.getProperty(propName); - if (prop != null && prop.length() > 0) - doErase = Boolean.getBoolean(prop); - if (doErase) { + if (shouldEraseTempDir()) { File tmpdirF = new File(tmpdir); if (tmpdirF.exists()) { eraseFiles(tmpdirF); @@ -493,6 +492,16 @@ public class BaseNodeTest implements RuntimeTestSupport { } } + private boolean shouldEraseTempDir() { + if(tmpdir==null) + return false; + String propName = "antlr-javascript-erase-test-dir"; + String prop = System.getProperty(propName); + if (prop != null && prop.length() > 0) + return Boolean.getBoolean(prop); + else + return true; + } /** Sort a list */ public > List sort(List data) { diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java index 89d9d081f..deef63287 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java @@ -581,13 +581,7 @@ public class BasePHPTest implements RuntimeTestSupport { @Override public void eraseTempDir() { - boolean doErase = true; - String propName = getPropertyPrefix() + "-erase-test-dir"; - String prop = System.getProperty(propName); - if (prop != null && prop.length() > 0) { - doErase = Boolean.getBoolean(prop); - } - if (doErase) { + if (shouldEraseTempDir()) { File tmpdirF = new File(tmpdir); if (tmpdirF.exists()) { eraseFiles(tmpdirF); @@ -596,6 +590,17 @@ public class BasePHPTest implements RuntimeTestSupport { } } + private boolean shouldEraseTempDir() { + if(tmpdir==null) + return false; + String propName = getPropertyPrefix() + "-erase-test-dir"; + String prop = System.getProperty(propName); + if (prop != null && prop.length() > 0) + return Boolean.getBoolean(prop); + else + return true; + } + /** * Sort a list */ diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java index c8461ac9c..a3ea85b1b 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java @@ -793,12 +793,7 @@ public abstract class BasePythonTest implements RuntimeTestSupport { @Override public void eraseTempDir() { - boolean doErase = true; - String propName = getPropertyPrefix() + "-erase-test-dir"; - String prop = System.getProperty(propName); - if(prop!=null && prop.length()>0) - doErase = Boolean.getBoolean(prop); - if(doErase) { + if(shouldEraseTempDir()) { File tmpdirF = new File(tmpdir); if ( tmpdirF.exists() ) { eraseFiles(tmpdirF); @@ -807,6 +802,17 @@ public abstract class BasePythonTest implements RuntimeTestSupport { } } + private boolean shouldEraseTempDir() { + if(tmpdir==null) + return false; + String propName = getPropertyPrefix() + "-erase-test-dir"; + String prop = System.getProperty(propName); + if (prop != null && prop.length() > 0) + return Boolean.getBoolean(prop); + else + return true; + } + protected void eraseTempPyCache() { File tmpdirF = new File(tmpdir+"/__pycache__"); if ( tmpdirF.exists() ) { diff --git a/runtime/Python2/tests/mocks/TestLexer.py b/runtime/Python2/tests/mocks/TestLexer.py index 27894ddc4..89c438b63 100644 --- a/runtime/Python2/tests/mocks/TestLexer.py +++ b/runtime/Python2/tests/mocks/TestLexer.py @@ -39,7 +39,7 @@ class TestLexer(Lexer): def __init__(self, input=None): super(TestLexer, self).__init__(input) - self.checkVersion("4.9") + self.checkVersion("4.9.1") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None self._predicates = None @@ -95,7 +95,7 @@ class TestLexer2(Lexer): def __init__(self, input=None): super(TestLexer2, self).__init__(input) - self.checkVersion("4.8") + self.checkVersion("4.9.1") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None self._predicates = None diff --git a/runtime/Python3/test/TestFileStream.py b/runtime/Python3/tests/TestFileStream.py similarity index 100% rename from runtime/Python3/test/TestFileStream.py rename to runtime/Python3/tests/TestFileStream.py diff --git a/runtime/Python3/test/TestInputStream.py b/runtime/Python3/tests/TestInputStream.py similarity index 100% rename from runtime/Python3/test/TestInputStream.py rename to runtime/Python3/tests/TestInputStream.py diff --git a/runtime/Python3/test/TestIntervalSet.py b/runtime/Python3/tests/TestIntervalSet.py similarity index 100% rename from runtime/Python3/test/TestIntervalSet.py rename to runtime/Python3/tests/TestIntervalSet.py diff --git a/runtime/Python3/test/TestRecognizer.py b/runtime/Python3/tests/TestRecognizer.py similarity index 100% rename from runtime/Python3/test/TestRecognizer.py rename to runtime/Python3/tests/TestRecognizer.py diff --git a/runtime/Python3/test/TestTokenStreamRewriter.py b/runtime/Python3/tests/TestTokenStreamRewriter.py similarity index 100% rename from runtime/Python3/test/TestTokenStreamRewriter.py rename to runtime/Python3/tests/TestTokenStreamRewriter.py diff --git a/runtime/Python3/test/__init__.py b/runtime/Python3/tests/__init__.py similarity index 100% rename from runtime/Python3/test/__init__.py rename to runtime/Python3/tests/__init__.py diff --git a/runtime/Python3/test/c.c b/runtime/Python3/tests/c.c similarity index 100% rename from runtime/Python3/test/c.c rename to runtime/Python3/tests/c.c diff --git a/runtime/Python3/test/ctest.py b/runtime/Python3/tests/ctest.py similarity index 100% rename from runtime/Python3/test/ctest.py rename to runtime/Python3/tests/ctest.py diff --git a/runtime/Python3/test/expr/Expr.g4 b/runtime/Python3/tests/expr/Expr.g4 similarity index 100% rename from runtime/Python3/test/expr/Expr.g4 rename to runtime/Python3/tests/expr/Expr.g4 diff --git a/runtime/Python3/test/expr/ExprLexer.py b/runtime/Python3/tests/expr/ExprLexer.py similarity index 100% rename from runtime/Python3/test/expr/ExprLexer.py rename to runtime/Python3/tests/expr/ExprLexer.py diff --git a/runtime/Python3/test/expr/ExprParser.py b/runtime/Python3/tests/expr/ExprParser.py similarity index 100% rename from runtime/Python3/test/expr/ExprParser.py rename to runtime/Python3/tests/expr/ExprParser.py diff --git a/runtime/Python3/test/mocks/TestLexer.py b/runtime/Python3/tests/mocks/TestLexer.py similarity index 99% rename from runtime/Python3/test/mocks/TestLexer.py rename to runtime/Python3/tests/mocks/TestLexer.py index 27894ddc4..19a34896b 100644 --- a/runtime/Python3/test/mocks/TestLexer.py +++ b/runtime/Python3/tests/mocks/TestLexer.py @@ -95,7 +95,7 @@ class TestLexer2(Lexer): def __init__(self, input=None): super(TestLexer2, self).__init__(input) - self.checkVersion("4.8") + self.checkVersion("4.9.1") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None self._predicates = None diff --git a/runtime/Python3/test/mocks/__init__.py b/runtime/Python3/tests/mocks/__init__.py similarity index 100% rename from runtime/Python3/test/mocks/__init__.py rename to runtime/Python3/tests/mocks/__init__.py diff --git a/runtime/Python3/test/parser/__init__.py b/runtime/Python3/tests/parser/__init__.py similarity index 100% rename from runtime/Python3/test/parser/__init__.py rename to runtime/Python3/tests/parser/__init__.py diff --git a/runtime/Python3/test/parser/clexer.py b/runtime/Python3/tests/parser/clexer.py similarity index 100% rename from runtime/Python3/test/parser/clexer.py rename to runtime/Python3/tests/parser/clexer.py diff --git a/runtime/Python3/test/parser/cparser.py b/runtime/Python3/tests/parser/cparser.py similarity index 100% rename from runtime/Python3/test/parser/cparser.py rename to runtime/Python3/tests/parser/cparser.py diff --git a/runtime/Python3/test/run.py b/runtime/Python3/tests/run.py similarity index 100% rename from runtime/Python3/test/run.py rename to runtime/Python3/tests/run.py diff --git a/runtime/Python3/test/xpathtest.py b/runtime/Python3/tests/xpathtest.py similarity index 100% rename from runtime/Python3/test/xpathtest.py rename to runtime/Python3/tests/xpathtest.py From 5521eb2936f7c26fba3516a4bbdc98348f42cd27 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Wed, 27 Jan 2021 01:30:53 +0800 Subject: [PATCH 35/42] update readme --- README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 72462f557..f26482ff3 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,14 @@ # ANTLR v4 +[![Java 7+](https://img.shields.io/badge/java-7+-4c7e9f.svg)](http://java.oracle.com) +[![License](https://img.shields.io/badge/license-BSD-blue.svg)](https://raw.githubusercontent.com/antlr/antlr4/master/LICENSE.txt) + +**Build status** + [![Github CI Build Status (MacOSX)](https://img.shields.io/github/workflow/status/antlr/antlr4/MacOSX?label=MacOSX)](https://github.com/antlr/antlr4/actions) [![AppVeyor CI Build Status (Windows)](https://img.shields.io/appveyor/build/parrt/antlr4?label=Windows)](https://ci.appveyor.com/project/parrt/antlr4) [![Circle CI Build Status (Linux)](https://img.shields.io/circleci/build/gh/antlr/antlr4/master?label=Linux)](https://app.circleci.com/pipelines/github/antlr/antlr4) -[![Travis-CI Build Status (Swift-Linux)](https://img.shields.io/travis/antlr/antlr4.svg?label=Linux-Swift&branch=master)](https://travis-ci.org/antlr/antlr4) -[![Java 7+](https://img.shields.io/badge/java-7+-4c7e9f.svg)](http://java.oracle.com) -[![License](https://img.shields.io/badge/license-BSD-blue.svg)](https://raw.githubusercontent.com/antlr/antlr4/master/LICENSE.txt) +[![Travis-CI Build Status (Swift-Linux)](https://img.shields.io/travis/antlr/antlr4.svg?label=Linux-Swift&branch=master)](https://travis-ci.com/github/antlr/antlr4) **ANTLR** (ANother Tool for Language Recognition) is a powerful parser generator for reading, processing, executing, or translating structured text or binary files. It's widely used to build languages, tools, and frameworks. From a grammar, ANTLR generates a parser that can build parse trees and also generates a listener interface (or visitor) that makes it easy to respond to the recognition of phrases of interest. @@ -18,8 +21,8 @@ * [Terence Parr](http://www.cs.usfca.edu/~parrt/), parrt@cs.usfca.edu ANTLR project lead and supreme dictator for life [University of San Francisco](http://www.usfca.edu/) -* [Sam Harwell](http://tunnelvisionlabs.com/) (Tool co-author, Java and C# target) -* Eric Vergnaud (Javascript, Python2, Python3 targets and significant work on C# target) +* [Sam Harwell](http://tunnelvisionlabs.com/) (Tool co-author, Java and original C# target) +* [Eric Vergnaud](https://github.com/ericvergnaud) (Javascript, Python2, Python3 targets and maintenance of C# target) * [Peter Boyer](https://github.com/pboyer) (Go target) * [Mike Lischke](http://www.soft-gems.net/) (C++ completed target) * Dan McLaughlin (C++ initial target) From 59f1f581de0075e9c8620868c7a8532b5ec5c1b0 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Wed, 27 Jan 2021 10:10:24 +0800 Subject: [PATCH 36/42] reduce noise --- .../test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java index 780c64839..136c320cd 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java @@ -534,7 +534,7 @@ public class BaseCppTest implements RuntimeTestSupport { } private String runProcess(ProcessBuilder builder, String description, boolean showStderr) throws Exception { - System.out.println("BUILDER: " + builder.command() + " @ " + builder.directory().toString()); + // System.out.println("BUILDER: " + builder.command() + " @ " + builder.directory().toString()); Process process = builder.start(); StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); From 68ea904826555a40d812167d014bbc6829e9b517 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Wed, 27 Jan 2021 10:15:46 +0800 Subject: [PATCH 37/42] change action runner sequence --- .github/workflows/macosx.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index cbce1a2ee..57541fc6d 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -28,6 +28,6 @@ jobs: run: mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V - name: Test with Maven run: arch -x86_64 .github/scripts/run-tests-${{ matrix.TARGET }}.sh - env: - TARGET: ${{ matrix.TARGET }} + env: GROUP: ${{ matrix.GROUP }} + TARGET: ${{ matrix.TARGET }} From 36bde45396444b15bb15e26590619201520f88ed Mon Sep 17 00:00:00 2001 From: ericvergnaud Date: Wed, 27 Jan 2021 10:58:42 +0800 Subject: [PATCH 38/42] Update macosx.yml --- .github/workflows/macosx.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 57541fc6d..6ed72fc8d 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -12,8 +12,8 @@ jobs: strategy: fail-fast: false matrix: - GROUP: [LEXER, PARSER, RECURSION] TARGET: [swift, cpp, dotnet] + GROUP: [LEXER, PARSER, RECURSION] steps: - uses: actions/checkout@v2 - name: Set up JDK 1.8 @@ -29,5 +29,5 @@ jobs: - name: Test with Maven run: arch -x86_64 .github/scripts/run-tests-${{ matrix.TARGET }}.sh env: - GROUP: ${{ matrix.GROUP }} TARGET: ${{ matrix.TARGET }} + GROUP: ${{ matrix.GROUP }} From be7bc5d81b91d40a1094ab794c3abf568bb806dc Mon Sep 17 00:00:00 2001 From: ericvergnaud Date: Wed, 27 Jan 2021 12:15:28 +0800 Subject: [PATCH 39/42] Update macosx.yml --- .github/workflows/macosx.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 6ed72fc8d..c53c97d18 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -12,7 +12,8 @@ jobs: strategy: fail-fast: false matrix: - TARGET: [swift, cpp, dotnet] +# TARGET: [swift, cpp, dotnet] disabling dotnet which is unstable on M1 + TARGET: [swift, cpp] GROUP: [LEXER, PARSER, RECURSION] steps: - uses: actions/checkout@v2 From 8bcc12de28e4a8f5fb582683d0f6075621fc8ca2 Mon Sep 17 00:00:00 2001 From: Kko <13607681+l215884529@users.noreply.github.com> Date: Mon, 25 Jan 2021 21:43:30 +0800 Subject: [PATCH 40/42] Fix #3052 --- runtime/Python3/src/antlr4/Parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/Python3/src/antlr4/Parser.py b/runtime/Python3/src/antlr4/Parser.py index 081af14ea..ae7f48cce 100644 --- a/runtime/Python3/src/antlr4/Parser.py +++ b/runtime/Python3/src/antlr4/Parser.py @@ -453,7 +453,7 @@ class Parser (Recognizer): def getInvokingContext(self, ruleIndex:int): ctx = self._ctx while ctx is not None: - if ctx.ruleIndex == ruleIndex: + if ctx.getRuleIndex() == ruleIndex: return ctx ctx = ctx.parentCtx return None From ac9d62a99459d75b8766aa88e6a4b71e0e158151 Mon Sep 17 00:00:00 2001 From: Kko <13607681+l215884529@users.noreply.github.com> Date: Mon, 25 Jan 2021 21:45:12 +0800 Subject: [PATCH 41/42] Update contributors.txt Fix #3052 --- contributors.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contributors.txt b/contributors.txt index d4f853975..3dcf53d07 100644 --- a/contributors.txt +++ b/contributors.txt @@ -284,3 +284,4 @@ YYYY/MM/DD, github id, Full name, email 2020/11/26, mr-c, Michael R. Crusoe, 1330696+mr-c@users.noreply.github.com 2020/12/01, maxence-lefebvre, Maxence Lefebvre, maxence-lefebvre@users.noreply.github.com 2020/12/03, electrum, David Phillips, david@acz.org +2021/01/25, l215884529, Qiheng Liu, 13607681+l215884529@users.noreply.github.com From e50ecf49615887b1914e50a069a6f9c7eb92f7cd Mon Sep 17 00:00:00 2001 From: ericvergnaud Date: Fri, 29 Jan 2021 19:54:59 +0800 Subject: [PATCH 42/42] Sanitize test code base (#3061) * sanitize test code base and factorize common code * fix failing tests * fix failing tests * fix failing tests --- .../v4/test/runtime/BaseRuntimeTest.java | 15 +- .../test/runtime/BaseRuntimeTestSupport.java | 215 +++++ .../v4/test/runtime/MockIntTokenStream.java | 91 ++ .../v4/test/runtime/RuntimeTestSupport.java | 21 +- .../v4/test/runtime/RuntimeTestUtils.java | 89 ++ .../v4/test/runtime/cpp/BaseCppTest.java | 752 +--------------- .../test/runtime/csharp/BaseCSharpTest.java | 218 +---- .../v4/test/runtime/dart/BaseDartTest.java | 824 +----------------- .../antlr/v4/test/runtime/go/BaseGoTest.java | 722 +-------------- .../v4/test/runtime/java/BaseJavaTest.java | 673 +------------- .../test/runtime/javascript/BaseNodeTest.java | 230 +---- .../v4/test/runtime/php/BasePHPTest.java | 235 +---- .../test/runtime/python/BasePythonTest.java | 754 +--------------- .../test/runtime/python2/BasePython2Test.java | 4 +- .../test/runtime/python3/BasePython3Test.java | 4 +- .../v4/test/runtime/swift/BaseSwiftTest.java | 118 +-- .../antlr/v4/test/tool/BaseJavaToolTest.java | 5 +- .../v4/test/tool/TestATNInterpreter.java | 5 +- .../v4/test/tool/TestATNLexerInterpreter.java | 3 +- .../v4/test/tool/TestATNParserPrediction.java | 6 +- .../v4/test/tool/TestCompositeGrammars.java | 274 +++--- .../antlr/v4/test/tool/TestDollarParser.java | 2 +- .../antlr/v4/test/tool/TestParserExec.java | 8 +- .../v4/test/tool/TestParserProfiler.java | 2 +- .../antlr/v4/test/tool/TestPerformance.java | 17 +- 25 files changed, 779 insertions(+), 4508 deletions(-) create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTestSupport.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/MockIntTokenStream.java create mode 100644 runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestUtils.java diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java index 17c0d5e96..1604c6987 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java @@ -111,11 +111,6 @@ public abstract class BaseRuntimeTest { this.delegate = delegate; } - public static void mkdir(String dir) { - File f = new File(dir); - f.mkdirs(); - } - @Before public void setUp() throws Exception { // From http://junit.sourceforge.net/javadoc/org/junit/Assume.html @@ -159,7 +154,7 @@ public abstract class BaseRuntimeTest { } public void testParser(RuntimeTestDescriptor descriptor) throws Exception { - mkdir(delegate.getTmpDir()); + RuntimeTestUtils.mkdir(delegate.getTempParserDirPath()); Pair pair = descriptor.getGrammar(); @@ -176,7 +171,7 @@ public abstract class BaseRuntimeTest { g.registerRenderer(String.class, new StringRenderer()); g.importTemplates(targetTemplates); ST grammarST = new ST(g, spair.b); - writeFile(delegate.getTmpDir(), spair.a+".g4", grammarST.render()); + writeFile(delegate.getTempParserDirPath(), spair.a+".g4", grammarST.render()); } } @@ -201,7 +196,7 @@ public abstract class BaseRuntimeTest { } public void testLexer(RuntimeTestDescriptor descriptor) throws Exception { - mkdir(delegate.getTmpDir()); + RuntimeTestUtils.mkdir(delegate.getTempParserDirPath()); Pair pair = descriptor.getGrammar(); @@ -218,7 +213,7 @@ public abstract class BaseRuntimeTest { g.registerRenderer(String.class, new StringRenderer()); g.importTemplates(targetTemplates); ST grammarST = new ST(g, spair.b); - writeFile(delegate.getTmpDir(), spair.a+".g4", grammarST.render()); + writeFile(delegate.getTempParserDirPath(), spair.a+".g4", grammarST.render()); } } @@ -242,7 +237,7 @@ public abstract class BaseRuntimeTest { boolean defaultListener, String... extraOptions) { - mkdir(workdir); + RuntimeTestUtils.mkdir(workdir); writeFile(workdir, grammarFileName, grammarStr); return antlrOnString(workdir, targetName, grammarFileName, defaultListener, extraOptions); } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTestSupport.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTestSupport.java new file mode 100644 index 000000000..fb6d8571d --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTestSupport.java @@ -0,0 +1,215 @@ +package org.antlr.v4.test.runtime; + +import org.antlr.v4.Tool; +import org.antlr.v4.automata.LexerATNFactory; +import org.antlr.v4.automata.ParserATNFactory; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNDeserializer; +import org.antlr.v4.runtime.atn.ATNSerializer; +import org.antlr.v4.semantics.SemanticPipeline; +import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.LexerGrammar; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; + +import java.io.File; +import java.util.Locale; +import java.util.logging.Logger; + +import static org.junit.Assert.assertEquals; + +public abstract class BaseRuntimeTestSupport implements RuntimeTestSupport { + + // -J-Dorg.antlr.v4.test.BaseTest.level=FINE + protected static final Logger logger = Logger.getLogger(BaseRuntimeTestSupport.class.getName()); + + public static final String NEW_LINE = System.getProperty("line.separator"); + public static final String PATH_SEP = System.getProperty("path.separator"); + + private File tempTestDir = null; + + /** If error during parser execution, store stderr here; can't return + * stdout and stderr. This doesn't trap errors from running antlr. + */ + private String parseErrors; + + /** Errors found while running antlr */ + private StringBuilder antlrToolErrors; + + @org.junit.Rule + public final TestRule testWatcher = new TestWatcher() { + + @Override + protected void succeeded(Description description) { + testSucceeded(description); + } + + }; + + protected void testSucceeded(Description description) { + // remove tmpdir if no error. + eraseTempDir(); + } + + @Override + public File getTempParserDir() { + return getTempTestDir(); + } + + @Override + public String getTempParserDirPath() { + return getTempParserDir() == null ? null : getTempParserDir().getAbsolutePath(); + } + + @Override + public final File getTempTestDir() { + return tempTestDir; + } + + @Override + public final String getTempDirPath() { + return tempTestDir ==null ? null : tempTestDir.getAbsolutePath(); + } + + + public void setParseErrors(String errors) { + this.parseErrors = errors; + } + + public String getParseErrors() { + return parseErrors; + } + + public String getANTLRToolErrors() { + if ( antlrToolErrors.length()==0 ) { + return null; + } + return antlrToolErrors.toString(); + } + + protected abstract String getPropertyPrefix(); + + @Override + public void testSetUp() throws Exception { + createTempDir(); + antlrToolErrors = new StringBuilder(); + } + + private void createTempDir() { + // new output dir for each test + String propName = getPropertyPrefix() + "-test-dir"; + String prop = System.getProperty(propName); + if(prop!=null && prop.length()>0) { + tempTestDir = new File(prop); + } + else { + String dirName = getClass().getSimpleName() + "-" + Thread.currentThread().getName() + "-" + System.currentTimeMillis(); + tempTestDir = new File(System.getProperty("java.io.tmpdir"), dirName); + } + } + + @Override + public void testTearDown() throws Exception { + } + + @Override + public void beforeTest(RuntimeTestDescriptor descriptor) { + } + + @Override + public void afterTest(RuntimeTestDescriptor descriptor) { + } + + public void eraseTempDir() { + if(shouldEraseTempDir()) { + eraseDirectory(getTempTestDir()); + } + } + + protected boolean shouldEraseTempDir() { + if(tempTestDir == null) + return false; + String propName = getPropertyPrefix() + "-erase-test-dir"; + String prop = System.getProperty(propName); + if (prop != null && prop.length() > 0) + return Boolean.getBoolean(prop); + else + return true; + } + + public static void eraseDirectory(File dir) { + if ( dir.exists() ) { + eraseFilesInDir(dir); + dir.delete(); + } + } + + + public static void eraseFilesInDir(File dir) { + String[] files = dir.list(); + for(int i = 0; files!=null && i < files.length; i++) { + new File(dir,files[i]).delete(); + } + } + + private static String detectedOS; + + public static String getOS() { + if (detectedOS == null) { + String os = System.getProperty("os.name", "generic").toLowerCase(Locale.ENGLISH); + if ((os.indexOf("mac") >= 0) || (os.indexOf("darwin") >= 0)) { + detectedOS = "mac"; + } + else if (os.indexOf("win") >= 0) { + detectedOS = "windows"; + } + else if (os.indexOf("nux") >= 0) { + detectedOS = "linux"; + } + else { + detectedOS = "unknown"; + } + } + return detectedOS; + } + + + public static boolean isWindows() { + return getOS().equalsIgnoreCase("windows"); + } + + protected ATN createATN(Grammar g, boolean useSerializer) { + if ( g.atn==null ) { + semanticProcess(g); + assertEquals(0, g.tool.getNumErrors()); + + ParserATNFactory f = g.isLexer() ? new LexerATNFactory((LexerGrammar) g) : new ParserATNFactory(g); + + g.atn = f.createATN(); + assertEquals(0, g.tool.getNumErrors()); + } + + ATN atn = g.atn; + if ( useSerializer ) { + char[] serialized = ATNSerializer.getSerializedAsChars(atn); + return new ATNDeserializer().deserialize(serialized); + } + + return atn; + } + protected void semanticProcess(Grammar g) { + if ( g.ast!=null && !g.ast.hasErrors ) { +// System.out.println(g.ast.toStringTree()); + Tool antlr = new Tool(); + SemanticPipeline sem = new SemanticPipeline(g); + sem.process(); + if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any) + for (Grammar imp : g.getImportedGrammars()) { + antlr.processNonCombinedGrammar(imp, false); + } + } + } + } + +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/MockIntTokenStream.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/MockIntTokenStream.java new file mode 100644 index 000000000..06eacddfb --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/MockIntTokenStream.java @@ -0,0 +1,91 @@ +package org.antlr.v4.test.runtime; + +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.misc.IntegerList; +import org.antlr.v4.runtime.misc.Interval; + +public class MockIntTokenStream implements TokenStream { + + public IntegerList types; + int p=0; + + public MockIntTokenStream(IntegerList types) { this.types = types; } + + @Override + public void consume() { p++; } + + @Override + public int LA(int i) { return LT(i).getType(); } + + @Override + public int mark() { + return index(); + } + + @Override + public int index() { return p; } + + @Override + public void release(int marker) { + seek(marker); + } + + @Override + public void seek(int index) { + p = index; + } + + @Override + public int size() { + return types.size(); + } + + @Override + public String getSourceName() { + return UNKNOWN_SOURCE_NAME; + } + + @Override + public Token LT(int i) { + CommonToken t; + int rawIndex = p + i - 1; + if ( rawIndex>=types.size() ) t = new CommonToken(Token.EOF); + else t = new CommonToken(types.get(rawIndex)); + t.setTokenIndex(rawIndex); + return t; + } + + @Override + public Token get(int i) { + return new org.antlr.v4.runtime.CommonToken(types.get(i)); + } + + @Override + public TokenSource getTokenSource() { + return null; + } + + + @Override + public String getText() { + throw new UnsupportedOperationException("can't give strings"); + } + + + @Override + public String getText(Interval interval) { + throw new UnsupportedOperationException("can't give strings"); + } + + + @Override + public String getText(RuleContext ctx) { + throw new UnsupportedOperationException("can't give strings"); + } + + + @Override + public String getText(Token start, Token stop) { + throw new UnsupportedOperationException("can't give strings"); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestSupport.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestSupport.java index ca0950986..87fcc763e 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestSupport.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestSupport.java @@ -6,6 +6,8 @@ package org.antlr.v4.test.runtime; +import java.io.File; + /** This interface describes functionality needed to execute a runtime test. * Unfortunately the Base*Test.java files are big junk drawers. This is * an attempt to make it more obvious what new target implementers have to @@ -14,13 +16,22 @@ package org.antlr.v4.test.runtime; * @since 4.6 */ public interface RuntimeTestSupport { - void testSetUp() throws Exception; - void testTearDown() throws Exception; + + // dir containing grammar input and output + File getTempParserDir(); + String getTempParserDirPath(); + + // dir containing test input and output + File getTempTestDir(); + String getTempDirPath(); void eraseTempDir(); - String getTmpDir(); + void testSetUp() throws Exception; + void testTearDown() throws Exception; + + void beforeTest(RuntimeTestDescriptor descriptor); + void afterTest(RuntimeTestDescriptor descriptor); - String getStdout(); String getParseErrors(); String getANTLRToolErrors(); @@ -40,6 +51,4 @@ public interface RuntimeTestSupport { String input, boolean showDiagnosticErrors); - void beforeTest(RuntimeTestDescriptor descriptor); - void afterTest(RuntimeTestDescriptor descriptor); } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestUtils.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestUtils.java new file mode 100644 index 000000000..d2c30800c --- /dev/null +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestUtils.java @@ -0,0 +1,89 @@ +package org.antlr.v4.test.runtime; + +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.LexerATNSimulator; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.misc.IntegerList; +import org.antlr.v4.tool.LexerGrammar; + +import java.io.*; +import java.util.*; + +public abstract class RuntimeTestUtils { + + /** Sort a list */ + public static > List sort(List data) { + List dup = new ArrayList(data); + dup.addAll(data); + Collections.sort(dup); + return dup; + } + + /** Return map sorted by key */ + public static ,V> LinkedHashMap sort(Map data) { + LinkedHashMap dup = new LinkedHashMap(); + List keys = new ArrayList(data.keySet()); + Collections.sort(keys); + for (K k : keys) { + dup.put(k, data.get(k)); + } + return dup; + } + + public static List getTokenTypes(LexerGrammar lg, + ATN atn, + CharStream input) { + LexerATNSimulator interp = new LexerATNSimulator(atn, new DFA[]{new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE))}, null); + List tokenTypes = new ArrayList(); + int ttype; + boolean hitEOF = false; + do { + if ( hitEOF ) { + tokenTypes.add("EOF"); + break; + } + int t = input.LA(1); + ttype = interp.match(input, Lexer.DEFAULT_MODE); + if ( ttype==Token.EOF ) { + tokenTypes.add("EOF"); + } + else { + tokenTypes.add(lg.typeToTokenList.get(ttype)); + } + + if ( t== IntStream.EOF ) { + hitEOF = true; + } + } while ( ttype!=Token.EOF ); + return tokenTypes; + } + + public static IntegerList getTokenTypesViaATN(String input, LexerATNSimulator lexerATN) { + ANTLRInputStream in = new ANTLRInputStream(input); + IntegerList tokenTypes = new IntegerList(); + int ttype; + do { + ttype = lexerATN.match(in, Lexer.DEFAULT_MODE); + tokenTypes.add(ttype); + } while ( ttype!= Token.EOF ); + return tokenTypes; + } + + public static void copyFile(File source, File dest) throws IOException { + InputStream is = new FileInputStream(source); + OutputStream os = new FileOutputStream(dest); + byte[] buf = new byte[4 << 10]; + int l; + while ((l = is.read(buf)) > -1) { + os.write(buf, 0, l); + } + is.close(); + os.close(); + } + + public static void mkdir(String dir) { + File f = new File(dir); + f.mkdirs(); + } +} diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java index 136c320cd..e6a4526c2 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java @@ -5,68 +5,19 @@ */ package org.antlr.v4.test.runtime.cpp; -import org.antlr.v4.Tool; -import org.antlr.v4.automata.ATNFactory; -import org.antlr.v4.automata.ATNPrinter; -import org.antlr.v4.automata.LexerATNFactory; -import org.antlr.v4.automata.ParserATNFactory; -import org.antlr.v4.codegen.CodeGenerator; -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CommonToken; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.IntStream; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.RuleContext; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenSource; -import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.WritableToken; -import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.runtime.atn.ATNDeserializer; -import org.antlr.v4.runtime.atn.ATNSerializer; -import org.antlr.v4.runtime.atn.ATNState; -import org.antlr.v4.runtime.atn.DecisionState; -import org.antlr.v4.runtime.atn.LexerATNSimulator; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.misc.IntegerList; -import org.antlr.v4.runtime.misc.Interval; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.semantics.SemanticPipeline; -import org.antlr.v4.test.runtime.ErrorQueue; -import org.antlr.v4.test.runtime.RuntimeTestDescriptor; -import org.antlr.v4.test.runtime.RuntimeTestSupport; -import org.antlr.v4.test.runtime.StreamVacuum; -import org.antlr.v4.tool.ANTLRMessage; -import org.antlr.v4.tool.DOTGenerator; -import org.antlr.v4.tool.Grammar; -import org.antlr.v4.tool.GrammarSemanticsMessage; -import org.antlr.v4.tool.LexerGrammar; -import org.antlr.v4.tool.Rule; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; +import org.antlr.v4.test.runtime.*; import org.stringtemplate.v4.ST; -import org.stringtemplate.v4.STGroup; -import org.stringtemplate.v4.STGroupString; import java.io.File; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString; import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; @@ -76,268 +27,12 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -public class BaseCppTest implements RuntimeTestSupport { - // -J-Dorg.antlr.v4.test.BaseTest.level=FINE - // private static final Logger LOGGER = Logger.getLogger(BaseTest.class.getName()); - public static final String newline = System.getProperty("line.separator"); - public static final String pathSep = System.getProperty("path.separator"); +public class BaseCppTest extends BaseRuntimeTestSupport implements RuntimeTestSupport { - public String tmpdir = null; - - /** If error during parser execution, store stderr here; can't return - * stdout and stderr. This doesn't trap errors from running antlr. - */ - protected String stderrDuringParse; - - /** Errors found while running antlr */ - protected StringBuilder antlrToolErrors; - - @org.junit.Rule - public final TestRule testWatcher = new TestWatcher() { - - @Override - protected void succeeded(Description description) { - // remove tmpdir if no error. - eraseTempDir(); - } - - }; - - private String getPropertyPrefix() { + protected String getPropertyPrefix() { return "antlr-" + getLanguage().toLowerCase(); } - @Override - public void testSetUp() throws Exception { - // new output dir for each test - String propName = getPropertyPrefix() + "-test-dir"; - String prop = System.getProperty(propName); - if(prop!=null && prop.length()>0) { - tmpdir = prop; - } - else { - tmpdir = new File(System.getProperty("java.io.tmpdir"), - getClass().getSimpleName()+"-"+Thread.currentThread().getName()+"-"+System.currentTimeMillis()).getAbsolutePath(); - } - antlrToolErrors = new StringBuilder(); - } - - @Override - public void testTearDown() throws Exception { - } - - @Override - public void beforeTest(RuntimeTestDescriptor descriptor) { - System.out.println(descriptor.getTestName()); - } - - @Override - public void afterTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public String getTmpDir() { - return tmpdir; - } - - @Override - public String getStdout() { - return null; - } - - @Override - public String getParseErrors() { - return stderrDuringParse; - } - - @Override - public String getANTLRToolErrors() { - if ( antlrToolErrors.length()==0 ) { - return null; - } - return antlrToolErrors.toString(); - } - - protected org.antlr.v4.Tool newTool(String[] args) { - Tool tool = new Tool(args); - return tool; - } - - protected Tool newTool() { - org.antlr.v4.Tool tool = new Tool(new String[] {"-o", tmpdir}); - return tool; - } - - protected ATN createATN(Grammar g, boolean useSerializer) { - if ( g.atn==null ) { - semanticProcess(g); - assertEquals(0, g.tool.getNumErrors()); - - ParserATNFactory f; - if ( g.isLexer() ) { - f = new LexerATNFactory((LexerGrammar)g); - } - else { - f = new ParserATNFactory(g); - } - - g.atn = f.createATN(); - assertEquals(0, g.tool.getNumErrors()); - } - - ATN atn = g.atn; - if (useSerializer) { - char[] serialized = ATNSerializer.getSerializedAsChars(atn); - return new ATNDeserializer().deserialize(serialized); - } - - return atn; - } - - protected void semanticProcess(Grammar g) { - if ( g.ast!=null && !g.ast.hasErrors ) { - System.out.println(g.ast.toStringTree()); - Tool antlr = new Tool(); - SemanticPipeline sem = new SemanticPipeline(g); - sem.process(); - if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any) - for (Grammar imp : g.getImportedGrammars()) { - antlr.processNonCombinedGrammar(imp, false); - } - } - } - } - - public DFA createDFA(Grammar g, DecisionState s) { -// PredictionDFAFactory conv = new PredictionDFAFactory(g, s); -// DFA dfa = conv.createDFA(); -// conv.issueAmbiguityWarnings(); -// System.out.print("DFA="+dfa); -// return dfa; - return null; - } - -// public void minimizeDFA(DFA dfa) { -// DFAMinimizer dmin = new DFAMinimizer(dfa); -// dfa.minimized = dmin.minimize(); -// } - - IntegerList getTypesFromString(Grammar g, String expecting) { - IntegerList expectingTokenTypes = new IntegerList(); - if ( expecting!=null && !expecting.trim().isEmpty() ) { - for (String tname : expecting.replace(" ", "").split(",")) { - int ttype = g.getTokenType(tname); - expectingTokenTypes.add(ttype); - } - } - return expectingTokenTypes; - } - - public IntegerList getTokenTypesViaATN(String input, LexerATNSimulator lexerATN) { - ANTLRInputStream in = new ANTLRInputStream(input); - IntegerList tokenTypes = new IntegerList(); - int ttype; - do { - ttype = lexerATN.match(in, Lexer.DEFAULT_MODE); - tokenTypes.add(ttype); - } while ( ttype!= Token.EOF ); - return tokenTypes; - } - - public List getTokenTypes(LexerGrammar lg, - ATN atn, - CharStream input) - { - LexerATNSimulator interp = new LexerATNSimulator(atn,new DFA[] { new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE)) },null); - List tokenTypes = new ArrayList(); - int ttype; - boolean hitEOF = false; - do { - if ( hitEOF ) { - tokenTypes.add("EOF"); - break; - } - int t = input.LA(1); - ttype = interp.match(input, Lexer.DEFAULT_MODE); - if ( ttype == Token.EOF ) { - tokenTypes.add("EOF"); - } - else { - tokenTypes.add(lg.typeToTokenList.get(ttype)); - } - - if ( t== IntStream.EOF ) { - hitEOF = true; - } - } while ( ttype!=Token.EOF ); - return tokenTypes; - } - - List checkRuleDFA(String gtext, String ruleName, String expecting) - throws Exception - { - ErrorQueue equeue = new ErrorQueue(); - Grammar g = new Grammar(gtext, equeue); - ATN atn = createATN(g, false); - ATNState s = atn.ruleToStartState[g.getRule(ruleName).index]; - if ( s==null ) { - System.err.println("no such rule: "+ruleName); - return null; - } - ATNState t = s.transition(0).target; - if ( !(t instanceof DecisionState) ) { - System.out.println(ruleName+" has no decision"); - return null; - } - DecisionState blk = (DecisionState)t; - checkRuleDFA(g, blk, expecting); - return equeue.all; - } - - List checkRuleDFA(String gtext, int decision, String expecting) - throws Exception - { - ErrorQueue equeue = new ErrorQueue(); - Grammar g = new Grammar(gtext, equeue); - ATN atn = createATN(g, false); - DecisionState blk = atn.decisionToState.get(decision); - checkRuleDFA(g, blk, expecting); - return equeue.all; - } - - void checkRuleDFA(Grammar g, DecisionState blk, String expecting) - throws Exception - { - DFA dfa = createDFA(g, blk); - String result = null; - if ( dfa!=null ) result = dfa.toString(); - assertEquals(expecting, result); - } - - List checkLexerDFA(String gtext, String expecting) - throws Exception - { - return checkLexerDFA(gtext, LexerGrammar.DEFAULT_MODE_NAME, expecting); - } - - List checkLexerDFA(String gtext, String modeName, String expecting) - throws Exception - { - ErrorQueue equeue = new ErrorQueue(); - LexerGrammar g = new LexerGrammar(gtext, equeue); - g.atn = createATN(g, false); -// LexerATNToDFAConverter conv = new LexerATNToDFAConverter(g); -// DFA dfa = conv.createDFA(modeName); -// g.setLookaheadDFA(0, dfa); // only one decision to worry about -// -// String result = null; -// if ( dfa!=null ) result = dfa.toString(); -// assertEquals(expecting, result); -// -// return equeue.all; - return null; - } - protected String getLanguage() { return "Cpp"; } @@ -362,44 +57,13 @@ public class BaseCppTest implements RuntimeTestSupport { null, lexerName,"-no-listener"); assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); writeLexerTestFile(lexerName, showDFA); String output = execModule("Test.cpp"); return output; } - public ParseTree execStartRule(String startRuleName, Parser parser) - throws IllegalAccessException, InvocationTargetException, - NoSuchMethodException - { - Method startRule = null; - Object[] args = null; - try { - startRule = parser.getClass().getMethod(startRuleName); - } - catch (NoSuchMethodException nsme) { - // try with int _p arg for recursive func - startRule = parser.getClass().getMethod(startRuleName, int.class); - args = new Integer[] {0}; - } - ParseTree result = (ParseTree)startRule.invoke(parser, args); -// System.out.println("parse tree = "+result.toStringTree(parser)); - return result; - } -// protected String execParser(String grammarFileName, -// String grammarStr, -// String parserName, -// String lexerName, -// String listenerName, -// String visitorName, -// String startRuleName, -// String input, -// boolean debug) { -// return execParser(grammarFileName, grammarStr, parserName, lexerName, -// listenerName, visitorName, startRuleName, input, debug); -// } -// @Override public String execParser(String grammarFileName, String grammarStr, @@ -417,7 +81,7 @@ public class BaseCppTest implements RuntimeTestSupport { lexerName, "-visitor"); assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); rawBuildRecognizerTestFile(parserName, lexerName, listenerName, @@ -447,7 +111,7 @@ public class BaseCppTest implements RuntimeTestSupport { String... extraOptions) { ErrorQueue equeue = - antlrOnString(getTmpDir(), "Cpp", grammarFileName, grammarStr, defaultListener, extraOptions); + antlrOnString(getTempDirPath(), "Cpp", grammarFileName, grammarStr, defaultListener, extraOptions); if (!equeue.errors.isEmpty()) { return false; } @@ -481,7 +145,7 @@ public class BaseCppTest implements RuntimeTestSupport { boolean debug, boolean trace) { - this.stderrDuringParse = null; + setParseErrors(null); if ( parserName==null ) { writeLexerTestFile(lexerName, false); } @@ -500,26 +164,6 @@ public class BaseCppTest implements RuntimeTestSupport { } - private static String detectedOS; - public static String getOS() { - if (detectedOS == null) { - String os = System.getProperty("os.name", "generic").toLowerCase(Locale.ENGLISH); - if ((os.indexOf("mac") >= 0) || (os.indexOf("darwin") >= 0)) { - detectedOS = "mac"; - } - else if (os.indexOf("win") >= 0) { - detectedOS = "windows"; - } - else if (os.indexOf("nux") >= 0) { - detectedOS = "linux"; - } - else { - detectedOS = "unknown"; - } - } - return detectedOS; - } - public List allCppFiles(String path) { ArrayList files = new ArrayList(); File folder = new File(path); @@ -545,16 +189,16 @@ public class BaseCppTest implements RuntimeTestSupport { stderrVacuum.join(); String output = stdoutVacuum.toString(); if ( stderrVacuum.toString().length()>0 ) { - this.stderrDuringParse = stderrVacuum.toString(); - if ( showStderr ) System.err.println(this.stderrDuringParse); + setParseErrors(stderrVacuum.toString()); + if ( showStderr ) System.err.println(getParseErrors()); } if (errcode != 0) { String err = "execution of '"+description+"' failed with error code: "+errcode; - if ( this.stderrDuringParse!=null ) { - this.stderrDuringParse += err; + if ( getParseErrors()!=null ) { + setParseErrors(getParseErrors() + err); } else { - this.stderrDuringParse = err; + setParseErrors(err); } } @@ -621,15 +265,15 @@ public class BaseCppTest implements RuntimeTestSupport { public String execModule(String fileName) { String runtimePath = locateRuntime(); String includePath = runtimePath + "/runtime/src"; - String binPath = new File(new File(tmpdir), "a.out").getAbsolutePath(); - String inputPath = new File(new File(tmpdir), "input").getAbsolutePath(); + String binPath = new File(getTempTestDir(), "a.out").getAbsolutePath(); + String inputPath = new File(getTempTestDir(), "input").getAbsolutePath(); // Build runtime using cmake once. synchronized (runtimeBuiltOnce) { if ( !runtimeBuiltOnce ) { try { String command[] = {"clang++", "--version"}; - String output = runCommand(command, tmpdir, "printing compiler version", false); + String output = runCommand(command, getTempDirPath(), "printing compiler version", false); System.out.println("Compiler version is: "+output); } catch (Exception e) { @@ -649,7 +293,7 @@ public class BaseCppTest implements RuntimeTestSupport { String libExtension = (getOS().equals("mac")) ? "dylib" : "so"; try { String command[] = { "ln", "-s", runtimePath + "/dist/libantlr4-runtime." + libExtension }; - if (runCommand(command, tmpdir, "sym linking C++ runtime", true) == null) + if (runCommand(command, getTempDirPath(), "sym linking C++ runtime", true) == null) return null; } catch (Exception e) { @@ -660,8 +304,8 @@ public class BaseCppTest implements RuntimeTestSupport { try { List command2 = new ArrayList(Arrays.asList("clang++", "-std=c++11", "-I", includePath, "-L.", "-lantlr4-runtime", "-o", "a.out")); - command2.addAll(allCppFiles(tmpdir)); - if (runCommand(command2.toArray(new String[0]), tmpdir, "building test binary", true) == null) { + command2.addAll(allCppFiles(getTempDirPath())); + if (runCommand(command2.toArray(new String[0]), getTempDirPath(), "building test binary", true) == null) { return null; } } @@ -672,10 +316,10 @@ public class BaseCppTest implements RuntimeTestSupport { } // Now run the newly minted binary. Reset the error output, as we could have got compiler warnings which are not relevant here. - this.stderrDuringParse = null; + setParseErrors(null); try { ProcessBuilder builder = new ProcessBuilder(binPath, inputPath); - builder.directory(new File(tmpdir)); + builder.directory(getTempTestDir()); Map env = builder.environment(); env.put("LD_PRELOAD", runtimePath + "/dist/libantlr4-runtime." + libExtension); String output = runProcess(builder, "running test binary", false); @@ -717,151 +361,6 @@ public class BaseCppTest implements RuntimeTestSupport { return p; } - List getMessagesOfType(List msgs, Class c) { - List filtered = new ArrayList(); - for (ANTLRMessage m : msgs) { - if ( m.getClass() == c ) filtered.add(m); - } - return filtered; - } - - void checkRuleATN(Grammar g, String ruleName, String expecting) { - ParserATNFactory f = new ParserATNFactory(g); - ATN atn = f.createATN(); - - DOTGenerator dot = new DOTGenerator(g); - System.out.println(dot.getDOT(atn.ruleToStartState[g.getRule(ruleName).index])); - - Rule r = g.getRule(ruleName); - ATNState startState = atn.ruleToStartState[r.index]; - ATNPrinter serializer = new ATNPrinter(g, startState); - String result = serializer.asString(); - - //System.out.print(result); - assertEquals(expecting, result); - } - - public void testActions(String templates, String actionName, String action, String expected) throws org.antlr.runtime.RecognitionException { - int lp = templates.indexOf('('); - String name = templates.substring(0, lp); - STGroup group = new STGroupString(templates); - ST st = group.getInstanceOf(name); - st.add(actionName, action); - String grammar = st.render(); - ErrorQueue equeue = new ErrorQueue(); - Grammar g = new Grammar(grammar, equeue); - if ( g.ast!=null && !g.ast.hasErrors ) { - SemanticPipeline sem = new SemanticPipeline(g); - sem.process(); - - ATNFactory factory = new ParserATNFactory(g); - if ( g.isLexer() ) factory = new LexerATNFactory((LexerGrammar)g); - g.atn = factory.createATN(); - - CodeGenerator gen = new CodeGenerator(g); - ST outputFileST = gen.generateParser(); - String output = outputFileST.render(); - //System.out.println(output); - String b = "#" + actionName + "#"; - int start = output.indexOf(b); - String e = "#end-" + actionName + "#"; - int end = output.indexOf(e); - String snippet = output.substring(start+b.length(),end); - assertEquals(expected, snippet); - } - if ( equeue.size()>0 ) { - System.err.println(equeue.toString()); - } - } - - protected void checkGrammarSemanticsError(ErrorQueue equeue, - GrammarSemanticsMessage expectedMessage) - throws Exception - { - ANTLRMessage foundMsg = null; - for (int i = 0; i < equeue.errors.size(); i++) { - ANTLRMessage m = equeue.errors.get(i); - if (m.getErrorType()==expectedMessage.getErrorType() ) { - foundMsg = m; - } - } - assertNotNull("no error; "+expectedMessage.getErrorType()+" expected", foundMsg); - assertTrue("error is not a GrammarSemanticsMessage", - foundMsg instanceof GrammarSemanticsMessage); - assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs())); - if ( equeue.size()!=1 ) { - System.err.println(equeue); - } - } - - protected void checkGrammarSemanticsWarning(ErrorQueue equeue, - GrammarSemanticsMessage expectedMessage) - throws Exception - { - ANTLRMessage foundMsg = null; - for (int i = 0; i < equeue.warnings.size(); i++) { - ANTLRMessage m = equeue.warnings.get(i); - if (m.getErrorType()==expectedMessage.getErrorType() ) { - foundMsg = m; - } - } - assertNotNull("no error; "+expectedMessage.getErrorType()+" expected", foundMsg); - assertTrue("error is not a GrammarSemanticsMessage", - foundMsg instanceof GrammarSemanticsMessage); - assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs())); - if ( equeue.size()!=1 ) { - System.err.println(equeue); - } - } - - protected void checkError(ErrorQueue equeue, - ANTLRMessage expectedMessage) - throws Exception - { - //System.out.println("errors="+equeue); - ANTLRMessage foundMsg = null; - for (int i = 0; i < equeue.errors.size(); i++) { - ANTLRMessage m = equeue.errors.get(i); - if (m.getErrorType()==expectedMessage.getErrorType() ) { - foundMsg = m; - } - } - assertTrue("no error; "+expectedMessage.getErrorType()+" expected", !equeue.errors.isEmpty()); - assertTrue("too many errors; "+equeue.errors, equeue.errors.size()<=1); - assertNotNull("couldn't find expected error: "+expectedMessage.getErrorType(), foundMsg); - /* - assertTrue("error is not a GrammarSemanticsMessage", - foundMsg instanceof GrammarSemanticsMessage); - */ - assertArrayEquals(expectedMessage.getArgs(), foundMsg.getArgs()); - } - - public static class FilteringTokenStream extends CommonTokenStream { - public FilteringTokenStream(TokenSource src) { super(src); } - Set hide = new HashSet(); - @Override - protected boolean sync(int i) { - if (!super.sync(i)) { - return false; - } - - Token t = get(i); - if ( hide.contains(t.getType()) ) { - ((WritableToken)t).setChannel(Token.HIDDEN_CHANNEL); - } - - return true; - } - public void setTokenTypeChannel(int ttype, int channel) { - hide.add(ttype); - } - } - - protected void mkdir(String dir) { - File f = new File(dir); - f.mkdirs(); - } - protected void writeParserTestFile(String parserName, String lexerName, String listenerName, String visitorName, String parserStartRuleName, boolean debug, boolean trace) { @@ -922,7 +421,7 @@ public class BaseCppTest implements RuntimeTestSupport { outputFileST.add("listenerName", listenerName); outputFileST.add("visitorName", visitorName); outputFileST.add("parserStartRuleName", parserStartRuleName); - writeFile(tmpdir, "Test.cpp", outputFileST.render()); + writeFile(getTempDirPath(), "Test.cpp", outputFileST.render()); } protected void writeLexerTestFile(String lexerName, boolean showDFA) { @@ -947,215 +446,8 @@ public class BaseCppTest implements RuntimeTestSupport { + " return 0;\n" + "}\n"); outputFileST.add("lexerName", lexerName); - writeFile(tmpdir, "Test.cpp", outputFileST.render()); + writeFile(getTempDirPath(), "Test.cpp", outputFileST.render()); } - public void writeRecognizer(String parserName, String lexerName, - String listenerName, String visitorName, - String parserStartRuleName, boolean debug, boolean trace) { - if ( parserName==null ) { - writeLexerTestFile(lexerName, debug); - } - else { - writeParserTestFile(parserName, - lexerName, - listenerName, - visitorName, - parserStartRuleName, - debug, - trace); - } - } - - - protected void eraseFiles(final String filesEndingWith) { - File tmpdirF = new File(tmpdir); - String[] files = tmpdirF.list(); - for(int i = 0; files!=null && i < files.length; i++) { - if ( files[i].endsWith(filesEndingWith) ) { - new File(tmpdir+"/"+files[i]).delete(); - } - } - } - - protected void eraseFiles(File dir) { - String[] files = dir.list(); - for(int i = 0; files!=null && i < files.length; i++) { - new File(dir,files[i]).delete(); - } - } - - @Override - public void eraseTempDir() { - if (shouldEraseTempDir()) { - File tmpdirF = new File(tmpdir); - if (tmpdirF.exists()) { - eraseFiles(tmpdirF); - tmpdirF.delete(); - } - } - } - - private boolean shouldEraseTempDir() { - if(tmpdir==null) - return false; - String propName = getPropertyPrefix() + "-erase-test-dir"; - String prop = System.getProperty(propName); - if (prop != null && prop.length() > 0) - return Boolean.getBoolean(prop); - else - return true; - } - - public String getFirstLineOfException() { - if ( this.stderrDuringParse ==null ) { - return null; - } - String[] lines = this.stderrDuringParse.split("\n"); - String prefix="Exception in thread \"main\" "; - return lines[0].substring(prefix.length(),lines[0].length()); - } - - /** - * When looking at a result set that consists of a Map/HashTable - * we cannot rely on the output order, as the hashing algorithm or other aspects - * of the implementation may be different on different JDKs or platforms. Hence - * we take the Map, convert the keys to a List, sort them and Stringify the Map, which is a - * bit of a hack, but guarantees that we get the same order on all systems. We assume that - * the keys are strings. - * - * @param m The Map that contains keys we wish to return in sorted order - * @return A string that represents all the keys in sorted order. - */ - public String sortMapToString(Map m) { - // Pass in crap, and get nothing back - // - if (m == null) { - return null; - } - - System.out.println("Map toString looks like: " + m.toString()); - - // Sort the keys in the Map - // - TreeMap nset = new TreeMap(m); - - System.out.println("Tree map looks like: " + nset.toString()); - return nset.toString(); - } - - public List realElements(List elements) { - return elements.subList(Token.MIN_USER_TOKEN_TYPE, elements.size()); - } - - public void assertNotNullOrEmpty(String message, String text) { - assertNotNull(message, text); - assertFalse(message, text.isEmpty()); - } - - public void assertNotNullOrEmpty(String text) { - assertNotNull(text); - assertFalse(text.isEmpty()); - } - - public static class IntTokenStream implements TokenStream { - IntegerList types; - int p=0; - public IntTokenStream(IntegerList types) { this.types = types; } - - @Override - public void consume() { p++; } - - @Override - public int LA(int i) { return LT(i).getType(); } - - @Override - public int mark() { - return index(); - } - - @Override - public int index() { return p; } - - @Override - public void release(int marker) { - seek(marker); - } - - @Override - public void seek(int index) { - p = index; - } - - @Override - public int size() { - return types.size(); - } - - @Override - public String getSourceName() { - return null; - } - - @Override - public Token LT(int i) { - CommonToken t; - int rawIndex = p + i - 1; - if ( rawIndex>=types.size() ) t = new CommonToken(Token.EOF); - else t = new CommonToken(types.get(rawIndex)); - t.setTokenIndex(rawIndex); - return t; - } - - @Override - public Token get(int i) { - return new org.antlr.v4.runtime.CommonToken(types.get(i)); - } - - @Override - public TokenSource getTokenSource() { - return null; - } - - @Override - public String getText() { - throw new UnsupportedOperationException("can't give strings"); - } - - @Override - public String getText(Interval interval) { - throw new UnsupportedOperationException("can't give strings"); - } - - @Override - public String getText(RuleContext ctx) { - throw new UnsupportedOperationException("can't give strings"); - } - - @Override - public String getText(Token start, Token stop) { - throw new UnsupportedOperationException("can't give strings"); - } - } - - /** Sort a list */ - public > List sort(List data) { - List dup = new ArrayList(); - dup.addAll(data); - Collections.sort(dup); - return dup; - } - - /** Return map sorted by key */ - public ,V> LinkedHashMap sort(Map data) { - LinkedHashMap dup = new LinkedHashMap(); - List keys = new ArrayList(); - keys.addAll(data.keySet()); - Collections.sort(keys); - for (K k : keys) { - dup.put(k, data.get(k)); - } - return dup; - } } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java index 2cd12d015..f6c4ab5c5 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java @@ -5,36 +5,14 @@ */ package org.antlr.v4.test.runtime.csharp; -import org.antlr.v4.Tool; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenSource; -import org.antlr.v4.runtime.WritableToken; import org.antlr.v4.runtime.misc.Utils; import org.antlr.v4.test.runtime.*; -import org.antlr.v4.tool.ANTLRMessage; -import org.antlr.v4.tool.GrammarSemanticsMessage; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; import org.stringtemplate.v4.ST; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathFactory; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.io.OutputStream; import java.net.URL; import java.nio.file.Path; @@ -53,134 +31,11 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -public class BaseCSharpTest implements RuntimeTestSupport { - public static final String newline = System.getProperty("line.separator"); - - /** - * When the {@code antlr.preserve-test-dir} runtime property is set to - * {@code true}, the temporary directories created by the test run will not - * be removed at the end of the test run, even for tests that completed - * successfully. - * - *

- * The default behavior (used in all other cases) is removing the temporary - * directories for all tests which completed successfully, and preserving - * the directories for tests which failed.

- */ - public static final boolean PRESERVE_TEST_DIR = Boolean.parseBoolean(System.getProperty("antlr-preserve-csharp-test-dir")); - - /** - * The base test directory is the directory where generated files get placed - * during unit test execution. - * - *

- * The default value for this property is the {@code java.io.tmpdir} system - * property, and can be overridden by setting the - * {@code antlr.java-test-dir} property to a custom location. Note that the - * {@code antlr.java-test-dir} property directly affects the - * {@link #CREATE_PER_TEST_DIRECTORIES} value as well.

- */ - public static final String BASE_TEST_DIR; - - /** - * When {@code true}, a temporary directory will be created for each test - * executed during the test run. - * - *

- * This value is {@code true} when the {@code antlr.java-test-dir} system - * property is set, and otherwise {@code false}.

- */ - public static final boolean CREATE_PER_TEST_DIRECTORIES; - - static { - String baseTestDir = System.getProperty("antlr-csharp-test-dir"); - boolean perTestDirectories = false; - if (baseTestDir == null || baseTestDir.isEmpty()) { - baseTestDir = System.getProperty("java.io.tmpdir"); - perTestDirectories = true; - } - - if (!new File(baseTestDir).isDirectory()) { - throw new UnsupportedOperationException("The specified base test directory does not exist: " + baseTestDir); - } - - BASE_TEST_DIR = baseTestDir; - CREATE_PER_TEST_DIRECTORIES = perTestDirectories; - } - - public String tmpdir = null; - - /** - * If error during parser execution, store stderr here; can't return - * stdout and stderr. This doesn't trap errors from running antlr. - */ - protected String stderrDuringParse; - - /** - * Errors found while running antlr - */ - protected StringBuilder antlrToolErrors; - - @org.junit.Rule - public final TestRule testWatcher = new TestWatcher() { - - @Override - protected void succeeded(Description description) { - // remove tmpdir if no error. - eraseTempDir(); - } - - }; +public class BaseCSharpTest extends BaseRuntimeTestSupport implements RuntimeTestSupport { @Override - public void testSetUp() throws Exception { - if (CREATE_PER_TEST_DIRECTORIES) { - // new output dir for each test - String testDirectory = getClass().getSimpleName() + "-" + Thread.currentThread().getName() + "-" + System.currentTimeMillis(); - tmpdir = new File(BASE_TEST_DIR, testDirectory).getAbsolutePath(); - } else { - tmpdir = new File(BASE_TEST_DIR).getAbsolutePath(); - if (!PRESERVE_TEST_DIR && new File(tmpdir).exists()) { - eraseDirectory(new File(tmpdir)); - } - } - antlrToolErrors = new StringBuilder(); - } - - @Override - public void testTearDown() throws Exception { - } - - @Override - public void beforeTest(RuntimeTestDescriptor descriptor) { - System.out.println(descriptor.getTestName()); - } - - @Override - public void afterTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public String getTmpDir() { - return tmpdir; - } - - @Override - public String getStdout() { - return null; - } - - @Override - public String getParseErrors() { - return stderrDuringParse; - } - - @Override - public String getANTLRToolErrors() { - if (antlrToolErrors.length() == 0) { - return null; - } - return antlrToolErrors.toString(); + protected String getPropertyPrefix() { + return "antlr4-csharp"; } protected String execLexer(String grammarFileName, @@ -201,12 +56,12 @@ public class BaseCSharpTest implements RuntimeTestSupport { null, lexerName); assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); writeLexerTestFile(lexerName, showDFA); addSourceFiles("Test.cs"); if (!compile()) { System.err.println("Failed to compile!"); - return stderrDuringParse; + return getParseErrors(); } String output = execTest(); if (output != null && output.length() == 0) { @@ -218,8 +73,7 @@ public class BaseCSharpTest implements RuntimeTestSupport { Set sourceFiles = new HashSet<>(); private void addSourceFiles(String... files) { - for (String file : files) - this.sourceFiles.add(file); + Collections.addAll(sourceFiles, files); } @Override @@ -238,7 +92,7 @@ public class BaseCSharpTest implements RuntimeTestSupport { lexerName, "-visitor"); assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); return rawExecRecognizer(parserName, lexerName, startRuleName, @@ -265,7 +119,7 @@ public class BaseCSharpTest implements RuntimeTestSupport { String lexerName, boolean defaultListener, String... extraOptions) { - ErrorQueue equeue = antlrOnString(getTmpDir(), "CSharp", grammarFileName, grammarStr, defaultListener, extraOptions); + ErrorQueue equeue = antlrOnString(getTempDirPath(), "CSharp", grammarFileName, grammarStr, defaultListener, extraOptions); if (!equeue.errors.isEmpty()) { return false; } @@ -295,7 +149,7 @@ public class BaseCSharpTest implements RuntimeTestSupport { String lexerName, String parserStartRuleName, boolean debug) { - this.stderrDuringParse = null; + setParseErrors(null); if (parserName == null) { writeLexerTestFile(lexerName, false); } else { @@ -330,14 +184,14 @@ public class BaseCSharpTest implements RuntimeTestSupport { } private String locateExec() { - return new File(tmpdir, "bin/Release/netcoreapp3.1/Test.dll").getAbsolutePath(); + return new File(getTempTestDir(), "bin/Release/netcoreapp3.1/Test.dll").getAbsolutePath(); } public boolean buildProject() { try { // save auxiliary files String pack = BaseCSharpTest.class.getPackage().getName().replace(".", "/") + "/"; - saveResourceAsFile(pack + "Antlr4.Test.csproj", new File(tmpdir, "Antlr4.Test.csproj")); + saveResourceAsFile(pack + "Antlr4.Test.csproj", new File(getTempTestDir(), "Antlr4.Test.csproj")); // find runtime package final ClassLoader loader = Thread.currentThread().getContextClassLoader(); @@ -356,7 +210,7 @@ public class BaseCSharpTest implements RuntimeTestSupport { "reference", runtimeProjPath }; - boolean success = runProcess(args, tmpdir); + boolean success = runProcess(args, getTempDirPath()); assertTrue(success); // build test @@ -367,7 +221,7 @@ public class BaseCSharpTest implements RuntimeTestSupport { "-c", "Release" }; - success = runProcess(args, tmpdir); + success = runProcess(args, getTempDirPath()); assertTrue(success); } catch (Exception e) { e.printStackTrace(System.err); @@ -395,11 +249,11 @@ public class BaseCSharpTest implements RuntimeTestSupport { int exitValue = process.exitValue(); boolean success = (exitValue == 0); if (!success) { - this.stderrDuringParse = stderrVacuum.toString(); + setParseErrors(stderrVacuum.toString()); System.err.println("runProcess command: " + Utils.join(args, " ")); System.err.println("runProcess exitValue: " + exitValue); System.err.println("runProcess stdoutVacuum: " + stdoutVacuum.toString()); - System.err.println("runProcess stderrVacuum: " + stderrDuringParse); + System.err.println("runProcess stderrVacuum: " + getParseErrors()); } if (exitValue == 132) { // Retry after SIGILL. We are seeing this intermittently on @@ -434,7 +288,7 @@ public class BaseCSharpTest implements RuntimeTestSupport { public String execTest() { String exec = locateExec(); try { - File tmpdirFile = new File(tmpdir); + File tmpdirFile = new File(getTempDirPath()); Path output = tmpdirFile.toPath().resolve("output"); Path errorOutput = tmpdirFile.toPath().resolve("error-output"); String[] args = getExecTestArgs(exec, output, errorOutput); @@ -449,7 +303,7 @@ public class BaseCSharpTest implements RuntimeTestSupport { stdoutVacuum.join(); stderrVacuum.join(); String writtenOutput = TestOutputReading.read(output); - this.stderrDuringParse = TestOutputReading.read(errorOutput); + setParseErrors(TestOutputReading.read(errorOutput)); int exitValue = process.exitValue(); String stdoutString = stdoutVacuum.toString().trim(); String stderrString = stderrVacuum.toString().trim(); @@ -473,7 +327,7 @@ public class BaseCSharpTest implements RuntimeTestSupport { private String[] getExecTestArgs(String exec, Path output, Path errorOutput) { return new String[]{ - "dotnet", exec, new File(tmpdir, "input").getAbsolutePath(), + "dotnet", exec, new File(getTempTestDir(), "input").getAbsolutePath(), output.toAbsolutePath().toString(), errorOutput.toAbsolutePath().toString() }; @@ -533,7 +387,7 @@ public class BaseCSharpTest implements RuntimeTestSupport { outputFileST.add("parserName", parserName); outputFileST.add("lexerName", lexerName); outputFileST.add("parserStartRuleName", parserStartRuleName); - writeFile(tmpdir, "Test.cs", outputFileST.render()); + writeFile(getTempDirPath(), "Test.cs", outputFileST.render()); } protected void writeLexerTestFile(String lexerName, boolean showDFA) { @@ -562,36 +416,7 @@ public class BaseCSharpTest implements RuntimeTestSupport { ); outputFileST.add("lexerName", lexerName); - writeFile(tmpdir, "Test.cs", outputFileST.render()); - } - - protected void eraseDirectory(File dir) { - File[] files = dir.listFiles(); - if (files != null) { - for (File file : files) { - if (file.isDirectory()) { - eraseDirectory(file); - } else { - file.delete(); - } - } - } - dir.delete(); - } - - @Override - public void eraseTempDir() { - if (shouldEraseTempDir()) { - File tmpdirF = new File(tmpdir); - if (tmpdirF.exists()) { - eraseDirectory(tmpdirF); - tmpdirF.delete(); - } - } - } - - private boolean shouldEraseTempDir() { - return tmpdir!=null && !PRESERVE_TEST_DIR; + writeFile(getTempDirPath(), "Test.cs", outputFileST.render()); } /** @@ -599,8 +424,7 @@ public class BaseCSharpTest implements RuntimeTestSupport { */ public , V> LinkedHashMap sort(Map data) { LinkedHashMap dup = new LinkedHashMap(); - List keys = new ArrayList(); - keys.addAll(data.keySet()); + List keys = new ArrayList(data.keySet()); Collections.sort(keys); for (K k : keys) { dup.put(k, data.get(k)); diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java index 5bedebde2..1848cfa46 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java @@ -6,39 +6,14 @@ package org.antlr.v4.test.runtime.dart; -import org.antlr.v4.Tool; -import org.antlr.v4.analysis.AnalysisPipeline; -import org.antlr.v4.automata.ATNFactory; -import org.antlr.v4.automata.ATNPrinter; -import org.antlr.v4.automata.LexerATNFactory; -import org.antlr.v4.automata.ParserATNFactory; -import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.misc.Utils; -import org.antlr.v4.runtime.*; -import org.antlr.v4.runtime.atn.*; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.misc.IntegerList; -import org.antlr.v4.runtime.misc.Interval; -import org.antlr.v4.runtime.misc.Pair; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.semantics.SemanticPipeline; import org.antlr.v4.test.runtime.*; import org.antlr.v4.test.runtime.descriptors.LexerExecDescriptors; import org.antlr.v4.test.runtime.descriptors.PerformanceDescriptors; -import org.antlr.v4.tool.*; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; import org.stringtemplate.v4.ST; -import org.stringtemplate.v4.STGroup; -import org.stringtemplate.v4.STGroupString; import java.io.*; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.net.URL; -import java.net.URLClassLoader; import java.util.*; import static junit.framework.TestCase.*; @@ -47,352 +22,19 @@ import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; import static org.junit.Assert.assertArrayEquals; -public class BaseDartTest implements RuntimeTestSupport { +public class BaseDartTest extends BaseRuntimeTestSupport implements RuntimeTestSupport { + private static final List AOT_COMPILE_TESTS = Arrays.asList( new PerformanceDescriptors.DropLoopEntryBranchInLRRule_4().input, new LexerExecDescriptors.LargeLexer().input ); - public static final String newline = System.getProperty("line.separator"); - public static final String pathSep = System.getProperty("path.separator"); - - - /** - * When the {@code antlr.preserve-test-dir} runtime property is set to - * {@code true}, the temporary directories created by the test run will not - * be removed at the end of the test run, even for tests that completed - * successfully. - *

- *

- * The default behavior (used in all other cases) is removing the temporary - * directories for all tests which completed successfully, and preserving - * the directories for tests which failed.

- */ - public static final boolean PRESERVE_TEST_DIR = Boolean.parseBoolean(System.getProperty("antlr.preserve-test-dir", "false")); - - /** - * The base test directory is the directory where generated files get placed - * during unit test execution. - *

- *

- * The default value for this property is the {@code java.io.tmpdir} system - * property, and can be overridden by setting the - * {@code antlr.java-test-dir} property to a custom location. Note that the - * {@code antlr.java-test-dir} property directly affects the - * {@link #CREATE_PER_TEST_DIRECTORIES} value as well.

- */ - public static final String BASE_TEST_DIR; - - /** - * When {@code true}, a temporary directory will be created for each test - * executed during the test run. - *

- *

- * This value is {@code true} when the {@code antlr.java-test-dir} system - * property is set, and otherwise {@code false}.

- */ - public static final boolean CREATE_PER_TEST_DIRECTORIES; - - static { - String baseTestDir = System.getProperty("antlr.dart-test-dir"); - boolean perTestDirectories = false; - if (baseTestDir == null || baseTestDir.isEmpty()) { - baseTestDir = System.getProperty("java.io.tmpdir"); - perTestDirectories = true; - } - - if (!new File(baseTestDir).isDirectory()) { - throw new UnsupportedOperationException("The specified base test directory does not exist: " + baseTestDir); - } - - BASE_TEST_DIR = baseTestDir; - CREATE_PER_TEST_DIRECTORIES = perTestDirectories; - } - - /** - * Build up the full classpath we need, including the surefire path (if present) - */ - public static final String CLASSPATH = System.getProperty("java.class.path"); - - public String tmpdir = null; - - /** - * If error during parser execution, store stderr here; can't return - * stdout and stderr. This doesn't trap errors from running antlr. - */ - protected String stderrDuringParse; - - /** - * Errors found while running antlr - */ - protected StringBuilder antlrToolErrors; - private static String cacheDartPackages; - @org.junit.Rule - public final TestRule testWatcher = new TestWatcher() { - - @Override - protected void succeeded(Description description) { - // remove tmpdir if no error. - eraseTempDir(); - } - - }; - - private String getPropertyPrefix() { + public String getPropertyPrefix() { return "antlr-dart"; } - @Override - public void testSetUp() throws Exception { - if (CREATE_PER_TEST_DIRECTORIES) { - // new output dir for each test - String threadName = Thread.currentThread().getName(); - String testDirectory = getClass().getSimpleName() + "-" + threadName + "-" + System.nanoTime(); - tmpdir = new File(BASE_TEST_DIR, testDirectory).getAbsolutePath(); - } else { - tmpdir = new File(BASE_TEST_DIR).getAbsolutePath(); - if (!PRESERVE_TEST_DIR && new File(tmpdir).exists()) { - eraseFiles(); - } - } - antlrToolErrors = new StringBuilder(); - } - - @Override - public void testTearDown() throws Exception { - } - - @Override - public void beforeTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public void afterTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public String getTmpDir() { - return tmpdir; - } - - @Override - public String getStdout() { - return null; - } - - @Override - public String getParseErrors() { - return stderrDuringParse; - } - - @Override - public String getANTLRToolErrors() { - if (antlrToolErrors.length() == 0) { - return null; - } - return antlrToolErrors.toString(); - } - - protected Tool newTool(String[] args) { - Tool tool = new Tool(args); - return tool; - } - - protected ATN createATN(Grammar g, boolean useSerializer) { - if (g.atn == null) { - semanticProcess(g); - assertEquals(0, g.tool.getNumErrors()); - - ParserATNFactory f; - if (g.isLexer()) { - f = new LexerATNFactory((LexerGrammar) g); - } else { - f = new ParserATNFactory(g); - } - - g.atn = f.createATN(); - assertEquals(0, g.tool.getNumErrors()); - } - - ATN atn = g.atn; - if (useSerializer) { - char[] serialized = ATNSerializer.getSerializedAsChars(atn); - return new ATNDeserializer().deserialize(serialized); - } - - return atn; - } - - protected void semanticProcess(Grammar g) { - if (g.ast != null && !g.ast.hasErrors) { -// System.out.println(g.ast.toStringTree()); - Tool antlr = new Tool(); - SemanticPipeline sem = new SemanticPipeline(g); - sem.process(); - if (g.getImportedGrammars() != null) { // process imported grammars (if any) - for (Grammar imp : g.getImportedGrammars()) { - antlr.processNonCombinedGrammar(imp, false); - } - } - } - } - - public DFA createDFA(Grammar g, DecisionState s) { -// PredictionDFAFactory conv = new PredictionDFAFactory(g, s); -// DFA dfa = conv.createDFA(); -// conv.issueAmbiguityWarnings(); -// System.out.print("DFA="+dfa); -// return dfa; - return null; - } - -// public void minimizeDFA(DFA dfa) { -// DFAMinimizer dmin = new DFAMinimizer(dfa); -// dfa.minimized = dmin.minimize(); -// } - - IntegerList getTypesFromString(Grammar g, String expecting) { - IntegerList expectingTokenTypes = new IntegerList(); - if (expecting != null && !expecting.trim().isEmpty()) { - for (String tname : expecting.replace(" ", "").split(",")) { - int ttype = g.getTokenType(tname); - expectingTokenTypes.add(ttype); - } - } - return expectingTokenTypes; - } - - public IntegerList getTokenTypesViaATN(String input, LexerATNSimulator lexerATN) { - ANTLRInputStream in = new ANTLRInputStream(input); - IntegerList tokenTypes = new IntegerList(); - int ttype; - do { - ttype = lexerATN.match(in, Lexer.DEFAULT_MODE); - tokenTypes.add(ttype); - } while (ttype != Token.EOF); - return tokenTypes; - } - - public List getTokenTypes(LexerGrammar lg, - ATN atn, - CharStream input) { - LexerATNSimulator interp = new LexerATNSimulator(atn, new DFA[]{new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE))}, null); - List tokenTypes = new ArrayList(); - int ttype; - boolean hitEOF = false; - do { - if (hitEOF) { - tokenTypes.add("EOF"); - break; - } - int t = input.LA(1); - ttype = interp.match(input, Lexer.DEFAULT_MODE); - if (ttype == Token.EOF) { - tokenTypes.add("EOF"); - } else { - tokenTypes.add(lg.typeToTokenList.get(ttype)); - } - - if (t == IntStream.EOF) { - hitEOF = true; - } - } while (ttype != Token.EOF); - return tokenTypes; - } - - List checkRuleDFA(String gtext, String ruleName, String expecting) - throws Exception { - ErrorQueue equeue = new ErrorQueue(); - Grammar g = new Grammar(gtext, equeue); - ATN atn = createATN(g, false); - ATNState s = atn.ruleToStartState[g.getRule(ruleName).index]; - if (s == null) { - System.err.println("no such rule: " + ruleName); - return null; - } - ATNState t = s.transition(0).target; - if (!(t instanceof DecisionState)) { - System.out.println(ruleName + " has no decision"); - return null; - } - DecisionState blk = (DecisionState) t; - checkRuleDFA(g, blk, expecting); - return equeue.all; - } - - List checkRuleDFA(String gtext, int decision, String expecting) - throws Exception { - ErrorQueue equeue = new ErrorQueue(); - Grammar g = new Grammar(gtext, equeue); - ATN atn = createATN(g, false); - DecisionState blk = atn.decisionToState.get(decision); - checkRuleDFA(g, blk, expecting); - return equeue.all; - } - - void checkRuleDFA(Grammar g, DecisionState blk, String expecting) - throws Exception { - DFA dfa = createDFA(g, blk); - String result = null; - if (dfa != null) result = dfa.toString(); - assertEquals(expecting, result); - } - - List checkLexerDFA(String gtext, String expecting) - throws Exception { - return checkLexerDFA(gtext, LexerGrammar.DEFAULT_MODE_NAME, expecting); - } - - List checkLexerDFA(String gtext, String modeName, String expecting) - throws Exception { - ErrorQueue equeue = new ErrorQueue(); - LexerGrammar g = new LexerGrammar(gtext, equeue); - g.atn = createATN(g, false); -// LexerATNToDFAConverter conv = new LexerATNToDFAConverter(g); -// DFA dfa = conv.createDFA(modeName); -// g.setLookaheadDFA(0, dfa); // only one decision to worry about -// -// String result = null; -// if ( dfa!=null ) result = dfa.toString(); -// assertEquals(expecting, result); -// -// return equeue.all; - return null; - } - - protected String load(String fileName, String encoding) - throws IOException { - if (fileName == null) { - return null; - } - - String fullFileName = getClass().getPackage().getName().replace('.', '/') + '/' + fileName; - int size = 65000; - InputStreamReader isr; - InputStream fis = getClass().getClassLoader().getResourceAsStream(fullFileName); - if (encoding != null) { - isr = new InputStreamReader(fis, encoding); - } else { - isr = new InputStreamReader(fis); - } - try { - char[] data = new char[size]; - int n = isr.read(data); - return new String(data, 0, n); - } finally { - isr.close(); - } - } - - protected String execLexer(String grammarFileName, - String grammarStr, - String lexerName, - String input) { - return execLexer(grammarFileName, grammarStr, lexerName, input, false); - } - @Override public String execLexer(String grammarFileName, String grammarStr, @@ -404,71 +46,12 @@ public class BaseDartTest implements RuntimeTestSupport { null, lexerName); assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); writeLexerTestFile(lexerName, showDFA); String output = execClass("Test", AOT_COMPILE_TESTS.contains(input)); return output; } - public ParseTree execParser(String startRuleName, String input, - String parserName, String lexerName) - throws Exception { - Pair pl = getParserAndLexer(input, parserName, lexerName); - Parser parser = pl.a; - return execStartRule(startRuleName, parser); - } - - public ParseTree execStartRule(String startRuleName, Parser parser) - throws IllegalAccessException, InvocationTargetException, - NoSuchMethodException { - Method startRule = null; - Object[] args = null; - try { - startRule = parser.getClass().getMethod(startRuleName); - } catch (NoSuchMethodException nsme) { - // try with int _p arg for recursive func - startRule = parser.getClass().getMethod(startRuleName, int.class); - args = new Integer[]{0}; - } - ParseTree result = (ParseTree) startRule.invoke(parser, args); -// System.out.println("parse tree = "+result.toStringTree(parser)); - return result; - } - - public Pair getParserAndLexer(String input, - String parserName, String lexerName) - throws Exception { - final Class lexerClass = loadLexerClassFromTempDir(lexerName); - final Class parserClass = loadParserClassFromTempDir(parserName); - - ANTLRInputStream in = new ANTLRInputStream(new StringReader(input)); - - Class c = lexerClass.asSubclass(Lexer.class); - Constructor ctor = c.getConstructor(CharStream.class); - Lexer lexer = ctor.newInstance(in); - - Class pc = parserClass.asSubclass(Parser.class); - Constructor pctor = pc.getConstructor(TokenStream.class); - CommonTokenStream tokens = new CommonTokenStream(lexer); - Parser parser = pctor.newInstance(tokens); - return new Pair(parser, lexer); - } - - public Class loadClassFromTempDir(String name) throws Exception { - ClassLoader loader = - new URLClassLoader(new URL[]{new File(tmpdir).toURI().toURL()}, - ClassLoader.getSystemClassLoader()); - return loader.loadClass(name); - } - - public Class loadLexerClassFromTempDir(String name) throws Exception { - return loadClassFromTempDir(name).asSubclass(Lexer.class); - } - - public Class loadParserClassFromTempDir(String name) throws Exception { - return loadClassFromTempDir(name).asSubclass(Parser.class); - } - @Override public String execParser(String grammarFileName, String grammarStr, @@ -499,7 +82,7 @@ public class BaseDartTest implements RuntimeTestSupport { lexerName, "-visitor"); assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); return rawExecRecognizer(parserName, lexerName, startRuleName, @@ -529,7 +112,7 @@ public class BaseDartTest implements RuntimeTestSupport { boolean defaultListener, String... extraOptions) { ErrorQueue equeue = - BaseRuntimeTest.antlrOnString(getTmpDir(), "Dart", grammarFileName, grammarStr, defaultListener, extraOptions); + BaseRuntimeTest.antlrOnString(getTempDirPath(), "Dart", grammarFileName, grammarStr, defaultListener, extraOptions); if (!equeue.errors.isEmpty()) { return false; } @@ -553,14 +136,14 @@ public class BaseDartTest implements RuntimeTestSupport { } String runtime = locateRuntime(); - writeFile(tmpdir, "pubspec.yaml", + writeFile(getTempDirPath(), "pubspec.yaml", "name: \"test\"\n" + "dependencies:\n" + " antlr4:\n" + " path: " + runtime + "\n"); if (cacheDartPackages == null) { try { - final Process process = Runtime.getRuntime().exec(new String[]{locatePub(), "get"}, null, new File(tmpdir)); + final Process process = Runtime.getRuntime().exec(new String[]{locatePub(), "get"}, null, getTempTestDir()); StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); stderrVacuum.start(); Timer timer = new Timer(); @@ -585,9 +168,9 @@ public class BaseDartTest implements RuntimeTestSupport { e.printStackTrace(); return false; } - cacheDartPackages = readFile(tmpdir, ".packages"); + cacheDartPackages = readFile(getTempDirPath(), ".packages"); } else { - writeFile(tmpdir, ".packages", cacheDartPackages); + writeFile(getTempDirPath(), ".packages", cacheDartPackages); } return true; // allIsWell: no compile } @@ -598,7 +181,7 @@ public class BaseDartTest implements RuntimeTestSupport { boolean debug, boolean profile, boolean aotCompile) { - this.stderrDuringParse = null; + setParseErrors(null); if (parserName == null) { writeLexerTestFile(lexerName, false); } else { @@ -622,7 +205,7 @@ public class BaseDartTest implements RuntimeTestSupport { String cmdLine = Utils.join(args, " "); System.err.println("Compile: " + cmdLine); final Process process = - Runtime.getRuntime().exec(args, null, new File(tmpdir)); + Runtime.getRuntime().exec(args, null, getTempTestDir()); StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); stderrVacuum.start(); Timer timer = new Timer(); @@ -647,18 +230,18 @@ public class BaseDartTest implements RuntimeTestSupport { String[] args; if (compile) { args = new String[]{ - new File(tmpdir, className).getAbsolutePath(), new File(tmpdir, "input").getAbsolutePath() + new File(getTempTestDir(), className).getAbsolutePath(), new File(getTempTestDir(), "input").getAbsolutePath() }; } else { args = new String[]{ locateDart(), - className + ".dart", new File(tmpdir, "input").getAbsolutePath() + className + ".dart", new File(getTempTestDir(), "input").getAbsolutePath() }; } //String cmdLine = Utils.join(args, " "); //System.err.println("execParser: " + cmdLine); final Process process = - Runtime.getRuntime().exec(args, null, new File(tmpdir)); + Runtime.getRuntime().exec(args, null, getTempTestDir()); StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); stdoutVacuum.start(); @@ -683,7 +266,7 @@ public class BaseDartTest implements RuntimeTestSupport { output = null; } if (stderrVacuum.toString().length() > 0) { - this.stderrDuringParse = stderrVacuum.toString(); + setParseErrors(stderrVacuum.toString()); } return output; } catch (Exception e) { @@ -786,187 +369,6 @@ public class BaseDartTest implements RuntimeTestSupport { return runtimeSrc.getPath(); } - private boolean isWindows() { - return System.getProperty("os.name").toLowerCase().contains("windows"); - } - -// void ambig(List msgs, int[] expectedAmbigAlts, String expectedAmbigInput) -// throws Exception -// { -// ambig(msgs, 0, expectedAmbigAlts, expectedAmbigInput); -// } - -// void ambig(List msgs, int i, int[] expectedAmbigAlts, String expectedAmbigInput) -// throws Exception -// { -// List amsgs = getMessagesOfType(msgs, AmbiguityMessage.class); -// AmbiguityMessage a = (AmbiguityMessage)amsgs.get(i); -// if ( a==null ) assertNull(expectedAmbigAlts); -// else { -// assertEquals(a.conflictingAlts.toString(), Arrays.toString(expectedAmbigAlts)); -// } -// assertEquals(expectedAmbigInput, a.input); -// } - -// void unreachable(List msgs, int[] expectedUnreachableAlts) -// throws Exception -// { -// unreachable(msgs, 0, expectedUnreachableAlts); -// } - -// void unreachable(List msgs, int i, int[] expectedUnreachableAlts) -// throws Exception -// { -// List amsgs = getMessagesOfType(msgs, UnreachableAltsMessage.class); -// UnreachableAltsMessage u = (UnreachableAltsMessage)amsgs.get(i); -// if ( u==null ) assertNull(expectedUnreachableAlts); -// else { -// assertEquals(u.conflictingAlts.toString(), Arrays.toString(expectedUnreachableAlts)); -// } -// } - - List getMessagesOfType(List msgs, Class c) { - List filtered = new ArrayList(); - for (ANTLRMessage m : msgs) { - if (m.getClass() == c) filtered.add(m); - } - return filtered; - } - - public void checkRuleATN(Grammar g, String ruleName, String expecting) { -// DOTGenerator dot = new DOTGenerator(g); -// System.out.println(dot.getDOT(g.atn.ruleToStartState[g.getRule(ruleName).index])); - - Rule r = g.getRule(ruleName); - ATNState startState = g.getATN().ruleToStartState[r.index]; - ATNPrinter serializer = new ATNPrinter(g, startState); - String result = serializer.asString(); - - //System.out.print(result); - assertEquals(expecting, result); - } - - public void testActions(String templates, String actionName, String action, String expected) throws org.antlr.runtime.RecognitionException { - int lp = templates.indexOf('('); - String name = templates.substring(0, lp); - STGroup group = new STGroupString(templates); - ST st = group.getInstanceOf(name); - st.add(actionName, action); - String grammar = st.render(); - ErrorQueue equeue = new ErrorQueue(); - Grammar g = new Grammar(grammar, equeue); - if (g.ast != null && !g.ast.hasErrors) { - SemanticPipeline sem = new SemanticPipeline(g); - sem.process(); - - ATNFactory factory = new ParserATNFactory(g); - if (g.isLexer()) factory = new LexerATNFactory((LexerGrammar) g); - g.atn = factory.createATN(); - - AnalysisPipeline anal = new AnalysisPipeline(g); - anal.process(); - - CodeGenerator gen = new CodeGenerator(g); - ST outputFileST = gen.generateParser(false); - String output = outputFileST.render(); - //System.out.println(output); - String b = "#" + actionName + "#"; - int start = output.indexOf(b); - String e = "#end-" + actionName + "#"; - int end = output.indexOf(e); - String snippet = output.substring(start + b.length(), end); - assertEquals(expected, snippet); - } - if (equeue.size() > 0) { -// System.err.println(equeue.toString()); - } - } - - protected void checkGrammarSemanticsError(ErrorQueue equeue, - GrammarSemanticsMessage expectedMessage) - throws Exception { - ANTLRMessage foundMsg = null; - for (int i = 0; i < equeue.errors.size(); i++) { - ANTLRMessage m = equeue.errors.get(i); - if (m.getErrorType() == expectedMessage.getErrorType()) { - foundMsg = m; - } - } - assertNotNull("no error; " + expectedMessage.getErrorType() + " expected", foundMsg); - assertTrue("error is not a GrammarSemanticsMessage", - foundMsg instanceof GrammarSemanticsMessage); - assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs())); - if (equeue.size() != 1) { - System.err.println(equeue); - } - } - - protected void checkGrammarSemanticsWarning(ErrorQueue equeue, - GrammarSemanticsMessage expectedMessage) - throws Exception { - ANTLRMessage foundMsg = null; - for (int i = 0; i < equeue.warnings.size(); i++) { - ANTLRMessage m = equeue.warnings.get(i); - if (m.getErrorType() == expectedMessage.getErrorType()) { - foundMsg = m; - } - } - assertNotNull("no error; " + expectedMessage.getErrorType() + " expected", foundMsg); - assertTrue("error is not a GrammarSemanticsMessage", - foundMsg instanceof GrammarSemanticsMessage); - assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs())); - if (equeue.size() != 1) { - System.err.println(equeue); - } - } - - protected void checkError(ErrorQueue equeue, - ANTLRMessage expectedMessage) - throws Exception { - //System.out.println("errors="+equeue); - ANTLRMessage foundMsg = null; - for (int i = 0; i < equeue.errors.size(); i++) { - ANTLRMessage m = equeue.errors.get(i); - if (m.getErrorType() == expectedMessage.getErrorType()) { - foundMsg = m; - } - } - assertTrue("no error; " + expectedMessage.getErrorType() + " expected", !equeue.errors.isEmpty()); - assertTrue("too many errors; " + equeue.errors, equeue.errors.size() <= 1); - assertNotNull("couldn't find expected error: " + expectedMessage.getErrorType(), foundMsg); - /* - * assertTrue("error is not a GrammarSemanticsMessage", foundMsg - * instanceof GrammarSemanticsMessage); - */ - assertArrayEquals(expectedMessage.getArgs(), foundMsg.getArgs()); - } - - public static class FilteringTokenStream extends CommonTokenStream { - public FilteringTokenStream(TokenSource src) { - super(src); - } - - Set hide = new HashSet(); - - @Override - protected boolean sync(int i) { - if (!super.sync(i)) { - return false; - } - - Token t = get(i); - if (hide.contains(t.getType())) { - ((WritableToken) t).setChannel(Token.HIDDEN_CHANNEL); - } - - return true; - } - - public void setTokenTypeChannel(int ttype, int channel) { - hide.add(ttype); - } - } - protected void writeTestFile(String parserName, String lexerName, String parserStartRuleName, @@ -1026,7 +428,7 @@ public class BaseDartTest implements RuntimeTestSupport { outputFileST.add("parserName", parserName); outputFileST.add("lexerName", lexerName); outputFileST.add("parserStartRuleName", parserStartRuleName); - writeFile(tmpdir, "Test.dart", outputFileST.render()); + writeFile(getTempDirPath(), "Test.dart", outputFileST.render()); } protected void writeLexerTestFile(String lexerName, boolean showDFA) { @@ -1050,197 +452,7 @@ public class BaseDartTest implements RuntimeTestSupport { ); outputFileST.add("lexerName", lexerName); - writeFile(tmpdir, "Test.dart", outputFileST.render()); + writeFile(getTempDirPath(), "Test.dart", outputFileST.render()); } - protected void eraseFiles() { - if (tmpdir == null) { - return; - } - - File tmpdirF = new File(tmpdir); - String[] files = tmpdirF.list(); - for (int i = 0; files != null && i < files.length; i++) { - new File(tmpdir + "/" + files[i]).delete(); - } - } - - @Override - public void eraseTempDir() { - if(shouldEraseTempDir()) { - File tmpdirF = new File(tmpdir); - if (tmpdirF.exists()) { - eraseFiles(); - tmpdirF.delete(); - } - } - } - - private boolean shouldEraseTempDir() { - return tmpdir!=null; - } - - public String getFirstLineOfException() { - if (this.stderrDuringParse == null) { - return null; - } - String[] lines = this.stderrDuringParse.split("\n"); - String prefix = "Exception in thread \"main\" "; - return lines[0].substring(prefix.length(), lines[0].length()); - } - - /** - * When looking at a result set that consists of a Map/HashTable - * we cannot rely on the output order, as the hashing algorithm or other aspects - * of the implementation may be different on differnt JDKs or platforms. Hence - * we take the Map, convert the keys to a List, sort them and Stringify the Map, which is a - * bit of a hack, but guarantees that we get the same order on all systems. We assume that - * the keys are strings. - * - * @param m The Map that contains keys we wish to return in sorted order - * @return A string that represents all the keys in sorted order. - */ - public String sortMapToString(Map m) { - // Pass in crap, and get nothing back - // - if (m == null) { - return null; - } - - System.out.println("Map toString looks like: " + m.toString()); - - // Sort the keys in the Map - // - TreeMap nset = new TreeMap(m); - - System.out.println("Tree map looks like: " + nset.toString()); - return nset.toString(); - } - - public List realElements(List elements) { - return elements.subList(Token.MIN_USER_TOKEN_TYPE, elements.size()); - } - - public void assertNotNullOrEmpty(String message, String text) { - assertNotNull(message, text); - assertFalse(message, text.isEmpty()); - } - - public void assertNotNullOrEmpty(String text) { - assertNotNull(text); - assertFalse(text.isEmpty()); - } - - public static class IntTokenStream implements TokenStream { - public IntegerList types; - int p = 0; - - public IntTokenStream(IntegerList types) { - this.types = types; - } - - @Override - public void consume() { - p++; - } - - @Override - public int LA(int i) { - return LT(i).getType(); - } - - @Override - public int mark() { - return index(); - } - - @Override - public int index() { - return p; - } - - @Override - public void release(int marker) { - seek(marker); - } - - @Override - public void seek(int index) { - p = index; - } - - @Override - public int size() { - return types.size(); - } - - @Override - public String getSourceName() { - return UNKNOWN_SOURCE_NAME; - } - - @Override - public Token LT(int i) { - CommonToken t; - int rawIndex = p + i - 1; - if (rawIndex >= types.size()) t = new CommonToken(Token.EOF); - else t = new CommonToken(types.get(rawIndex)); - t.setTokenIndex(rawIndex); - return t; - } - - @Override - public Token get(int i) { - return new CommonToken(types.get(i)); - } - - @Override - public TokenSource getTokenSource() { - return null; - } - - @Override - public String getText() { - throw new UnsupportedOperationException("can't give strings"); - } - - @Override - public String getText(Interval interval) { - throw new UnsupportedOperationException("can't give strings"); - } - - @Override - public String getText(RuleContext ctx) { - throw new UnsupportedOperationException("can't give strings"); - } - - @Override - public String getText(Token start, Token stop) { - throw new UnsupportedOperationException("can't give strings"); - } - } - - /** - * Sort a list - */ - public > List sort(List data) { - List dup = new ArrayList(); - dup.addAll(data); - Collections.sort(dup); - return dup; - } - - /** - * Return map sorted by key - */ - public , V> LinkedHashMap sort(Map data) { - LinkedHashMap dup = new LinkedHashMap(); - List keys = new ArrayList(); - keys.addAll(data.keySet()); - Collections.sort(keys); - for (K k : keys) { - dup.put(k, data.get(k)); - } - return dup; - } } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java index 1696d5307..1ff93d1fa 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java @@ -5,107 +5,36 @@ */ package org.antlr.v4.test.runtime.go; -import org.antlr.v4.Tool; -import org.antlr.v4.automata.ATNFactory; -import org.antlr.v4.automata.ATNPrinter; -import org.antlr.v4.automata.LexerATNFactory; -import org.antlr.v4.automata.ParserATNFactory; -import org.antlr.v4.codegen.CodeGenerator; -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CommonToken; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.IntStream; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.RuleContext; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenSource; -import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.WritableToken; -import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.runtime.atn.ATNDeserializer; -import org.antlr.v4.runtime.atn.ATNSerializer; -import org.antlr.v4.runtime.atn.ATNState; -import org.antlr.v4.runtime.atn.LexerATNSimulator; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.misc.IntegerList; -import org.antlr.v4.runtime.misc.Interval; -import org.antlr.v4.semantics.SemanticPipeline; -import org.antlr.v4.test.runtime.ErrorQueue; -import org.antlr.v4.test.runtime.RuntimeTestDescriptor; -import org.antlr.v4.test.runtime.RuntimeTestSupport; -import org.antlr.v4.test.runtime.StreamVacuum; -import org.antlr.v4.tool.ANTLRMessage; -import org.antlr.v4.tool.DOTGenerator; -import org.antlr.v4.tool.Grammar; -import org.antlr.v4.tool.GrammarSemanticsMessage; -import org.antlr.v4.tool.LexerGrammar; -import org.antlr.v4.tool.Rule; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; + +import org.antlr.v4.test.runtime.*; import org.stringtemplate.v4.ST; -import org.stringtemplate.v4.STGroup; -import org.stringtemplate.v4.STGroupString; import java.io.File; -import java.io.FileFilter; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import static junit.framework.TestCase.assertEquals; -import static junit.framework.TestCase.assertFalse; -import static junit.framework.TestCase.assertNotNull; -import static junit.framework.TestCase.assertTrue; +import static junit.framework.TestCase.*; import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString; import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; -import static org.junit.Assert.assertArrayEquals; -public class BaseGoTest implements RuntimeTestSupport { - public File overall_tmpdir = null; - public File tmpdir = null; // this is where the parser package is stored, typically inside the tmpdir +public class BaseGoTest extends BaseRuntimeTestSupport implements RuntimeTestSupport { + private static File tmpGopath = null; private static final String GO_RUNTIME_IMPORT_PATH = "github.com/antlr/antlr4/runtime/Go/antlr"; // TODO: Change this before merging with upstream - /** - * If error during parser execution, store stderr here; can't return stdout - * and stderr. This doesn't trap errors from running antlr. - */ - protected String stderrDuringParse; + private File parserTempDir; // "parser" with tempDir - /** Errors found while running antlr */ - protected StringBuilder antlrToolErrors; - - @org.junit.Rule - public final TestRule testWatcher = new TestWatcher() { - - @Override - protected void succeeded(Description description) { - // remove tmpdir if no error. - eraseTempDir(); - } - - }; + @Override + protected String getPropertyPrefix() { + return "antlr4-go"; + } /** * Copies all files from go runtime to a temporary folder that is inside a valid GOPATH project structure. */ public static void groupSetUp() throws Exception { - tmpGopath = new File(System.getProperty("java.io.tmpdir"), "antlr-goruntime-tmpgopath-" - + Long.toHexString(System.currentTimeMillis())); + tmpGopath = new File(System.getProperty("java.io.tmpdir"), "antlr-goruntime-tmpgopath-" + Long.toHexString(System.currentTimeMillis())); ArrayList pathsegments = new ArrayList(); pathsegments.add("src"); @@ -125,47 +54,12 @@ public class BaseGoTest implements RuntimeTestSupport { } for (File runtimeFile : runtimeFiles) { File dest = new File(tmpPackageDir, runtimeFile.getName()); - copyFile(runtimeFile, dest); + RuntimeTestUtils.copyFile(runtimeFile, dest); } cacheGoRuntime(tmpPackageDir); } - @Override - public void testTearDown() throws Exception { - } - - @Override - public void beforeTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public void afterTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public String getTmpDir() { - return tmpdir.getPath(); - } - - @Override - public String getStdout() { - return null; - } - - @Override - public String getParseErrors() { - return stderrDuringParse; - } - - @Override - public String getANTLRToolErrors() { - if ( antlrToolErrors.length()==0 ) { - return null; - } - return antlrToolErrors.toString(); - } - public static void groupTearDown() throws Exception { eraseDirectory(tmpGopath); } @@ -186,139 +80,22 @@ public class BaseGoTest implements RuntimeTestSupport { } } - private static void copyFile(File source, File dest) throws IOException { - InputStream is = new FileInputStream(source); - OutputStream os = new FileOutputStream(dest); - byte[] buf = new byte[4 << 10]; - int l; - while ((l = is.read(buf)) > -1) { - os.write(buf, 0, l); - } - is.close(); - os.close(); - } - public void testSetUp() throws Exception { - // new output dir for each test - String prop = System.getProperty("antlr-go-test-dir"); - if (prop != null && prop.length() > 0) { - overall_tmpdir = new File(prop); - } - else { - String threadName = Thread.currentThread().getName(); - overall_tmpdir = new File(System.getProperty("java.io.tmpdir"), - getClass().getSimpleName()+"-"+threadName+"-"+System.currentTimeMillis()); - } - - if ( overall_tmpdir.exists()) - this.eraseDirectory(overall_tmpdir); - - tmpdir = new File(overall_tmpdir, "parser"); - - if ( tmpdir.exists()) { - this.eraseDirectory(tmpdir); - } - antlrToolErrors = new StringBuilder(); + eraseParserTempDir(); + super.testSetUp(); + parserTempDir = new File(getTempTestDir(), "parser"); } - protected org.antlr.v4.Tool newTool(String[] args) { - return new Tool(args); + @Override + public File getTempParserDir() { + return parserTempDir; } - protected Tool newTool() { - return new Tool(new String[]{"-o", tmpdir.getPath()}); - } - - protected ATN createATN(Grammar g, boolean useSerializer) { - if (g.atn == null) { - semanticProcess(g); - assertEquals(0, g.tool.getNumErrors()); - - ParserATNFactory f; - if (g.isLexer()) { - f = new LexerATNFactory((LexerGrammar) g); - } - else { - f = new ParserATNFactory(g); - } - - g.atn = f.createATN(); - assertEquals(0, g.tool.getNumErrors()); + private void eraseParserTempDir() { + if(parserTempDir != null) { + eraseDirectory(parserTempDir); + parserTempDir = null; } - - ATN atn = g.atn; - if (useSerializer) { - char[] serialized = ATNSerializer.getSerializedAsChars(atn); - return new ATNDeserializer().deserialize(serialized); - } - - return atn; - } - - protected void semanticProcess(Grammar g) { - if (g.ast != null && !g.ast.hasErrors) { - System.out.println(g.ast.toStringTree()); - Tool antlr = new Tool(); - SemanticPipeline sem = new SemanticPipeline(g); - sem.process(); - if (g.getImportedGrammars() != null) { // process imported grammars - // (if any) - for (Grammar imp : g.getImportedGrammars()) { - antlr.processNonCombinedGrammar(imp, false); - } - } - } - } - - IntegerList getTypesFromString(Grammar g, String expecting) { - IntegerList expectingTokenTypes = new IntegerList(); - if (expecting != null && !expecting.trim().isEmpty()) { - for (String tname : expecting.replace(" ", "").split(",")) { - int ttype = g.getTokenType(tname); - expectingTokenTypes.add(ttype); - } - } - return expectingTokenTypes; - } - - public IntegerList getTokenTypesViaATN(String input, - LexerATNSimulator lexerATN) { - ANTLRInputStream in = new ANTLRInputStream(input); - IntegerList tokenTypes = new IntegerList(); - int ttype; - do { - ttype = lexerATN.match(in, Lexer.DEFAULT_MODE); - tokenTypes.add(ttype); - } while (ttype != Token.EOF); - return tokenTypes; - } - - public List getTokenTypes(LexerGrammar lg, ATN atn, CharStream input) { - LexerATNSimulator interp = new LexerATNSimulator(atn, - new DFA[] { new DFA( - atn.modeToStartState.get(Lexer.DEFAULT_MODE)) }, null); - List tokenTypes = new ArrayList(); - int ttype; - boolean hitEOF = false; - do { - if (hitEOF) { - tokenTypes.add("EOF"); - break; - } - int t = input.LA(1); - ttype = interp.match(input, Lexer.DEFAULT_MODE); - if (ttype == Token.EOF) { - tokenTypes.add("EOF"); - } - else { - tokenTypes.add(lg.typeToTokenList.get(ttype)); - } - - if (t == IntStream.EOF) { - hitEOF = true; - } - } while (ttype != Token.EOF); - return tokenTypes; } protected String execLexer(String grammarFileName, String grammarStr, @@ -332,25 +109,10 @@ public class BaseGoTest implements RuntimeTestSupport { boolean success = rawGenerateAndBuildRecognizer(grammarFileName, grammarStr, null, lexerName, "-no-listener"); assertTrue(success); - writeFile(overall_tmpdir.toString(), "input", input); + writeFile(getTempDirPath(), "input", input); writeLexerTestFile(lexerName, showDFA); - String output = execModule("Test.go"); - return output; + return execModule("Test.go"); } -// -// public String execParser(String grammarFileName, String grammarStr, -// String parserName, String lexerName, String listenerName, -// String visitorName, String startRuleName, String input, -// boolean debug) -// { -// boolean success = rawGenerateAndBuildRecognizer(grammarFileName, -// grammarStr, parserName, lexerName, "-visitor"); -// assertTrue(success); -// writeFile(overall_tmpdir, "input", input); -// rawBuildRecognizerTestFile(parserName, lexerName, listenerName, -// visitorName, startRuleName, debug); -// return execRecognizer(); -// } @Override public String execParser(String grammarFileName, String grammarStr, @@ -361,7 +123,7 @@ public class BaseGoTest implements RuntimeTestSupport { boolean success = rawGenerateAndBuildRecognizer(grammarFileName, grammarStr, parserName, lexerName, "-visitor"); assertTrue(success); - writeFile(overall_tmpdir.toString(), "input", input); + writeFile(getTempDirPath(), "input", input); rawBuildRecognizerTestFile(parserName, lexerName, listenerName, visitorName, startRuleName, showDiagnosticErrors); return execRecognizer(); @@ -379,7 +141,7 @@ public class BaseGoTest implements RuntimeTestSupport { protected boolean rawGenerateAndBuildRecognizer(String grammarFileName, String grammarStr, String parserName, String lexerName, boolean defaultListener, String... extraOptions) { - ErrorQueue equeue = antlrOnString(getTmpDir(), "Go", grammarFileName, grammarStr, + ErrorQueue equeue = antlrOnString(getTempParserDirPath(), "Go", grammarFileName, grammarStr, defaultListener, extraOptions); if (!equeue.errors.isEmpty()) { return false; @@ -390,7 +152,7 @@ public class BaseGoTest implements RuntimeTestSupport { protected void rawBuildRecognizerTestFile(String parserName, String lexerName, String listenerName, String visitorName, String parserStartRuleName, boolean debug) { - this.stderrDuringParse = null; + setParseErrors(null); if (parserName == null) { writeLexerTestFile(lexerName, false); } @@ -406,12 +168,12 @@ public class BaseGoTest implements RuntimeTestSupport { public String execModule(String fileName) { String goExecutable = locateGo(); - String modulePath = new File(overall_tmpdir, fileName).getAbsolutePath(); - String inputPath = new File(overall_tmpdir, "input").getAbsolutePath(); + String modulePath = new File(getTempTestDir(), fileName).getAbsolutePath(); + String inputPath = new File(getTempTestDir(), "input").getAbsolutePath(); try { ProcessBuilder builder = new ProcessBuilder(goExecutable, "run", modulePath, inputPath); builder.environment().put("GOPATH", tmpGopath.getPath()); - builder.directory(overall_tmpdir); + builder.directory(getTempTestDir()); Process process = builder.start(); StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); @@ -425,7 +187,7 @@ public class BaseGoTest implements RuntimeTestSupport { output = null; } if (stderrVacuum.toString().length() > 0) { - this.stderrDuringParse = stderrVacuum.toString(); + setParseErrors(stderrVacuum.toString()); } return output; } @@ -492,203 +254,6 @@ public class BaseGoTest implements RuntimeTestSupport { return runtimeDir; } - // void ambig(List msgs, int[] expectedAmbigAlts, String - // expectedAmbigInput) - // throws Exception - // { - // ambig(msgs, 0, expectedAmbigAlts, expectedAmbigInput); - // } - - // void ambig(List msgs, int i, int[] expectedAmbigAlts, String - // expectedAmbigInput) - // throws Exception - // { - // List amsgs = getMessagesOfType(msgs, AmbiguityMessage.class); - // AmbiguityMessage a = (AmbiguityMessage)amsgs.get(i); - // if ( a==null ) assertNull(expectedAmbigAlts); - // else { - // assertEquals(a.conflictingAlts.toString(), - // Arrays.toString(expectedAmbigAlts)); - // } - // assertEquals(expectedAmbigInput, a.input); - // } - - // void unreachable(List msgs, int[] expectedUnreachableAlts) - // throws Exception - // { - // unreachable(msgs, 0, expectedUnreachableAlts); - // } - - // void unreachable(List msgs, int i, int[] - // expectedUnreachableAlts) - // throws Exception - // { - // List amsgs = getMessagesOfType(msgs, - // UnreachableAltsMessage.class); - // UnreachableAltsMessage u = (UnreachableAltsMessage)amsgs.get(i); - // if ( u==null ) assertNull(expectedUnreachableAlts); - // else { - // assertEquals(u.conflictingAlts.toString(), - // Arrays.toString(expectedUnreachableAlts)); - // } - // } - - List getMessagesOfType(List msgs, - Class c) { - List filtered = new ArrayList(); - for (ANTLRMessage m : msgs) { - if (m.getClass() == c) - filtered.add(m); - } - return filtered; - } - - void checkRuleATN(Grammar g, String ruleName, String expecting) { - ParserATNFactory f = new ParserATNFactory(g); - ATN atn = f.createATN(); - - DOTGenerator dot = new DOTGenerator(g); - System.out - .println(dot.getDOT(atn.ruleToStartState[g.getRule(ruleName).index])); - - Rule r = g.getRule(ruleName); - ATNState startState = atn.ruleToStartState[r.index]; - ATNPrinter serializer = new ATNPrinter(g, startState); - String result = serializer.asString(); - - // System.out.print(result); - assertEquals(expecting, result); - } - - public void testActions(String templates, String actionName, String action, - String expected) throws org.antlr.runtime.RecognitionException { - int lp = templates.indexOf('('); - String name = templates.substring(0, lp); - STGroup group = new STGroupString(templates); - ST st = group.getInstanceOf(name); - st.add(actionName, action); - String grammar = st.render(); - ErrorQueue equeue = new ErrorQueue(); - Grammar g = new Grammar(grammar, equeue); - if (g.ast != null && !g.ast.hasErrors) { - SemanticPipeline sem = new SemanticPipeline(g); - sem.process(); - - ATNFactory factory = new ParserATNFactory(g); - if (g.isLexer()) - factory = new LexerATNFactory((LexerGrammar) g); - g.atn = factory.createATN(); - - CodeGenerator gen = new CodeGenerator(g); - ST outputFileST = gen.generateParser(); - String output = outputFileST.render(); - // System.out.println(output); - String b = "#" + actionName + "#"; - int start = output.indexOf(b); - String e = "#end-" + actionName + "#"; - int end = output.indexOf(e); - String snippet = output.substring(start + b.length(), end); - assertEquals(expected, snippet); - } - if (equeue.size() > 0) { - System.err.println(equeue.toString()); - } - } - - protected void checkGrammarSemanticsError(ErrorQueue equeue, - GrammarSemanticsMessage expectedMessage) throws Exception { - ANTLRMessage foundMsg = null; - for (int i = 0; i < equeue.errors.size(); i++) { - ANTLRMessage m = equeue.errors.get(i); - if (m.getErrorType() == expectedMessage.getErrorType()) { - foundMsg = m; - } - } - assertNotNull("no error; " + expectedMessage.getErrorType() - + " expected", foundMsg); - assertTrue("error is not a GrammarSemanticsMessage", - foundMsg instanceof GrammarSemanticsMessage); - assertEquals(Arrays.toString(expectedMessage.getArgs()), - Arrays.toString(foundMsg.getArgs())); - if (equeue.size() != 1) { - System.err.println(equeue); - } - } - - protected void checkGrammarSemanticsWarning(ErrorQueue equeue, - GrammarSemanticsMessage expectedMessage) throws Exception { - ANTLRMessage foundMsg = null; - for (int i = 0; i < equeue.warnings.size(); i++) { - ANTLRMessage m = equeue.warnings.get(i); - if (m.getErrorType() == expectedMessage.getErrorType()) { - foundMsg = m; - } - } - assertNotNull("no error; " + expectedMessage.getErrorType() - + " expected", foundMsg); - assertTrue("error is not a GrammarSemanticsMessage", - foundMsg instanceof GrammarSemanticsMessage); - assertEquals(Arrays.toString(expectedMessage.getArgs()), - Arrays.toString(foundMsg.getArgs())); - if (equeue.size() != 1) { - System.err.println(equeue); - } - } - - protected void checkError(ErrorQueue equeue, ANTLRMessage expectedMessage) - throws Exception { - // System.out.println("errors="+equeue); - ANTLRMessage foundMsg = null; - for (int i = 0; i < equeue.errors.size(); i++) { - ANTLRMessage m = equeue.errors.get(i); - if (m.getErrorType() == expectedMessage.getErrorType()) { - foundMsg = m; - } - } - assertTrue("no error; " + expectedMessage.getErrorType() + " expected", - !equeue.errors.isEmpty()); - assertTrue("too many errors; " + equeue.errors, - equeue.errors.size() <= 1); - assertNotNull( - "couldn't find expected error: " - + expectedMessage.getErrorType(), foundMsg); - /* - * assertTrue("error is not a GrammarSemanticsMessage", foundMsg - * instanceof GrammarSemanticsMessage); - */ - assertArrayEquals(expectedMessage.getArgs(), foundMsg.getArgs()); - } - - public static class FilteringTokenStream extends CommonTokenStream { - public FilteringTokenStream(TokenSource src) { - super(src); - } - - Set hide = new HashSet(); - - @Override - protected boolean sync(int i) { - if (!super.sync(i)) { - return false; - } - - Token t = get(i); - if (hide.contains(t.getType())) { - ((WritableToken) t).setChannel(Token.HIDDEN_CHANNEL); - } - - return true; - } - - public void setTokenTypeChannel(int ttype, int channel) { - hide.add(ttype); - } - } - - protected void mkdir(File dir) { - dir.mkdirs(); - } - protected void writeParserTestFile(String parserName, String lexerName, String listenerName, String visitorName, String parserStartRuleName, boolean debug) { @@ -746,7 +311,7 @@ public class BaseGoTest implements RuntimeTestSupport { outputFileST.add("listenerName", listenerName); outputFileST.add("visitorName", visitorName); outputFileST.add("parserStartRuleName", parserStartRuleName.substring(0, 1).toUpperCase() + parserStartRuleName.substring(1) ); - writeFile(overall_tmpdir.toString(), "Test.go", outputFileST.render()); + writeFile(getTempDirPath(), "Test.go", outputFileST.render()); } @@ -778,228 +343,7 @@ public class BaseGoTest implements RuntimeTestSupport { + "}\n" + "\n"); outputFileST.add("lexerName", lexerName); - writeFile(overall_tmpdir.toString(), "Test.go", outputFileST.render()); + writeFile(getTempDirPath(), "Test.go", outputFileST.render()); } - public void writeRecognizer(String parserName, String lexerName, - String listenerName, String visitorName, - String parserStartRuleName, boolean debug) { - if (parserName == null) { - writeLexerTestFile(lexerName, debug); - } - else { - writeParserTestFile(parserName, lexerName, listenerName, - visitorName, parserStartRuleName, debug); - } - } - - protected void eraseFilesEndingWith(final String filesEndingWith) { - File[] files = overall_tmpdir.listFiles(new FileFilter() { - @Override - public boolean accept(File pathname) { - return pathname.getName().endsWith(filesEndingWith); - } - }); - for (File file : files) { - file.delete(); - } - } - - protected static void eraseDirectory(File dir) { - File[] files = dir.listFiles(); - if (files != null) { - for (File file : files) { - if (file.isDirectory()) { - eraseDirectory(file); - } - else { - file.delete(); - } - } - } - dir.delete(); - } - - public void eraseTempDir() { - if (shouldEraseTempDir()) { - if ( overall_tmpdir.exists()) { - eraseDirectory(overall_tmpdir); - } - } - } - - private boolean shouldEraseTempDir() { - if(overall_tmpdir==null) - return false; - String propName = "antlr-go-erase-test-dir"; - String prop = System.getProperty(propName); - if (prop != null && prop.length() > 0) - return Boolean.getBoolean(prop); - else - return true; - } - - public String getFirstLineOfException() { - if (this.stderrDuringParse == null) { - return null; - } - String[] lines = this.stderrDuringParse.split("\n"); - String prefix = "Exception in thread \"main\" "; - return lines[0].substring(prefix.length(), lines[0].length()); - } - - /** - * When looking at a result set that consists of a Map/HashTable we cannot - * rely on the output order, as the hashing algorithm or other aspects of - * the implementation may be different on differnt JDKs or platforms. Hence - * we take the Map, convert the keys to a List, sort them and Stringify the - * Map, which is a bit of a hack, but guarantees that we get the same order - * on all systems. We assume that the keys are strings. - * - * @param m - * The Map that contains keys we wish to return in sorted order - * @return A string that represents all the keys in sorted order. - */ - public String sortMapToString(Map m) { - // Pass in crap, and get nothing back - // - if (m == null) { - return null; - } - - System.out.println("Map toString looks like: " + m.toString()); - - // Sort the keys in the Map - // - TreeMap nset = new TreeMap(m); - - System.out.println("Tree map looks like: " + nset.toString()); - return nset.toString(); - } - - public List realElements(List elements) { - return elements.subList(Token.MIN_USER_TOKEN_TYPE, elements.size()); - } - - public void assertNotNullOrEmpty(String message, String text) { - assertNotNull(message, text); - assertFalse(message, text.isEmpty()); - } - - public void assertNotNullOrEmpty(String text) { - assertNotNull(text); - assertFalse(text.isEmpty()); - } - - public static class IntTokenStream implements TokenStream { - IntegerList types; - int p = 0; - - public IntTokenStream(IntegerList types) { - this.types = types; - } - - @Override - public void consume() { - p++; - } - - @Override - public int LA(int i) { - return LT(i).getType(); - } - - @Override - public int mark() { - return index(); - } - - @Override - public int index() { - return p; - } - - @Override - public void release(int marker) { - seek(marker); - } - - @Override - public void seek(int index) { - p = index; - } - - @Override - public int size() { - return types.size(); - } - - @Override - public String getSourceName() { - return null; - } - - @Override - public Token LT(int i) { - CommonToken t; - int rawIndex = p + i - 1; - if (rawIndex >= types.size()) - t = new CommonToken(Token.EOF); - else - t = new CommonToken(types.get(rawIndex)); - t.setTokenIndex(rawIndex); - return t; - } - - @Override - public Token get(int i) { - return new org.antlr.v4.runtime.CommonToken(types.get(i)); - } - - @Override - public TokenSource getTokenSource() { - return null; - } - - @Override - public String getText() { - throw new UnsupportedOperationException("can't give strings"); - } - - @Override - public String getText(Interval interval) { - throw new UnsupportedOperationException("can't give strings"); - } - - @Override - public String getText(RuleContext ctx) { - throw new UnsupportedOperationException("can't give strings"); - } - - @Override - public String getText(Token start, Token stop) { - throw new UnsupportedOperationException("can't give strings"); - } - } - - /** Sort a list */ - public > List sort(List data) { - List dup = new ArrayList(); - dup.addAll(data); - Collections.sort(dup); - return dup; - } - - /** Return map sorted by key */ - public , V> LinkedHashMap sort( - Map data) { - LinkedHashMap dup = new LinkedHashMap(); - List keys = new ArrayList(); - keys.addAll(data.keySet()); - Collections.sort(keys); - for (K k : keys) { - dup.put(k, data.get(k)); - } - return dup; - } } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java index 606cdede2..cc03bdc11 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java @@ -5,7 +5,6 @@ */ package org.antlr.v4.test.runtime.java; -import org.antlr.v4.Tool; import org.antlr.v4.analysis.AnalysisPipeline; import org.antlr.v4.automata.ATNFactory; import org.antlr.v4.automata.ATNPrinter; @@ -14,25 +13,12 @@ import org.antlr.v4.automata.ParserATNFactory; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CommonToken; import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.IntStream; import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenSource; import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.WritableToken; -import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.runtime.atn.ATNDeserializer; -import org.antlr.v4.runtime.atn.ATNSerializer; import org.antlr.v4.runtime.atn.ATNState; -import org.antlr.v4.runtime.atn.DecisionState; -import org.antlr.v4.runtime.atn.LexerATNSimulator; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.misc.IntegerList; -import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.misc.Pair; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.semantics.SemanticPipeline; @@ -42,9 +28,6 @@ import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.GrammarSemanticsMessage; import org.antlr.v4.tool.LexerGrammar; import org.antlr.v4.tool.Rule; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; import org.stringtemplate.v4.ST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.STGroupString; @@ -68,13 +51,9 @@ import java.net.URL; import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.TreeMap; import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertFalse; @@ -83,9 +62,7 @@ import static junit.framework.TestCase.assertTrue; import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; import static org.junit.Assert.assertArrayEquals; -public class BaseJavaTest implements RuntimeTestSupport { - public static final String newline = System.getProperty("line.separator"); - public static final String pathSep = System.getProperty("path.separator"); +public class BaseJavaTest extends BaseRuntimeTestSupport implements RuntimeTestSupport { /** * When the {@code antlr.testinprocess} runtime property is set to @@ -103,308 +80,14 @@ public class BaseJavaTest implements RuntimeTestSupport { */ public static final boolean TEST_IN_SAME_PROCESS = Boolean.parseBoolean(System.getProperty("antlr.testinprocess")); - /** - * When the {@code antlr.preserve-test-dir} runtime property is set to - * {@code true}, the temporary directories created by the test run will not - * be removed at the end of the test run, even for tests that completed - * successfully. - *

- *

- * The default behavior (used in all other cases) is removing the temporary - * directories for all tests which completed successfully, and preserving - * the directories for tests which failed.

- */ - public static final boolean PRESERVE_TEST_DIR = true; //Boolean.parseBoolean(System.getProperty("antlr.preserve-test-dir")); - - /** - * The base test directory is the directory where generated files get placed - * during unit test execution. - *

- *

- * The default value for this property is the {@code java.io.tmpdir} system - * property, and can be overridden by setting the - * {@code antlr.java-test-dir} property to a custom location. Note that the - * {@code antlr.java-test-dir} property directly affects the - * {@link #CREATE_PER_TEST_DIRECTORIES} value as well.

- */ - public static final String BASE_TEST_DIR; - - /** - * When {@code true}, a temporary directory will be created for each test - * executed during the test run. - *

- *

- * This value is {@code true} when the {@code antlr.java-test-dir} system - * property is set, and otherwise {@code false}.

- */ - public static final boolean CREATE_PER_TEST_DIRECTORIES; - - static { - String baseTestDir = System.getProperty("antlr.java-test-dir"); - boolean perTestDirectories = false; - if ( baseTestDir==null || baseTestDir.isEmpty() ) { - baseTestDir = System.getProperty("java.io.tmpdir"); - perTestDirectories = true; - } - - if ( !new File(baseTestDir).isDirectory() ) { - throw new UnsupportedOperationException("The specified base test directory does not exist: "+baseTestDir); - } - - BASE_TEST_DIR = baseTestDir; - CREATE_PER_TEST_DIRECTORIES = perTestDirectories; - } - /** * Build up the full classpath we need, including the surefire path (if present) */ public static final String CLASSPATH = System.getProperty("java.class.path"); - public String tmpdir = null; - - /** - * If error during parser execution, store stderr here; can't return - * stdout and stderr. This doesn't trap errors from running antlr. - */ - protected String stderrDuringParse; - - /** - * Errors found while running antlr - */ - protected StringBuilder antlrToolErrors; - - @org.junit.Rule - public final TestRule testWatcher = new TestWatcher() { - - @Override - protected void succeeded(Description description) { - // remove tmpdir if no error. - eraseTempDir(); - } - - }; - @Override - public void testSetUp() throws Exception { -// STGroup.verbose = true; -// System.err.println("testSetUp "+Thread.currentThread().getName()); - if ( CREATE_PER_TEST_DIRECTORIES ) { - // new output dir for each test - String threadName = Thread.currentThread().getName(); - String testDirectory = getClass().getSimpleName()+"-"+threadName+"-"+System.nanoTime(); - tmpdir = new File(BASE_TEST_DIR, testDirectory).getAbsolutePath(); - } - else { - tmpdir = new File(BASE_TEST_DIR).getAbsolutePath(); - if ( !PRESERVE_TEST_DIR && new File(tmpdir).exists() ) { - eraseFiles(); - } - } - antlrToolErrors = new StringBuilder(); - } - - @Override - public void testTearDown() throws Exception { - } - - @Override - public void beforeTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public void afterTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public String getTmpDir() { - return tmpdir; - } - - @Override - public String getStdout() { - return null; - } - - @Override - public String getParseErrors() { - return stderrDuringParse; - } - - @Override - public String getANTLRToolErrors() { - if ( antlrToolErrors.length()==0 ) { - return null; - } - return antlrToolErrors.toString(); - } - - protected org.antlr.v4.Tool newTool(String[] args) { - Tool tool = new Tool(args); - return tool; - } - - protected ATN createATN(Grammar g, boolean useSerializer) { - if ( g.atn==null ) { - semanticProcess(g); - assertEquals(0, g.tool.getNumErrors()); - - ParserATNFactory f; - if ( g.isLexer() ) { - f = new LexerATNFactory((LexerGrammar) g); - } - else { - f = new ParserATNFactory(g); - } - - g.atn = f.createATN(); - assertEquals(0, g.tool.getNumErrors()); - } - - ATN atn = g.atn; - if ( useSerializer ) { - char[] serialized = ATNSerializer.getSerializedAsChars(atn); - return new ATNDeserializer().deserialize(serialized); - } - - return atn; - } - - protected void semanticProcess(Grammar g) { - if ( g.ast!=null && !g.ast.hasErrors ) { -// System.out.println(g.ast.toStringTree()); - Tool antlr = new Tool(); - SemanticPipeline sem = new SemanticPipeline(g); - sem.process(); - if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any) - for (Grammar imp : g.getImportedGrammars()) { - antlr.processNonCombinedGrammar(imp, false); - } - } - } - } - - public DFA createDFA(Grammar g, DecisionState s) { -// PredictionDFAFactory conv = new PredictionDFAFactory(g, s); -// DFA dfa = conv.createDFA(); -// conv.issueAmbiguityWarnings(); -// System.out.print("DFA="+dfa); -// return dfa; - return null; - } - -// public void minimizeDFA(DFA dfa) { -// DFAMinimizer dmin = new DFAMinimizer(dfa); -// dfa.minimized = dmin.minimize(); -// } - - IntegerList getTypesFromString(Grammar g, String expecting) { - IntegerList expectingTokenTypes = new IntegerList(); - if ( expecting!=null && !expecting.trim().isEmpty() ) { - for (String tname : expecting.replace(" ", "").split(",")) { - int ttype = g.getTokenType(tname); - expectingTokenTypes.add(ttype); - } - } - return expectingTokenTypes; - } - - public IntegerList getTokenTypesViaATN(String input, LexerATNSimulator lexerATN) { - ANTLRInputStream in = new ANTLRInputStream(input); - IntegerList tokenTypes = new IntegerList(); - int ttype; - do { - ttype = lexerATN.match(in, Lexer.DEFAULT_MODE); - tokenTypes.add(ttype); - } while ( ttype!=Token.EOF ); - return tokenTypes; - } - - public List getTokenTypes(LexerGrammar lg, - ATN atn, - CharStream input) { - LexerATNSimulator interp = new LexerATNSimulator(atn, new DFA[]{new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE))}, null); - List tokenTypes = new ArrayList(); - int ttype; - boolean hitEOF = false; - do { - if ( hitEOF ) { - tokenTypes.add("EOF"); - break; - } - int t = input.LA(1); - ttype = interp.match(input, Lexer.DEFAULT_MODE); - if ( ttype==Token.EOF ) { - tokenTypes.add("EOF"); - } - else { - tokenTypes.add(lg.typeToTokenList.get(ttype)); - } - - if ( t==IntStream.EOF ) { - hitEOF = true; - } - } while ( ttype!=Token.EOF ); - return tokenTypes; - } - - List checkRuleDFA(String gtext, String ruleName, String expecting) - throws Exception { - ErrorQueue equeue = new ErrorQueue(); - Grammar g = new Grammar(gtext, equeue); - ATN atn = createATN(g, false); - ATNState s = atn.ruleToStartState[g.getRule(ruleName).index]; - if ( s==null ) { - System.err.println("no such rule: "+ruleName); - return null; - } - ATNState t = s.transition(0).target; - if ( !(t instanceof DecisionState) ) { - System.out.println(ruleName+" has no decision"); - return null; - } - DecisionState blk = (DecisionState) t; - checkRuleDFA(g, blk, expecting); - return equeue.all; - } - - List checkRuleDFA(String gtext, int decision, String expecting) - throws Exception { - ErrorQueue equeue = new ErrorQueue(); - Grammar g = new Grammar(gtext, equeue); - ATN atn = createATN(g, false); - DecisionState blk = atn.decisionToState.get(decision); - checkRuleDFA(g, blk, expecting); - return equeue.all; - } - - void checkRuleDFA(Grammar g, DecisionState blk, String expecting) - throws Exception { - DFA dfa = createDFA(g, blk); - String result = null; - if ( dfa!=null ) result = dfa.toString(); - assertEquals(expecting, result); - } - - List checkLexerDFA(String gtext, String expecting) - throws Exception { - return checkLexerDFA(gtext, LexerGrammar.DEFAULT_MODE_NAME, expecting); - } - - List checkLexerDFA(String gtext, String modeName, String expecting) - throws Exception { - ErrorQueue equeue = new ErrorQueue(); - LexerGrammar g = new LexerGrammar(gtext, equeue); - g.atn = createATN(g, false); -// LexerATNToDFAConverter conv = new LexerATNToDFAConverter(g); -// DFA dfa = conv.createDFA(modeName); -// g.setLookaheadDFA(0, dfa); // only one decision to worry about -// -// String result = null; -// if ( dfa!=null ) result = dfa.toString(); -// assertEquals(expecting, result); -// -// return equeue.all; - return null; + protected String getPropertyPrefix() { + return "antrl4-java"; } protected String load(String fileName, String encoding) @@ -439,7 +122,7 @@ public class BaseJavaTest implements RuntimeTestSupport { protected boolean compile(String... fileNames) { List files = new ArrayList(); for (String fileName : fileNames) { - File f = new File(tmpdir, fileName); + File f = new File(getTempTestDir(), fileName); files.add(f); } @@ -454,7 +137,7 @@ public class BaseJavaTest implements RuntimeTestSupport { fileManager.getJavaFileObjectsFromFiles(files); Iterable compileOptions = - Arrays.asList("-g", "-source", "1.6", "-target", "1.6", "-implicit:class", "-Xlint:-options", "-d", tmpdir, "-cp", tmpdir+pathSep+CLASSPATH); + Arrays.asList("-g", "-source", "1.6", "-target", "1.6", "-implicit:class", "-Xlint:-options", "-d", getTempDirPath(), "-cp", getTempDirPath() + PATH_SEP + CLASSPATH); JavaCompiler.CompilationTask task = compiler.getTask(null, fileManager, null, compileOptions, null, @@ -488,11 +171,10 @@ public class BaseJavaTest implements RuntimeTestSupport { null, lexerName); assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); writeLexerTestFile(lexerName, showDFA); compile("Test.java"); - String output = execClass("Test"); - return output; + return execClass("Test"); } public ParseTree execParser(String startRuleName, String input, @@ -542,7 +224,7 @@ public class BaseJavaTest implements RuntimeTestSupport { public Class loadClassFromTempDir(String name) throws Exception { ClassLoader loader = - new URLClassLoader(new URL[]{new File(tmpdir).toURI().toURL()}, + new URLClassLoader(new URL[]{getTempTestDir().toURI().toURL()}, ClassLoader.getSystemClassLoader()); return loader.loadClass(name); } @@ -590,7 +272,7 @@ public class BaseJavaTest implements RuntimeTestSupport { lexerName, "-visitor"); assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); return rawExecRecognizer(parserName, lexerName, startRuleName, @@ -617,7 +299,7 @@ public class BaseJavaTest implements RuntimeTestSupport { String... extraOptions) { ErrorQueue equeue = - BaseRuntimeTest.antlrOnString(getTmpDir(), "Java", grammarFileName, grammarStr, defaultListener, extraOptions); + BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", grammarFileName, grammarStr, defaultListener, extraOptions); if (!equeue.errors.isEmpty()) { return false; } @@ -639,8 +321,7 @@ public class BaseJavaTest implements RuntimeTestSupport { files.add(grammarName+"BaseVisitor.java"); } } - boolean allIsWell = compile(files.toArray(new String[files.size()])); - return allIsWell; + return compile(files.toArray(new String[0])); } protected String rawExecRecognizer(String parserName, @@ -649,7 +330,7 @@ public class BaseJavaTest implements RuntimeTestSupport { boolean debug, boolean profile) { - this.stderrDuringParse = null; + setParseErrors(null); if ( parserName==null ) { writeLexerTestFile(lexerName, false); } @@ -672,7 +353,7 @@ public class BaseJavaTest implements RuntimeTestSupport { public String execClass(String className) { if (TEST_IN_SAME_PROCESS) { try { - ClassLoader loader = new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() }, ClassLoader.getSystemClassLoader()); + ClassLoader loader = new URLClassLoader(new URL[] { getTempTestDir().toURI().toURL() }, ClassLoader.getSystemClassLoader()); final Class mainClass = (Class)loader.loadClass(className); final Method mainMethod = mainClass.getDeclaredMethod("main", String[].class); PipedInputStream stdoutIn = new PipedInputStream(); @@ -690,7 +371,7 @@ public class BaseJavaTest implements RuntimeTestSupport { System.setErr(new PrintStream(stderrOut)); stdoutVacuum.start(); stderrVacuum.start(); - mainMethod.invoke(null, (Object)new String[] { new File(tmpdir, "input").getAbsolutePath() }); + mainMethod.invoke(null, (Object)new String[] { new File(getTempTestDir(), "input").getAbsolutePath() }); } finally { System.setErr(originalErr); @@ -709,7 +390,7 @@ public class BaseJavaTest implements RuntimeTestSupport { output = null; } if ( stderrVacuum.toString().length()>0 ) { - this.stderrDuringParse = stderrVacuum.toString(); + setParseErrors(stderrVacuum.toString()); } return output; } @@ -720,14 +401,14 @@ public class BaseJavaTest implements RuntimeTestSupport { try { String[] args = new String[] { - "java", "-classpath", tmpdir+pathSep+CLASSPATH, + "java", "-classpath", getTempDirPath() + PATH_SEP + CLASSPATH, "-Dfile.encoding=UTF-8", - className, new File(tmpdir, "input").getAbsolutePath() + className, new File(getTempTestDir(), "input").getAbsolutePath() }; // String cmdLine = Utils.join(args, " "); // System.err.println("execParser: "+cmdLine); Process process = - Runtime.getRuntime().exec(args, null, new File(tmpdir)); + Runtime.getRuntime().exec(args, null, getTempTestDir()); StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); stdoutVacuum.start(); @@ -740,7 +421,7 @@ public class BaseJavaTest implements RuntimeTestSupport { output = null; } if ( stderrVacuum.toString().length()>0 ) { - this.stderrDuringParse = stderrVacuum.toString(); + setParseErrors(stderrVacuum.toString()); } return output; } @@ -751,49 +432,6 @@ public class BaseJavaTest implements RuntimeTestSupport { return null; } -// void ambig(List msgs, int[] expectedAmbigAlts, String expectedAmbigInput) -// throws Exception -// { -// ambig(msgs, 0, expectedAmbigAlts, expectedAmbigInput); -// } - -// void ambig(List msgs, int i, int[] expectedAmbigAlts, String expectedAmbigInput) -// throws Exception -// { -// List amsgs = getMessagesOfType(msgs, AmbiguityMessage.class); -// AmbiguityMessage a = (AmbiguityMessage)amsgs.get(i); -// if ( a==null ) assertNull(expectedAmbigAlts); -// else { -// assertEquals(a.conflictingAlts.toString(), Arrays.toString(expectedAmbigAlts)); -// } -// assertEquals(expectedAmbigInput, a.input); -// } - -// void unreachable(List msgs, int[] expectedUnreachableAlts) -// throws Exception -// { -// unreachable(msgs, 0, expectedUnreachableAlts); -// } - -// void unreachable(List msgs, int i, int[] expectedUnreachableAlts) -// throws Exception -// { -// List amsgs = getMessagesOfType(msgs, UnreachableAltsMessage.class); -// UnreachableAltsMessage u = (UnreachableAltsMessage)amsgs.get(i); -// if ( u==null ) assertNull(expectedUnreachableAlts); -// else { -// assertEquals(u.conflictingAlts.toString(), Arrays.toString(expectedUnreachableAlts)); -// } -// } - - List getMessagesOfType(List msgs, Class c) { - List filtered = new ArrayList(); - for (ANTLRMessage m : msgs) { - if ( m.getClass() == c ) filtered.add(m); - } - return filtered; - } - public void checkRuleATN(Grammar g, String ruleName, String expecting) { // DOTGenerator dot = new DOTGenerator(g); // System.out.println(dot.getDOT(g.atn.ruleToStartState[g.getRule(ruleName).index])); @@ -843,25 +481,7 @@ public class BaseJavaTest implements RuntimeTestSupport { } } - protected void checkGrammarSemanticsError(ErrorQueue equeue, - GrammarSemanticsMessage expectedMessage) - throws Exception - { - ANTLRMessage foundMsg = null; - for (int i = 0; i < equeue.errors.size(); i++) { - ANTLRMessage m = equeue.errors.get(i); - if (m.getErrorType()==expectedMessage.getErrorType() ) { - foundMsg = m; - } - } - assertNotNull("no error; "+expectedMessage.getErrorType()+" expected", foundMsg); - assertTrue("error is not a GrammarSemanticsMessage", - foundMsg instanceof GrammarSemanticsMessage); - assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs())); - if ( equeue.size()!=1 ) { - System.err.println(equeue); - } - } + protected void checkGrammarSemanticsWarning(ErrorQueue equeue, GrammarSemanticsMessage expectedMessage) @@ -883,49 +503,6 @@ public class BaseJavaTest implements RuntimeTestSupport { } } - protected void checkError(ErrorQueue equeue, - ANTLRMessage expectedMessage) - throws Exception - { - //System.out.println("errors="+equeue); - ANTLRMessage foundMsg = null; - for (int i = 0; i < equeue.errors.size(); i++) { - ANTLRMessage m = equeue.errors.get(i); - if (m.getErrorType()==expectedMessage.getErrorType() ) { - foundMsg = m; - } - } - assertTrue("no error; "+expectedMessage.getErrorType()+" expected", !equeue.errors.isEmpty()); - assertTrue("too many errors; "+equeue.errors, equeue.errors.size()<=1); - assertNotNull("couldn't find expected error: "+expectedMessage.getErrorType(), foundMsg); - /* - assertTrue("error is not a GrammarSemanticsMessage", - foundMsg instanceof GrammarSemanticsMessage); - */ - assertArrayEquals(expectedMessage.getArgs(), foundMsg.getArgs()); - } - - public static class FilteringTokenStream extends CommonTokenStream { - public FilteringTokenStream(TokenSource src) { super(src); } - Set hide = new HashSet(); - @Override - protected boolean sync(int i) { - if (!super.sync(i)) { - return false; - } - - Token t = get(i); - if ( hide.contains(t.getType()) ) { - ((WritableToken)t).setChannel(Token.HIDDEN_CHANNEL); - } - - return true; - } - public void setTokenTypeChannel(int ttype, int channel) { - hide.add(ttype); - } - } - protected void writeTestFile(String parserName, String lexerName, String parserStartRuleName, @@ -988,7 +565,7 @@ public class BaseJavaTest implements RuntimeTestSupport { outputFileST.add("parserName", parserName); outputFileST.add("lexerName", lexerName); outputFileST.add("parserStartRuleName", parserStartRuleName); - writeFile(tmpdir, "Test.java", outputFileST.render()); + writeFile(getTempDirPath(), "Test.java", outputFileST.render()); } protected void writeLexerTestFile(String lexerName, boolean showDFA) { @@ -1009,219 +586,13 @@ public class BaseJavaTest implements RuntimeTestSupport { ); outputFileST.add("lexerName", lexerName); - writeFile(tmpdir, "Test.java", outputFileST.render()); + writeFile(getTempDirPath(), "Test.java", outputFileST.render()); } - public void writeRecognizerAndCompile(String parserName, String lexerName, - String parserStartRuleName, - boolean debug, - boolean profile) { - if ( parserName==null ) { - writeLexerTestFile(lexerName, debug); - } - else { - writeTestFile(parserName, - lexerName, - parserStartRuleName, - debug, - profile); - } - - compile("Test.java"); - } - - - protected void eraseFiles(final String filesEndingWith) { - if (tmpdir == null) { - return; - } - File tmpdirF = new File(tmpdir); - String[] files = tmpdirF.list(); - for(int i = 0; files!=null && i < files.length; i++) { - if ( files[i].endsWith(filesEndingWith) ) { - new File(tmpdir+"/"+files[i]).delete(); - } - } - } - - protected void eraseFiles() { - if (tmpdir == null) { - return; - } - File tmpdirF = new File(tmpdir); - String[] files = tmpdirF.list(); - for(int i = 0; files!=null && i < files.length; i++) { - new File(tmpdir+"/"+files[i]).delete(); - } - } - - public void eraseTempDir() { - if (shouldEraseTempDir()) { - File tmpdirF = new File(tmpdir); - if (tmpdirF.exists()) { - eraseFiles(); - tmpdirF.delete(); - } - } - } - - private boolean shouldEraseTempDir() { - return tmpdir != null; - } - - - public String getFirstLineOfException() { - if ( this.stderrDuringParse ==null ) { - return null; - } - String[] lines = this.stderrDuringParse.split("\n"); - String prefix="Exception in thread \"main\" "; - return lines[0].substring(prefix.length(),lines[0].length()); - } - - /** - * When looking at a result set that consists of a Map/HashTable - * we cannot rely on the output order, as the hashing algorithm or other aspects - * of the implementation may be different on differnt JDKs or platforms. Hence - * we take the Map, convert the keys to a List, sort them and Stringify the Map, which is a - * bit of a hack, but guarantees that we get the same order on all systems. We assume that - * the keys are strings. - * - * @param m The Map that contains keys we wish to return in sorted order - * @return A string that represents all the keys in sorted order. - */ - public String sortMapToString(Map m) { - // Pass in crap, and get nothing back - // - if (m == null) { - return null; - } - - System.out.println("Map toString looks like: " + m.toString()); - - // Sort the keys in the Map - // - TreeMap nset = new TreeMap(m); - - System.out.println("Tree map looks like: " + nset.toString()); - return nset.toString(); - } public List realElements(List elements) { return elements.subList(Token.MIN_USER_TOKEN_TYPE, elements.size()); } - public void assertNotNullOrEmpty(String message, String text) { - assertNotNull(message, text); - assertFalse(message, text.isEmpty()); - } - public void assertNotNullOrEmpty(String text) { - assertNotNull(text); - assertFalse(text.isEmpty()); - } - - public static class IntTokenStream implements TokenStream { - public IntegerList types; - int p=0; - public IntTokenStream(IntegerList types) { this.types = types; } - - @Override - public void consume() { p++; } - - @Override - public int LA(int i) { return LT(i).getType(); } - - @Override - public int mark() { - return index(); - } - - @Override - public int index() { return p; } - - @Override - public void release(int marker) { - seek(marker); - } - - @Override - public void seek(int index) { - p = index; - } - - @Override - public int size() { - return types.size(); - } - - @Override - public String getSourceName() { - return UNKNOWN_SOURCE_NAME; - } - - @Override - public Token LT(int i) { - CommonToken t; - int rawIndex = p + i - 1; - if ( rawIndex>=types.size() ) t = new CommonToken(Token.EOF); - else t = new CommonToken(types.get(rawIndex)); - t.setTokenIndex(rawIndex); - return t; - } - - @Override - public Token get(int i) { - return new org.antlr.v4.runtime.CommonToken(types.get(i)); - } - - @Override - public TokenSource getTokenSource() { - return null; - } - - - @Override - public String getText() { - throw new UnsupportedOperationException("can't give strings"); - } - - - @Override - public String getText(Interval interval) { - throw new UnsupportedOperationException("can't give strings"); - } - - - @Override - public String getText(RuleContext ctx) { - throw new UnsupportedOperationException("can't give strings"); - } - - - @Override - public String getText(Token start, Token stop) { - throw new UnsupportedOperationException("can't give strings"); - } - } - - /** Sort a list */ - public > List sort(List data) { - List dup = new ArrayList(); - dup.addAll(data); - Collections.sort(dup); - return dup; - } - - /** Return map sorted by key */ - public ,V> LinkedHashMap sort(Map data) { - LinkedHashMap dup = new LinkedHashMap(); - List keys = new ArrayList(); - keys.addAll(data.keySet()); - Collections.sort(keys); - for (K k : keys) { - dup.put(k, data.get(k)); - } - return dup; - } } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java index 99b063158..f58bd075c 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java @@ -5,19 +5,7 @@ */ package org.antlr.v4.test.runtime.javascript; -import org.antlr.v4.Tool; -import org.antlr.v4.automata.LexerATNFactory; -import org.antlr.v4.automata.ParserATNFactory; -import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.runtime.atn.ATNDeserializer; -import org.antlr.v4.runtime.atn.ATNSerializer; -import org.antlr.v4.semantics.SemanticPipeline; import org.antlr.v4.test.runtime.*; -import org.antlr.v4.tool.Grammar; -import org.antlr.v4.tool.LexerGrammar; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; import org.stringtemplate.v4.ST; import java.io.File; @@ -27,131 +15,13 @@ import java.util.*; import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString; import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; -public class BaseNodeTest implements RuntimeTestSupport { - // -J-Dorg.antlr.v4.test.BaseTest.level=FINE - // private static final Logger LOGGER = - // Logger.getLogger(BaseTest.class.getName()); - - public static final String newline = System.getProperty("line.separator"); - public static final String pathSep = System.getProperty("path.separator"); - - public String tmpdir = null; - - /** - * If error during parser execution, store stderr here; can't return stdout - * and stderr. This doesn't trap errors from running antlr. - */ - protected String stderrDuringParse; - - /** Errors found while running antlr */ - protected StringBuilder antlrToolErrors; - - @org.junit.Rule - public final TestRule testWatcher = new TestWatcher() { - - @Override - protected void succeeded(Description description) { - // remove tmpdir if no error. - eraseTempDir(); - } - - }; +public class BaseNodeTest extends BaseRuntimeTestSupport implements RuntimeTestSupport { @Override - public void testSetUp() throws Exception { - // new output dir for each test - String prop = System.getProperty("antlr-javascript-test-dir"); - if (prop != null && prop.length() > 0) { - tmpdir = prop; - } - else { - tmpdir = new File(System.getProperty("java.io.tmpdir"), getClass() - .getSimpleName()+"-"+Thread.currentThread().getName()+"-"+System.currentTimeMillis()) - .getAbsolutePath(); - } - File dir = new File(tmpdir); - if (dir.exists()) - this.eraseFiles(dir); - antlrToolErrors = new StringBuilder(); - } - - @Override - public void testTearDown() throws Exception { - } - - @Override - public void beforeTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public void afterTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public String getTmpDir() { - return tmpdir; - } - - @Override - public String getStdout() { - return null; - } - - @Override - public String getParseErrors() { - return stderrDuringParse; - } - - @Override - public String getANTLRToolErrors() { - if ( antlrToolErrors.length()==0 ) { - return null; - } - return antlrToolErrors.toString(); - } - - protected ATN createATN(Grammar g, boolean useSerializer) { - if (g.atn == null) { - semanticProcess(g); - assertEquals(0, g.tool.getNumErrors()); - - ParserATNFactory f; - if (g.isLexer()) { - f = new LexerATNFactory((LexerGrammar) g); - } - else { - f = new ParserATNFactory(g); - } - - g.atn = f.createATN(); - assertEquals(0, g.tool.getNumErrors()); - } - - ATN atn = g.atn; - if (useSerializer) { - char[] serialized = ATNSerializer.getSerializedAsChars(atn); - return new ATNDeserializer().deserialize(serialized); - } - - return atn; - } - - protected void semanticProcess(Grammar g) { - if (g.ast != null && !g.ast.hasErrors) { - System.out.println(g.ast.toStringTree()); - Tool antlr = new Tool(); - SemanticPipeline sem = new SemanticPipeline(g); - sem.process(); - if (g.getImportedGrammars() != null) { // process imported grammars - // (if any) - for (Grammar imp : g.getImportedGrammars()) { - antlr.processNonCombinedGrammar(imp, false); - } - } - } + protected String getPropertyPrefix() { + return "antlr4-javascript"; } protected String execLexer(String grammarFileName, String grammarStr, @@ -165,9 +35,9 @@ public class BaseNodeTest implements RuntimeTestSupport { boolean success = rawGenerateAndBuildRecognizer(grammarFileName, grammarStr, null, lexerName, "-no-listener"); assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); writeLexerTestFile(lexerName, showDFA); - writeFile(tmpdir, "package.json", "{\"type\": \"module\"}"); + writeFile(getTempDirPath(), "package.json", "{\"type\": \"module\"}"); String output = execModule("Test.js"); if ( output!=null && output.length()==0 ) { output = null; @@ -184,10 +54,10 @@ public class BaseNodeTest implements RuntimeTestSupport { boolean success = rawGenerateAndBuildRecognizer(grammarFileName, grammarStr, parserName, lexerName, "-visitor"); assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); rawBuildRecognizerTestFile(parserName, lexerName, listenerName, visitorName, startRuleName, showDiagnosticErrors); - writeFile(tmpdir, "package.json", "{\"type\": \"module\"}"); + writeFile(getTempDirPath(), "package.json", "{\"type\": \"module\"}"); return execRecognizer(); } @@ -203,7 +73,7 @@ public class BaseNodeTest implements RuntimeTestSupport { protected boolean rawGenerateAndBuildRecognizer(String grammarFileName, String grammarStr, String parserName, String lexerName, boolean defaultListener, String... extraOptions) { - ErrorQueue equeue = antlrOnString(getTmpDir(), "JavaScript", grammarFileName, grammarStr, + ErrorQueue equeue = antlrOnString(getTempDirPath(), "JavaScript", grammarFileName, grammarStr, defaultListener, extraOptions); if (!equeue.errors.isEmpty()) { return false; @@ -234,7 +104,7 @@ public class BaseNodeTest implements RuntimeTestSupport { protected void rawBuildRecognizerTestFile(String parserName, String lexerName, String listenerName, String visitorName, String parserStartRuleName, boolean debug) { - this.stderrDuringParse = null; + setParseErrors(null); if (parserName == null) { writeLexerTestFile(lexerName, false); } @@ -255,16 +125,16 @@ public class BaseNodeTest implements RuntimeTestSupport { installRuntime(npmPath); registerRuntime(npmPath); } - String modulePath = new File(new File(tmpdir), fileName) + String modulePath = new File(getTempTestDir(), fileName) .getAbsolutePath(); linkRuntime(npmPath); String nodejsPath = locateNodeJS(); - String inputPath = new File(new File(tmpdir), "input") + String inputPath = new File(getTempTestDir(), "input") .getAbsolutePath(); ProcessBuilder builder = new ProcessBuilder(nodejsPath, modulePath, inputPath); - builder.environment().put("NODE_PATH", tmpdir); - builder.directory(new File(tmpdir)); + builder.environment().put("NODE_PATH", getTempDirPath()); + builder.directory(getTempTestDir()); Process process = builder.start(); StreamVacuum stdoutVacuum = new StreamVacuum( process.getInputStream()); @@ -283,7 +153,7 @@ public class BaseNodeTest implements RuntimeTestSupport { output = null; } if (stderrVacuum.toString().length() > 0) { - this.stderrDuringParse = stderrVacuum.toString(); + setParseErrors(stderrVacuum.toString()); } return output; } catch (Exception e) { @@ -298,8 +168,8 @@ public class BaseNodeTest implements RuntimeTestSupport { String runtimePath = locateRuntime(); ProcessBuilder builder = new ProcessBuilder(npmPath, "install"); builder.directory(new File(runtimePath)); - builder.redirectError(new File(tmpdir, "error.txt")); - builder.redirectOutput(new File(tmpdir, "output.txt")); + builder.redirectError(new File(getTempTestDir(), "error.txt")); + builder.redirectOutput(new File(getTempTestDir(), "output.txt")); Process process = builder.start(); // TODO switch to jdk 8 process.waitFor(); @@ -314,8 +184,8 @@ public class BaseNodeTest implements RuntimeTestSupport { String runtimePath = locateRuntime(); ProcessBuilder builder = new ProcessBuilder(npmPath, "link"); builder.directory(new File(runtimePath)); - builder.redirectError(new File(tmpdir, "error.txt")); - builder.redirectOutput(new File(tmpdir, "output.txt")); + builder.redirectError(new File(getTempTestDir(), "error.txt")); + builder.redirectOutput(new File(getTempTestDir(), "output.txt")); Process process = builder.start(); // TODO switch to jdk 8 process.waitFor(); @@ -332,9 +202,9 @@ public class BaseNodeTest implements RuntimeTestSupport { args.add("sudo"); args.addAll(Arrays.asList(npmPath, "link", "antlr4")); ProcessBuilder builder = new ProcessBuilder(args.toArray(new String[0])); - builder.directory(new File(tmpdir)); - builder.redirectError(new File(tmpdir, "error.txt")); - builder.redirectOutput(new File(tmpdir, "output.txt")); + builder.directory(getTempTestDir()); + builder.redirectError(new File(getTempTestDir(), "error.txt")); + builder.redirectOutput(new File(getTempTestDir(), "output.txt")); Process process = builder.start(); // TODO switch to jdk 8 process.waitFor(); @@ -396,11 +266,6 @@ public class BaseNodeTest implements RuntimeTestSupport { return runtimeSrc.getPath(); } - private boolean isWindows() { - return System.getProperty("os.name").toLowerCase().contains("windows"); - } - - protected void writeParserTestFile(String parserName, String lexerName, String listenerName, String visitorName, String parserStartRuleName, boolean debug) { @@ -451,7 +316,7 @@ public class BaseNodeTest implements RuntimeTestSupport { outputFileST.add("listenerName", listenerName); outputFileST.add("visitorName", visitorName); outputFileST.add("parserStartRuleName", parserStartRuleName); - writeFile(tmpdir, "Test.js", outputFileST.render()); + writeFile(getTempDirPath(), "Test.js", outputFileST.render()); } protected void writeLexerTestFile(String lexerName, boolean showDFA) { @@ -471,54 +336,7 @@ public class BaseNodeTest implements RuntimeTestSupport { : "") + "}\n" + "\n" + "main(process.argv);\n" + "\n"); outputFileST.add("lexerName", lexerName); - writeFile(tmpdir, "Test.js", outputFileST.render()); + writeFile(getTempDirPath(), "Test.js", outputFileST.render()); } - protected void eraseFiles(File dir) { - String[] files = dir.list(); - for (int i = 0; files != null && i < files.length; i++) { - new File(dir, files[i]).delete(); - } - } - - @Override - public void eraseTempDir() { - if (shouldEraseTempDir()) { - File tmpdirF = new File(tmpdir); - if (tmpdirF.exists()) { - eraseFiles(tmpdirF); - tmpdirF.delete(); - } - } - } - - private boolean shouldEraseTempDir() { - if(tmpdir==null) - return false; - String propName = "antlr-javascript-erase-test-dir"; - String prop = System.getProperty(propName); - if (prop != null && prop.length() > 0) - return Boolean.getBoolean(prop); - else - return true; - } - - /** Sort a list */ - public > List sort(List data) { - List dup = new ArrayList(data); - Collections.sort(dup); - return dup; - } - - /** Return map sorted by key */ - public , V> LinkedHashMap sort( - Map data) { - LinkedHashMap dup = new LinkedHashMap(); - List keys = new ArrayList(data.keySet()); - Collections.sort(keys); - for (K k : keys) { - dup.put(k, data.get(k)); - } - return dup; - } } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java index deef63287..dd5af7cb9 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java @@ -11,28 +11,11 @@ import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Set; -import org.antlr.v4.Tool; -import org.antlr.v4.automata.LexerATNFactory; -import org.antlr.v4.automata.ParserATNFactory; -import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.runtime.atn.ATNDeserializer; -import org.antlr.v4.runtime.atn.ATNSerializer; -import org.antlr.v4.semantics.SemanticPipeline; -import org.antlr.v4.test.runtime.ErrorQueue; -import org.antlr.v4.test.runtime.RuntimeTestDescriptor; -import org.antlr.v4.test.runtime.RuntimeTestSupport; -import org.antlr.v4.test.runtime.StreamVacuum; -import org.antlr.v4.tool.Grammar; -import org.antlr.v4.tool.LexerGrammar; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; + +import org.antlr.v4.test.runtime.*; import org.stringtemplate.v4.ST; import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString; @@ -40,143 +23,12 @@ import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -public class BasePHPTest implements RuntimeTestSupport { - public static final String newline = System.getProperty("line.separator"); +public class BasePHPTest extends BaseRuntimeTestSupport implements RuntimeTestSupport { - public String tmpdir = null; - - /** - * If error during parser execution, store stderr here; can't return - * stdout and stderr. This doesn't trap errors from running antlr. - */ - protected String stderrDuringParse; - - /** - * Errors found while running antlr - */ - protected StringBuilder antlrToolErrors; - - @org.junit.Rule - public final TestRule testWatcher = new TestWatcher() { - - @Override - protected void succeeded(Description description) { - // remove tmpdir if no error. - eraseTempDir(); - } - - }; - - private String getPropertyPrefix() { + public String getPropertyPrefix() { return "antlr-php"; } - @Override - public void testSetUp() throws Exception { - // new output dir for each test - String propName = getPropertyPrefix() + "-test-dir"; - String prop = System.getProperty(propName); - - if (prop != null && prop.length() > 0) { - tmpdir = prop; - } else { - String classSimpleName = getClass().getSimpleName(); - String threadName = Thread.currentThread().getName(); - String childPath = String.format("%s-%s-%s", classSimpleName, threadName, System.currentTimeMillis()); - tmpdir = new File(System.getProperty("java.io.tmpdir"), childPath).getAbsolutePath(); - } - - antlrToolErrors = new StringBuilder(); - } - - @Override - public void testTearDown() throws Exception { - } - - @Override - public void beforeTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public void afterTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public String getTmpDir() { - return tmpdir; - } - - @Override - public String getStdout() { - return null; - } - - @Override - public String getParseErrors() { - return stderrDuringParse; - } - - @Override - public String getANTLRToolErrors() { - if (antlrToolErrors.length() == 0) { - return null; - } - - return antlrToolErrors.toString(); - } - - protected ATN createATN(Grammar g, boolean useSerializer) { - if (g.atn == null) { - semanticProcess(g); - - assertEquals(0, g.tool.getNumErrors()); - - ParserATNFactory f; - - if (g.isLexer()) { - f = new LexerATNFactory((LexerGrammar) g); - } else { - f = new ParserATNFactory(g); - } - - g.atn = f.createATN(); - assertEquals(0, g.tool.getNumErrors()); - } - - ATN atn = g.atn; - - if (useSerializer) { - char[] serialized = ATNSerializer.getSerializedAsChars(atn); - - return new ATNDeserializer().deserialize(serialized); - } - - return atn; - } - - protected void semanticProcess(Grammar g) { - if (g.ast != null && !g.ast.hasErrors) { - Tool antlr = new Tool(); - SemanticPipeline sem = new SemanticPipeline(g); - sem.process(); - - if (g.getImportedGrammars() != null) { - for (Grammar imp: g.getImportedGrammars()) { - antlr.processNonCombinedGrammar(imp, false); - } - } - } - } - - protected String execLexer( - String grammarFileName, - String grammarStr, - String lexerName, - String input - ) { - return execLexer(grammarFileName, grammarStr, lexerName, input, false); - } - @Override public String execLexer( String grammarFileName, @@ -193,11 +45,9 @@ public class BasePHPTest implements RuntimeTestSupport { "-no-listener" ); assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); writeLexerTestFile(lexerName, showDFA); - String output = execModule("Test.php"); - - return output; + return execModule("Test.php"); } public String execParser( @@ -247,7 +97,7 @@ public class BasePHPTest implements RuntimeTestSupport { assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); rawBuildRecognizerTestFile( parserName, @@ -293,7 +143,7 @@ public class BasePHPTest implements RuntimeTestSupport { boolean defaultListener, String... extraOptions ) { - ErrorQueue equeue = antlrOnString(getTmpDir(), "PHP", grammarFileName, grammarStr, defaultListener, extraOptions); + ErrorQueue equeue = antlrOnString(getTempDirPath(), "PHP", grammarFileName, grammarStr, defaultListener, extraOptions); if (!equeue.errors.isEmpty()) { return false; @@ -330,7 +180,7 @@ public class BasePHPTest implements RuntimeTestSupport { boolean debug, boolean trace ) { - this.stderrDuringParse = null; + setParseErrors(null); if (parserName == null) { writeLexerTestFile(lexerName, false); } else { @@ -354,15 +204,14 @@ public class BasePHPTest implements RuntimeTestSupport { String phpPath = locatePhp(); String runtimePath = locateRuntime(); - File tmpdirFile = new File(tmpdir); - String modulePath = new File(tmpdirFile, fileName).getAbsolutePath(); - String inputPath = new File(tmpdirFile, "input").getAbsolutePath(); - Path outputPath = tmpdirFile.toPath().resolve("output").toAbsolutePath(); + String modulePath = new File(getTempTestDir(), fileName).getAbsolutePath(); + String inputPath = new File(getTempTestDir(), "input").getAbsolutePath(); + Path outputPath = getTempTestDir().toPath().resolve("output").toAbsolutePath(); try { ProcessBuilder builder = new ProcessBuilder(phpPath, modulePath, inputPath, outputPath.toString()); builder.environment().put("RUNTIME", runtimePath); - builder.directory(tmpdirFile); + builder.directory(getTempTestDir()); Process process = builder.start(); StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); @@ -378,7 +227,7 @@ public class BasePHPTest implements RuntimeTestSupport { } if (stderrVacuum.toString().length() > 0) { - this.stderrDuringParse = stderrVacuum.toString(); + setParseErrors(stderrVacuum.toString()); } return output; @@ -487,7 +336,7 @@ public class BasePHPTest implements RuntimeTestSupport { outputFileST.add("lexerName", lexerName); - writeFile(tmpdir, "Test.php", outputFileST.render()); + writeFile(getTempDirPath(), "Test.php", outputFileST.render()); } protected void writeParserTestFile( @@ -569,59 +418,7 @@ public class BasePHPTest implements RuntimeTestSupport { outputFileST.add("visitorName", visitorName); outputFileST.add("parserStartRuleName", parserStartRuleName); - writeFile(tmpdir, "Test.php", outputFileST.render()); + writeFile(getTempDirPath(), "Test.php", outputFileST.render()); } - protected void eraseFiles(File dir) { - String[] files = dir.list(); - for (int i = 0; files != null && i < files.length; i++) { - new File(dir, files[i]).delete(); - } - } - - @Override - public void eraseTempDir() { - if (shouldEraseTempDir()) { - File tmpdirF = new File(tmpdir); - if (tmpdirF.exists()) { - eraseFiles(tmpdirF); - tmpdirF.delete(); - } - } - } - - private boolean shouldEraseTempDir() { - if(tmpdir==null) - return false; - String propName = getPropertyPrefix() + "-erase-test-dir"; - String prop = System.getProperty(propName); - if (prop != null && prop.length() > 0) - return Boolean.getBoolean(prop); - else - return true; - } - - /** - * Sort a list - */ - public > List sort(List data) { - List dup = new ArrayList(); - dup.addAll(data); - Collections.sort(dup); - return dup; - } - - /** - * Return map sorted by key - */ - public , V> LinkedHashMap sort(Map data) { - LinkedHashMap dup = new LinkedHashMap(); - List keys = new ArrayList(); - keys.addAll(data.keySet()); - Collections.sort(keys); - for (K k: keys) { - dup.put(k, data.get(k)); - } - return dup; - } } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java index a3ea85b1b..c12967869 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java @@ -5,344 +5,38 @@ */ package org.antlr.v4.test.runtime.python; -import org.antlr.v4.Tool; -import org.antlr.v4.automata.ATNFactory; -import org.antlr.v4.automata.ATNPrinter; -import org.antlr.v4.automata.LexerATNFactory; -import org.antlr.v4.automata.ParserATNFactory; -import org.antlr.v4.codegen.CodeGenerator; -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CommonToken; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.IntStream; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.RuleContext; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenSource; -import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.WritableToken; -import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.runtime.atn.ATNDeserializer; -import org.antlr.v4.runtime.atn.ATNSerializer; -import org.antlr.v4.runtime.atn.ATNState; -import org.antlr.v4.runtime.atn.DecisionState; -import org.antlr.v4.runtime.atn.LexerATNSimulator; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.misc.IntegerList; -import org.antlr.v4.runtime.misc.Interval; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.semantics.SemanticPipeline; import org.antlr.v4.test.runtime.*; -import org.antlr.v4.tool.ANTLRMessage; -import org.antlr.v4.tool.DOTGenerator; -import org.antlr.v4.tool.Grammar; -import org.antlr.v4.tool.GrammarSemanticsMessage; -import org.antlr.v4.tool.LexerGrammar; -import org.antlr.v4.tool.Rule; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; import org.junit.runner.Description; -import org.stringtemplate.v4.ST; -import org.stringtemplate.v4.STGroup; -import org.stringtemplate.v4.STGroupString; import java.io.File; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.net.URL; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.TreeMap; import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString; import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; -public abstract class BasePythonTest implements RuntimeTestSupport { - // -J-Dorg.antlr.v4.test.BaseTest.level=FINE - // private static final Logger LOGGER = Logger.getLogger(BaseTest.class.getName()); - public static final String newline = System.getProperty("line.separator"); - public static final String pathSep = System.getProperty("path.separator"); +public abstract class BasePythonTest extends BaseRuntimeTestSupport implements RuntimeTestSupport { - public String tmpdir = null; + @Override + protected void testSucceeded(Description description) { + eraseTempPyCache(); + eraseTempDir(); + } - /** If error during parser execution, store stderr here; can't return - * stdout and stderr. This doesn't trap errors from running antlr. - */ - protected String stderrDuringParse; - - /** Errors found while running antlr */ - protected StringBuilder antlrToolErrors; - - @org.junit.Rule - public final TestRule testWatcher = new TestWatcher() { - - @Override - protected void succeeded(Description description) { - // remove tmpdir if no error. - eraseTempPyCache(); - eraseTempDir(); - } - - }; - - private String getPropertyPrefix() { + @Override + protected String getPropertyPrefix() { return "antlr-" + getLanguage().toLowerCase(); } - @Override - public void testSetUp() throws Exception { - // new output dir for each test - String propName = getPropertyPrefix() + "-test-dir"; - String prop = System.getProperty(propName); - if(prop!=null && prop.length()>0) { - tmpdir = prop; - } - else { - tmpdir = new File(System.getProperty("java.io.tmpdir"), getClass().getSimpleName()+ - "-"+Thread.currentThread().getName()+"-"+System.currentTimeMillis()).getAbsolutePath(); - } - antlrToolErrors = new StringBuilder(); - } - - @Override - public void testTearDown() throws Exception { - } - - @Override - public void beforeTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public void afterTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public String getTmpDir() { - return tmpdir; - } - - @Override - public String getStdout() { - return null; - } - - @Override - public String getParseErrors() { - return stderrDuringParse; - } - - @Override - public String getANTLRToolErrors() { - if ( antlrToolErrors.length()==0 ) { - return null; - } - return antlrToolErrors.toString(); - } - - protected org.antlr.v4.Tool newTool(String[] args) { - Tool tool = new Tool(args); - return tool; - } - - protected Tool newTool() { - org.antlr.v4.Tool tool = new Tool(new String[] {"-o", tmpdir}); - return tool; - } - - protected ATN createATN(Grammar g, boolean useSerializer) { - if ( g.atn==null ) { - semanticProcess(g); - assertEquals(0, g.tool.getNumErrors()); - - ParserATNFactory f; - if ( g.isLexer() ) { - f = new LexerATNFactory((LexerGrammar)g); - } - else { - f = new ParserATNFactory(g); - } - - g.atn = f.createATN(); - assertEquals(0, g.tool.getNumErrors()); - } - - ATN atn = g.atn; - if (useSerializer) { - char[] serialized = ATNSerializer.getSerializedAsChars(atn); - return new ATNDeserializer().deserialize(serialized); - } - - return atn; - } - - protected void semanticProcess(Grammar g) { - if ( g.ast!=null && !g.ast.hasErrors ) { - System.out.println(g.ast.toStringTree()); - Tool antlr = new Tool(); - SemanticPipeline sem = new SemanticPipeline(g); - sem.process(); - if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any) - for (Grammar imp : g.getImportedGrammars()) { - antlr.processNonCombinedGrammar(imp, false); - } - } - } - } - - public DFA createDFA(Grammar g, DecisionState s) { -// PredictionDFAFactory conv = new PredictionDFAFactory(g, s); -// DFA dfa = conv.createDFA(); -// conv.issueAmbiguityWarnings(); -// System.out.print("DFA="+dfa); -// return dfa; - return null; - } - -// public void minimizeDFA(DFA dfa) { -// DFAMinimizer dmin = new DFAMinimizer(dfa); -// dfa.minimized = dmin.minimize(); -// } - - IntegerList getTypesFromString(Grammar g, String expecting) { - IntegerList expectingTokenTypes = new IntegerList(); - if ( expecting!=null && !expecting.trim().isEmpty() ) { - for (String tname : expecting.replace(" ", "").split(",")) { - int ttype = g.getTokenType(tname); - expectingTokenTypes.add(ttype); - } - } - return expectingTokenTypes; - } - - public IntegerList getTokenTypesViaATN(String input, LexerATNSimulator lexerATN) { - ANTLRInputStream in = new ANTLRInputStream(input); - IntegerList tokenTypes = new IntegerList(); - int ttype; - do { - ttype = lexerATN.match(in, Lexer.DEFAULT_MODE); - tokenTypes.add(ttype); - } while ( ttype!= Token.EOF ); - return tokenTypes; - } - - public List getTokenTypes(LexerGrammar lg, - ATN atn, - CharStream input) - { - LexerATNSimulator interp = new LexerATNSimulator(atn,new DFA[] { new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE)) },null); - List tokenTypes = new ArrayList(); - int ttype; - boolean hitEOF = false; - do { - if ( hitEOF ) { - tokenTypes.add("EOF"); - break; - } - int t = input.LA(1); - ttype = interp.match(input, Lexer.DEFAULT_MODE); - if ( ttype == Token.EOF ) { - tokenTypes.add("EOF"); - } - else { - tokenTypes.add(lg.typeToTokenList.get(ttype)); - } - - if ( t== IntStream.EOF ) { - hitEOF = true; - } - } while ( ttype!=Token.EOF ); - return tokenTypes; - } - - List checkRuleDFA(String gtext, String ruleName, String expecting) - throws Exception - { - ErrorQueue equeue = new ErrorQueue(); - Grammar g = new Grammar(gtext, equeue); - ATN atn = createATN(g, false); - ATNState s = atn.ruleToStartState[g.getRule(ruleName).index]; - if ( s==null ) { - System.err.println("no such rule: "+ruleName); - return null; - } - ATNState t = s.transition(0).target; - if ( !(t instanceof DecisionState) ) { - System.out.println(ruleName+" has no decision"); - return null; - } - DecisionState blk = (DecisionState)t; - checkRuleDFA(g, blk, expecting); - return equeue.all; - } - - List checkRuleDFA(String gtext, int decision, String expecting) - throws Exception - { - ErrorQueue equeue = new ErrorQueue(); - Grammar g = new Grammar(gtext, equeue); - ATN atn = createATN(g, false); - DecisionState blk = atn.decisionToState.get(decision); - checkRuleDFA(g, blk, expecting); - return equeue.all; - } - - void checkRuleDFA(Grammar g, DecisionState blk, String expecting) - throws Exception - { - DFA dfa = createDFA(g, blk); - String result = null; - if ( dfa!=null ) result = dfa.toString(); - assertEquals(expecting, result); - } - - List checkLexerDFA(String gtext, String expecting) - throws Exception - { - return checkLexerDFA(gtext, LexerGrammar.DEFAULT_MODE_NAME, expecting); - } - - List checkLexerDFA(String gtext, String modeName, String expecting) - throws Exception - { - ErrorQueue equeue = new ErrorQueue(); - LexerGrammar g = new LexerGrammar(gtext, equeue); - g.atn = createATN(g, false); -// LexerATNToDFAConverter conv = new LexerATNToDFAConverter(g); -// DFA dfa = conv.createDFA(modeName); -// g.setLookaheadDFA(0, dfa); // only one decision to worry about -// -// String result = null; -// if ( dfa!=null ) result = dfa.toString(); -// assertEquals(expecting, result); -// -// return equeue.all; - return null; - } protected abstract String getLanguage(); - protected String execLexer(String grammarFileName, - String grammarStr, - String lexerName, - String input) - { - return execLexer(grammarFileName, grammarStr, lexerName, input, false); - } - @Override public String execLexer(String grammarFileName, String grammarStr, @@ -355,31 +49,12 @@ public abstract class BasePythonTest implements RuntimeTestSupport { null, lexerName,"-no-listener"); assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); writeLexerTestFile(lexerName, showDFA); - String output = execModule("Test.py"); - return output; - } - - public ParseTree execStartRule(String startRuleName, Parser parser) - throws IllegalAccessException, InvocationTargetException, - NoSuchMethodException - { - Method startRule = null; - Object[] args = null; - try { - startRule = parser.getClass().getMethod(startRuleName); - } - catch (NoSuchMethodException nsme) { - // try with int _p arg for recursive func - startRule = parser.getClass().getMethod(startRuleName, int.class); - args = new Integer[] {0}; - } - ParseTree result = (ParseTree)startRule.invoke(parser, args); -// System.out.println("parse tree = "+result.toStringTree(parser)); - return result; + return execModule("Test.py"); } + @Override public String execParser(String grammarFileName, String grammarStr, String parserName, @@ -410,7 +85,7 @@ public abstract class BasePythonTest implements RuntimeTestSupport { lexerName, "-visitor"); assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); rawBuildRecognizerTestFile(parserName, lexerName, listenerName, @@ -439,8 +114,7 @@ public abstract class BasePythonTest implements RuntimeTestSupport { boolean defaultListener, String... extraOptions) { - ErrorQueue equeue = - antlrOnString(getTmpDir(), getLanguage(), grammarFileName, grammarStr, defaultListener, extraOptions); + ErrorQueue equeue = antlrOnString(getTempDirPath(), getLanguage(), grammarFileName, grammarStr, defaultListener, extraOptions); if (!equeue.errors.isEmpty()) { return false; } @@ -470,7 +144,7 @@ public abstract class BasePythonTest implements RuntimeTestSupport { boolean debug, boolean trace) { - this.stderrDuringParse = null; + setParseErrors(null); if ( parserName==null ) { writeLexerTestFile(lexerName, false); } @@ -491,7 +165,7 @@ public abstract class BasePythonTest implements RuntimeTestSupport { public String execModule(String fileName) { String pythonPath = locatePython(); String runtimePath = locateRuntime(); - File tmpdirFile = new File(tmpdir); + File tmpdirFile = new File(getTempDirPath()); String modulePath = new File(tmpdirFile, fileName).getAbsolutePath(); String inputPath = new File(tmpdirFile, "input").getAbsolutePath(); Path outputPath = tmpdirFile.toPath().resolve("output").toAbsolutePath(); @@ -507,7 +181,7 @@ public abstract class BasePythonTest implements RuntimeTestSupport { stderrVacuum.join(); String output = TestOutputReading.read(outputPath); if ( stderrVacuum.toString().length()>0 ) { - this.stderrDuringParse = stderrVacuum.toString(); + setParseErrors(stderrVacuum.toString()); } return output; } @@ -560,190 +234,6 @@ public abstract class BasePythonTest implements RuntimeTestSupport { return runtimeSrc.getPath(); } - private boolean isWindows() { - return System.getProperty("os.name").toLowerCase().contains("windows"); - } - -// void ambig(List msgs, int[] expectedAmbigAlts, String expectedAmbigInput) -// throws Exception -// { -// ambig(msgs, 0, expectedAmbigAlts, expectedAmbigInput); -// } - -// void ambig(List msgs, int i, int[] expectedAmbigAlts, String expectedAmbigInput) -// throws Exception -// { -// List amsgs = getMessagesOfType(msgs, AmbiguityMessage.class); -// AmbiguityMessage a = (AmbiguityMessage)amsgs.get(i); -// if ( a==null ) assertNull(expectedAmbigAlts); -// else { -// assertEquals(a.conflictingAlts.toString(), Arrays.toString(expectedAmbigAlts)); -// } -// assertEquals(expectedAmbigInput, a.input); -// } - -// void unreachable(List msgs, int[] expectedUnreachableAlts) -// throws Exception -// { -// unreachable(msgs, 0, expectedUnreachableAlts); -// } - -// void unreachable(List msgs, int i, int[] expectedUnreachableAlts) -// throws Exception -// { -// List amsgs = getMessagesOfType(msgs, UnreachableAltsMessage.class); -// UnreachableAltsMessage u = (UnreachableAltsMessage)amsgs.get(i); -// if ( u==null ) assertNull(expectedUnreachableAlts); -// else { -// assertEquals(u.conflictingAlts.toString(), Arrays.toString(expectedUnreachableAlts)); -// } -// } - - List getMessagesOfType(List msgs, Class c) { - List filtered = new ArrayList(); - for (ANTLRMessage m : msgs) { - if ( m.getClass() == c ) filtered.add(m); - } - return filtered; - } - - void checkRuleATN(Grammar g, String ruleName, String expecting) { - ParserATNFactory f = new ParserATNFactory(g); - ATN atn = f.createATN(); - - DOTGenerator dot = new DOTGenerator(g); - System.out.println(dot.getDOT(atn.ruleToStartState[g.getRule(ruleName).index])); - - Rule r = g.getRule(ruleName); - ATNState startState = atn.ruleToStartState[r.index]; - ATNPrinter serializer = new ATNPrinter(g, startState); - String result = serializer.asString(); - - //System.out.print(result); - assertEquals(expecting, result); - } - - public void testActions(String templates, String actionName, String action, String expected) throws org.antlr.runtime.RecognitionException { - int lp = templates.indexOf('('); - String name = templates.substring(0, lp); - STGroup group = new STGroupString(templates); - ST st = group.getInstanceOf(name); - st.add(actionName, action); - String grammar = st.render(); - ErrorQueue equeue = new ErrorQueue(); - Grammar g = new Grammar(grammar, equeue); - if ( g.ast!=null && !g.ast.hasErrors ) { - SemanticPipeline sem = new SemanticPipeline(g); - sem.process(); - - ATNFactory factory = new ParserATNFactory(g); - if ( g.isLexer() ) factory = new LexerATNFactory((LexerGrammar)g); - g.atn = factory.createATN(); - - CodeGenerator gen = new CodeGenerator(g); - ST outputFileST = gen.generateParser(); - String output = outputFileST.render(); - //System.out.println(output); - String b = "#" + actionName + "#"; - int start = output.indexOf(b); - String e = "#end-" + actionName + "#"; - int end = output.indexOf(e); - String snippet = output.substring(start+b.length(),end); - assertEquals(expected, snippet); - } - if ( equeue.size()>0 ) { - System.err.println(equeue.toString()); - } - } - - protected void checkGrammarSemanticsError(ErrorQueue equeue, - GrammarSemanticsMessage expectedMessage) - throws Exception - { - ANTLRMessage foundMsg = null; - for (int i = 0; i < equeue.errors.size(); i++) { - ANTLRMessage m = equeue.errors.get(i); - if (m.getErrorType()==expectedMessage.getErrorType() ) { - foundMsg = m; - } - } - assertNotNull("no error; "+expectedMessage.getErrorType()+" expected", foundMsg); - assertTrue("error is not a GrammarSemanticsMessage", - foundMsg instanceof GrammarSemanticsMessage); - assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs())); - if ( equeue.size()!=1 ) { - System.err.println(equeue); - } - } - - protected void checkGrammarSemanticsWarning(ErrorQueue equeue, - GrammarSemanticsMessage expectedMessage) - throws Exception - { - ANTLRMessage foundMsg = null; - for (int i = 0; i < equeue.warnings.size(); i++) { - ANTLRMessage m = equeue.warnings.get(i); - if (m.getErrorType()==expectedMessage.getErrorType() ) { - foundMsg = m; - } - } - assertNotNull("no error; "+expectedMessage.getErrorType()+" expected", foundMsg); - assertTrue("error is not a GrammarSemanticsMessage", - foundMsg instanceof GrammarSemanticsMessage); - assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs())); - if ( equeue.size()!=1 ) { - System.err.println(equeue); - } - } - - protected void checkError(ErrorQueue equeue, - ANTLRMessage expectedMessage) - throws Exception - { - //System.out.println("errors="+equeue); - ANTLRMessage foundMsg = null; - for (int i = 0; i < equeue.errors.size(); i++) { - ANTLRMessage m = equeue.errors.get(i); - if (m.getErrorType()==expectedMessage.getErrorType() ) { - foundMsg = m; - } - } - assertTrue("no error; "+expectedMessage.getErrorType()+" expected", !equeue.errors.isEmpty()); - assertTrue("too many errors; "+equeue.errors, equeue.errors.size()<=1); - assertNotNull("couldn't find expected error: "+expectedMessage.getErrorType(), foundMsg); - /* - assertTrue("error is not a GrammarSemanticsMessage", - foundMsg instanceof GrammarSemanticsMessage); - */ - assertArrayEquals(expectedMessage.getArgs(), foundMsg.getArgs()); - } - - public static class FilteringTokenStream extends CommonTokenStream { - public FilteringTokenStream(TokenSource src) { super(src); } - Set hide = new HashSet(); - @Override - protected boolean sync(int i) { - if (!super.sync(i)) { - return false; - } - - Token t = get(i); - if ( hide.contains(t.getType()) ) { - ((WritableToken)t).setChannel(Token.HIDDEN_CHANNEL); - } - - return true; - } - public void setTokenTypeChannel(int ttype, int channel) { - hide.add(ttype); - } - } - - protected void mkdir(String dir) { - File f = new File(dir); - f.mkdirs(); - } - protected abstract void writeParserTestFile(String parserName, String lexerName, String listenerName, @@ -756,219 +246,13 @@ public abstract class BasePythonTest implements RuntimeTestSupport { protected abstract void writeLexerTestFile(String lexerName, boolean showDFA); - public void writeRecognizer(String parserName, String lexerName, - String listenerName, String visitorName, - String parserStartRuleName, boolean debug, boolean trace) { - if ( parserName==null ) { - writeLexerTestFile(lexerName, debug); - } - else { - writeParserTestFile(parserName, - lexerName, - listenerName, - visitorName, - parserStartRuleName, - debug, - trace); - } - } - - - protected void eraseFiles(final String filesEndingWith) { - File tmpdirF = new File(tmpdir); - String[] files = tmpdirF.list(); - for(int i = 0; files!=null && i < files.length; i++) { - if ( files[i].endsWith(filesEndingWith) ) { - new File(tmpdir+"/"+files[i]).delete(); - } - } - } - - protected void eraseFiles(File dir) { - String[] files = dir.list(); - for(int i = 0; files!=null && i < files.length; i++) { - new File(dir,files[i]).delete(); - } - } - - @Override - public void eraseTempDir() { - if(shouldEraseTempDir()) { - File tmpdirF = new File(tmpdir); - if ( tmpdirF.exists() ) { - eraseFiles(tmpdirF); - tmpdirF.delete(); - } - } - } - - private boolean shouldEraseTempDir() { - if(tmpdir==null) - return false; - String propName = getPropertyPrefix() + "-erase-test-dir"; - String prop = System.getProperty(propName); - if (prop != null && prop.length() > 0) - return Boolean.getBoolean(prop); - else - return true; - } protected void eraseTempPyCache() { - File tmpdirF = new File(tmpdir+"/__pycache__"); + File tmpdirF = new File(getTempTestDir() + "/__pycache__"); if ( tmpdirF.exists() ) { - eraseFiles(tmpdirF); + eraseFilesInDir(tmpdirF); tmpdirF.delete(); } } - public String getFirstLineOfException() { - if ( this.stderrDuringParse ==null ) { - return null; - } - String[] lines = this.stderrDuringParse.split("\n"); - String prefix="Exception in thread \"main\" "; - return lines[0].substring(prefix.length(),lines[0].length()); - } - - /** - * When looking at a result set that consists of a Map/HashTable - * we cannot rely on the output order, as the hashing algorithm or other aspects - * of the implementation may be different on differnt JDKs or platforms. Hence - * we take the Map, convert the keys to a List, sort them and Stringify the Map, which is a - * bit of a hack, but guarantees that we get the same order on all systems. We assume that - * the keys are strings. - * - * @param m The Map that contains keys we wish to return in sorted order - * @return A string that represents all the keys in sorted order. - */ - public String sortMapToString(Map m) { - // Pass in crap, and get nothing back - // - if (m == null) { - return null; - } - - System.out.println("Map toString looks like: " + m.toString()); - - // Sort the keys in the Map - // - TreeMap nset = new TreeMap(m); - - System.out.println("Tree map looks like: " + nset.toString()); - return nset.toString(); - } - - public List realElements(List elements) { - return elements.subList(Token.MIN_USER_TOKEN_TYPE, elements.size()); - } - - public void assertNotNullOrEmpty(String message, String text) { - assertNotNull(message, text); - assertFalse(message, text.isEmpty()); - } - - public void assertNotNullOrEmpty(String text) { - assertNotNull(text); - assertFalse(text.isEmpty()); - } - - public static class IntTokenStream implements TokenStream { - IntegerList types; - int p=0; - public IntTokenStream(IntegerList types) { this.types = types; } - - @Override - public void consume() { p++; } - - @Override - public int LA(int i) { return LT(i).getType(); } - - @Override - public int mark() { - return index(); - } - - @Override - public int index() { return p; } - - @Override - public void release(int marker) { - seek(marker); - } - - @Override - public void seek(int index) { - p = index; - } - - @Override - public int size() { - return types.size(); - } - - @Override - public String getSourceName() { - return null; - } - - @Override - public Token LT(int i) { - CommonToken t; - int rawIndex = p + i - 1; - if ( rawIndex>=types.size() ) t = new CommonToken(Token.EOF); - else t = new CommonToken(types.get(rawIndex)); - t.setTokenIndex(rawIndex); - return t; - } - - @Override - public Token get(int i) { - return new org.antlr.v4.runtime.CommonToken(types.get(i)); - } - - @Override - public TokenSource getTokenSource() { - return null; - } - - @Override - public String getText() { - throw new UnsupportedOperationException("can't give strings"); - } - - @Override - public String getText(Interval interval) { - throw new UnsupportedOperationException("can't give strings"); - } - - @Override - public String getText(RuleContext ctx) { - throw new UnsupportedOperationException("can't give strings"); - } - - @Override - public String getText(Token start, Token stop) { - throw new UnsupportedOperationException("can't give strings"); - } - } - - /** Sort a list */ - public > List sort(List data) { - List dup = new ArrayList(); - dup.addAll(data); - Collections.sort(dup); - return dup; - } - - /** Return map sorted by key */ - public ,V> LinkedHashMap sort(Map data) { - LinkedHashMap dup = new LinkedHashMap(); - List keys = new ArrayList(); - keys.addAll(data.keySet()); - Collections.sort(keys); - for (K k : keys) { - dup.put(k, data.get(k)); - } - return dup; - } } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/BasePython2Test.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/BasePython2Test.java index ced2665c2..7167a290e 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/BasePython2Test.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/BasePython2Test.java @@ -47,7 +47,7 @@ public class BasePython2Test extends BasePythonTest { : "") + "\n" + "if __name__ == '__main__':\n" + " main(sys.argv)\n" + "\n"); outputFileST.add("lexerName", lexerName); - writeFile(tmpdir, "Test.py", outputFileST.render()); + writeFile(getTempDirPath(), "Test.py", outputFileST.render()); } @Override @@ -105,6 +105,6 @@ public class BasePython2Test extends BasePythonTest { outputFileST.add("listenerName", listenerName); outputFileST.add("visitorName", visitorName); outputFileST.add("parserStartRuleName", parserStartRuleName); - writeFile(tmpdir, "Test.py", outputFileST.render()); + writeFile(getTempDirPath(), "Test.py", outputFileST.render()); } } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java index 0c0bc814a..834d99174 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java @@ -44,7 +44,7 @@ public class BasePython3Test extends BasePythonTest { : "") + "\n" + "if __name__ == '__main__':\n" + " main(sys.argv)\n" + "\n"); outputFileST.add("lexerName", lexerName); - writeFile(tmpdir, "Test.py", outputFileST.render()); + writeFile(getTempDirPath(), "Test.py", outputFileST.render()); } @Override @@ -102,6 +102,6 @@ public class BasePython3Test extends BasePythonTest { outputFileST.add("listenerName", listenerName); outputFileST.add("visitorName", visitorName); outputFileST.add("parserStartRuleName", parserStartRuleName); - writeFile(tmpdir, "Test.py", outputFileST.render()); + writeFile(getTempDirPath(), "Test.py", outputFileST.render()); } } diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java index 2efd08a64..d142bb5d1 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java @@ -7,13 +7,7 @@ package org.antlr.v4.test.runtime.swift; import org.antlr.v4.runtime.misc.Pair; -import org.antlr.v4.test.runtime.ErrorQueue; -import org.antlr.v4.test.runtime.RuntimeTestDescriptor; -import org.antlr.v4.test.runtime.RuntimeTestSupport; -import org.antlr.v4.test.runtime.StreamVacuum; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; +import org.antlr.v4.test.runtime.*; import org.stringtemplate.v4.ST; import java.io.BufferedReader; @@ -24,11 +18,11 @@ import java.net.URL; import java.util.*; import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString; -import static org.antlr.v4.test.runtime.BaseRuntimeTest.mkdir; +import static org.antlr.v4.test.runtime.RuntimeTestUtils.mkdir; import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; import static org.junit.Assert.assertTrue; -public class BaseSwiftTest implements RuntimeTestSupport { +public class BaseSwiftTest extends BaseRuntimeTestSupport implements RuntimeTestSupport { private static final boolean USE_ARCH_ARM64 = false; private static final boolean VERBOSE = false; @@ -81,91 +75,16 @@ public class BaseSwiftTest implements RuntimeTestSupport { }); } - public String tmpdir = null; - - /** - * If error during parser execution, store stderr here; can't return - * stdout and stderr. This doesn't trap errors from running antlr. - */ - private String stderrDuringParse; - - /** - * Errors found while running antlr - */ - private StringBuilder antlrToolErrors; + @Override + protected String getPropertyPrefix() { + return "antrl4-swift"; + } /** * Source files used in each small swift project. */ private final Set sourceFiles = new HashSet<>(); - @org.junit.Rule - public final TestRule testWatcher = new TestWatcher() { - - @Override - protected void succeeded(Description description) { - // remove tmpdir if no error. - eraseTempDir(); - } - - }; - - @Override - public void testSetUp() throws Exception { - // new output dir for each test - String propName = "antlr-swift-test-dir"; - String prop = System.getProperty(propName); - if (prop != null && prop.length() > 0) { - tmpdir = prop; - } - else { - String classSimpleName = getClass().getSimpleName(); - String threadName = Thread.currentThread().getName(); - String childPath = String.format("%s-%s-%s", classSimpleName, threadName, System.currentTimeMillis()); - tmpdir = new File(System.getProperty("java.io.tmpdir"), childPath).getAbsolutePath(); - } - antlrToolErrors = new StringBuilder(); - } - - @Override - public void testTearDown() throws Exception { - } - - @Override - public void beforeTest(RuntimeTestDescriptor descriptor) { - System.out.println(descriptor.getTestName()); - } - - @Override - public void afterTest(RuntimeTestDescriptor descriptor) { - } - - @Override - public void eraseTempDir() { - } - - @Override - public String getTmpDir() { - return tmpdir; - } - - @Override - public String getStdout() { - return null; - } - - @Override - public String getParseErrors() { - return stderrDuringParse; - } - - @Override - public String getANTLRToolErrors() { - if (antlrToolErrors.length() == 0) { - return null; - } - return antlrToolErrors.toString(); - } @Override public String execLexer(String grammarFileName, String grammarStr, String lexerName, String input, boolean showDFA) { @@ -173,12 +92,12 @@ public class BaseSwiftTest implements RuntimeTestSupport { grammarStr, null, lexerName); - writeFile(tmpdir, "input", input); + writeFile(getTempDirPath(), "input", input); writeLexerTestFile(lexerName, showDFA); addSourceFiles("main.swift"); String projectName = "testcase-" + System.currentTimeMillis(); - String projectDir = getTmpDir() + "/" + projectName; + String projectDir = new File(getTempTestDir(), projectName).getAbsolutePath(); try { buildProject(projectDir, projectName); return execTest(projectDir, projectName); @@ -196,7 +115,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { parserName, lexerName, "-visitor"); - writeFile(getTmpDir(), "input", input); + writeFile(getTempDirPath(), "input", input); return execParser(parserName, lexerName, startRuleName, @@ -207,7 +126,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { try { Pair output = runProcess(projectDir, "./.build/debug/" + projectName, "input"); if (output.b.length() > 0) { - stderrDuringParse = output.b; + setParseErrors(output.b); } String stdout = output.a; return stdout.length() > 0 ? stdout : null; @@ -227,10 +146,10 @@ public class BaseSwiftTest implements RuntimeTestSupport { mkdir(projectDir); fastFailRunProcess(projectDir, SWIFT_CMD, "package", "init", "--type", "executable"); for (String sourceFile: sourceFiles) { - String absPath = getTmpDir() + "/" + sourceFile; - fastFailRunProcess(getTmpDir(), "mv", "-f", absPath, projectDir + "/Sources/" + projectName); + String absPath = new File(getTempTestDir(), sourceFile).getAbsolutePath(); + fastFailRunProcess(getTempDirPath(), "mv", "-f", absPath, projectDir + "/Sources/" + projectName); } - fastFailRunProcess(getTmpDir(), "mv", "-f", "input", projectDir); + fastFailRunProcess(getTempDirPath(), "mv", "-f", "input", projectDir); String dylibPath = ANTLR_RUNTIME_PATH + "/.build/debug/"; // System.err.println(dylibPath); Pair buildResult = runProcess(projectDir, SWIFT_CMD, "build", @@ -356,7 +275,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { addSourceFiles("main.swift"); String projectName = "testcase-" + System.currentTimeMillis(); - String projectDir = getTmpDir() + "/" + projectName; + String projectDir = new File(getTempTestDir(), projectName).getAbsolutePath(); try { buildProject(projectDir, projectName); return execTest(projectDir, projectName); @@ -421,7 +340,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { outputFileST.add("parserName", parserName); outputFileST.add("lexerName", lexerName); outputFileST.add("parserStartRuleName", parserStartRuleName); - writeFile(tmpdir, "main.swift", outputFileST.render()); + writeFile(getTempDirPath(), "main.swift", outputFileST.render()); } private void writeLexerTestFile(String lexerName, boolean showDFA) { @@ -443,7 +362,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { (showDFA ? "print(lex.getInterpreter().getDFA(Lexer.DEFAULT_MODE).toLexerString(), terminator: \"\" )\n" : "")); outputFileST.add("lexerName", lexerName); - writeFile(tmpdir, "main.swift", outputFileST.render()); + writeFile(getTempDirPath(), "main.swift", outputFileST.render()); } /** @@ -454,7 +373,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { String parserName, String lexerName, String... extraOptions) { - ErrorQueue equeue = antlrOnString(getTmpDir(), "Swift", grammarFileName, grammarStr, false, extraOptions); + ErrorQueue equeue = antlrOnString(getTempDirPath(), "Swift", grammarFileName, grammarStr, false, extraOptions); assertTrue(equeue.errors.isEmpty()); // System.out.println(getTmpDir()); @@ -480,4 +399,5 @@ public class BaseSwiftTest implements RuntimeTestSupport { } addSourceFiles(files.toArray(new String[0])); } + } diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/BaseJavaToolTest.java b/tool-testsuite/test/org/antlr/v4/test/tool/BaseJavaToolTest.java index 6c2dd465f..f3d0f7177 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/BaseJavaToolTest.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/BaseJavaToolTest.java @@ -16,6 +16,7 @@ import java.io.File; import static org.junit.Assert.assertEquals; public class BaseJavaToolTest extends BaseJavaTest { + public void testErrors(String[] pairs, boolean printTree) { for (int i = 0; i < pairs.length; i+=2) { String grammarStr = pairs[i]; @@ -23,10 +24,10 @@ public class BaseJavaToolTest extends BaseJavaTest { String[] lines = grammarStr.split("\n"); String fileName = getFilenameFromFirstLineOfGrammar(lines[0]); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, null, fileName, grammarStr, false); // use default language target in case test overrides + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), null, fileName, grammarStr, false); // use default language target in case test overrides String actual = equeue.toString(true); - actual = actual.replace(tmpdir + File.separator, ""); + actual = actual.replace(getTempDirPath() + File.separator, ""); // System.err.println(actual); String msg = grammarStr; msg = msg.replace("\n","\\n"); diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNInterpreter.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNInterpreter.java index 201463ed4..758d56518 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNInterpreter.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNInterpreter.java @@ -9,12 +9,14 @@ package org.antlr.v4.test.tool; import org.antlr.v4.automata.ParserATNFactory; import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.NoViableAltException; +import org.antlr.v4.runtime.TokenStream; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.atn.BlockStartState; import org.antlr.v4.runtime.atn.LexerATNSimulator; import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.misc.IntegerList; +import org.antlr.v4.test.runtime.MockIntTokenStream; import org.antlr.v4.tool.DOTGenerator; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.LexerGrammar; @@ -22,6 +24,7 @@ import org.antlr.v4.tool.Rule; import org.junit.Before; import org.junit.Test; +import static org.antlr.v4.test.runtime.RuntimeTestUtils.getTokenTypesViaATN; import static org.junit.Assert.assertEquals; // NOTICE: TOKENS IN LEXER, PARSER MUST BE SAME OR TOKEN TYPE MISMATCH @@ -373,7 +376,7 @@ public class TestATNInterpreter extends BaseJavaToolTest { ParserATNFactory f = new ParserATNFactory(g); ATN atn = f.createATN(); - IntTokenStream input = new IntTokenStream(types); + TokenStream input = new MockIntTokenStream(types); // System.out.println("input="+input.types); ParserInterpreterForTesting interp = new ParserInterpreterForTesting(g, input); ATNState startState = atn.ruleToStartState[g.getRule("a").index]; diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNLexerInterpreter.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNLexerInterpreter.java index 0d06a031b..4cd34deec 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNLexerInterpreter.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNLexerInterpreter.java @@ -11,6 +11,7 @@ import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.misc.Utils; +import org.antlr.v4.test.runtime.RuntimeTestUtils; import org.antlr.v4.tool.DOTGenerator; import org.antlr.v4.tool.LexerGrammar; import org.junit.Before; @@ -386,7 +387,7 @@ public class TestATNLexerInterpreter extends BaseJavaToolTest { DOTGenerator dot = new DOTGenerator(lg); // System.out.println(dot.getDOT(startState, true)); - List tokenTypes = getTokenTypes(lg, atn, input); + List tokenTypes = RuntimeTestUtils.getTokenTypes(lg, atn, input); String result = Utils.join(tokenTypes.iterator(), ", "); // System.out.println(tokenTypes); diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNParserPrediction.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNParserPrediction.java index 9e58fe9d7..3fb0dc9e6 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNParserPrediction.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNParserPrediction.java @@ -17,6 +17,7 @@ import org.antlr.v4.runtime.atn.LexerATNSimulator; import org.antlr.v4.runtime.atn.PredictionContextCache; import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.misc.IntegerList; +import org.antlr.v4.test.runtime.MockIntTokenStream; import org.antlr.v4.tool.DOTGenerator; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.LeftRecursiveRule; @@ -27,6 +28,7 @@ import org.junit.Test; import java.util.Arrays; +import static org.antlr.v4.test.runtime.RuntimeTestUtils.getTokenTypesViaATN; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -526,7 +528,7 @@ public class TestATNParserPrediction extends BaseJavaToolTest { // Check ATN prediction // ParserATNSimulator interp = new ParserATNSimulator(atn); - TokenStream input = new IntTokenStream(types); + TokenStream input = new MockIntTokenStream(types); ParserInterpreterForTesting interp = new ParserInterpreterForTesting(g, input); int alt = interp.adaptivePredict(input, decision, ParserRuleContext.EMPTY); @@ -559,7 +561,7 @@ public class TestATNParserPrediction extends BaseJavaToolTest { // Check DFA IntegerList types = getTokenTypesViaATN(inputString[i], lexInterp); // System.out.println(types); - TokenStream input = new IntTokenStream(types); + TokenStream input = new MockIntTokenStream(types); try { interp.adaptivePredict(input, decision, ParserRuleContext.EMPTY); } diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java index 03368785b..330ca7fa7 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java @@ -8,6 +8,7 @@ package org.antlr.v4.test.tool; import org.antlr.v4.test.runtime.BaseRuntimeTest; import org.antlr.v4.test.runtime.ErrorQueue; +import org.antlr.v4.test.runtime.RuntimeTestUtils; import org.antlr.v4.tool.ANTLRMessage; import org.antlr.v4.tool.ErrorType; import org.antlr.v4.tool.Grammar; @@ -18,6 +19,7 @@ import org.junit.Test; import java.io.File; import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; +import static org.antlr.v4.test.runtime.RuntimeTestUtils.sort; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -34,9 +36,9 @@ public class TestCompositeGrammars extends BaseJavaToolTest { String slave = "parser grammar S;\n" + "a : B {System.out.println(\"S.a\");} ;\n"; - BaseRuntimeTest.mkdir(tmpdir); - String subdir = tmpdir + "/sub"; - BaseRuntimeTest.mkdir(subdir); + RuntimeTestUtils.mkdir(getTempDirPath()); + String subdir = getTempDirPath() + PATH_SEP + "sub"; + RuntimeTestUtils.mkdir(subdir); writeFile(subdir, "S.g4", slave); String master = "grammar M;\n" + @@ -44,65 +46,65 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "s : a ;\n" + "B : 'b' ;" + // defines B from inherited token space "WS : (' '|'\\n') -> skip ;\n" ; - writeFile(tmpdir, "M.g4", master); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", subdir); + writeFile(getTempDirPath(), "M.g4", master); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "M.g4", false, "-lib", subdir); assertEquals(0, equeue.size()); } // Test for https://github.com/antlr/antlr4/issues/1317 @Test public void testImportSelfLoop() throws Exception { - BaseRuntimeTest.mkdir(tmpdir); + RuntimeTestUtils.mkdir(getTempDirPath()); String master = "grammar M;\n" + "import M;\n" + "s : 'a' ;\n"; - writeFile(tmpdir, "M.g4", master); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + writeFile(getTempDirPath(), "M.g4", master); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "M.g4", false, "-lib", getTempDirPath()); assertEquals(0, equeue.size()); } @Test public void testImportIntoLexerGrammar() throws Exception { - BaseRuntimeTest.mkdir(tmpdir); + RuntimeTestUtils.mkdir(getTempDirPath()); String master = "lexer grammar M;\n" + "import S;\n" + "A : 'a';\n" + "B : 'b';\n"; - writeFile(tmpdir, "M.g4", master); + writeFile(getTempDirPath(), "M.g4", master); - String slave = + String slave = "lexer grammar S;\n" + "C : 'c';\n"; - writeFile(tmpdir, "S.g4", slave); + writeFile(getTempDirPath(), "S.g4", slave); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "M.g4", false, "-lib", getTempDirPath()); assertEquals(0, equeue.errors.size()); } @Test public void testImportModesIntoLexerGrammar() throws Exception { - BaseRuntimeTest.mkdir(tmpdir); + RuntimeTestUtils.mkdir(getTempDirPath()); String master = "lexer grammar M;\n" + "import S;\n" + "A : 'a' -> pushMode(X);\n" + "B : 'b';\n"; - writeFile(tmpdir, "M.g4", master); + writeFile(getTempDirPath(), "M.g4", master); - String slave = + String slave = "lexer grammar S;\n" + "D : 'd';\n" + "mode X;\n" + "C : 'c' -> popMode;\n"; - writeFile(tmpdir, "S.g4", slave); + writeFile(getTempDirPath(), "S.g4", slave); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "M.g4", false, "-lib", getTempDirPath()); assertEquals(0, equeue.errors.size()); } - + @Test public void testImportChannelsIntoLexerGrammar() throws Exception { - BaseRuntimeTest.mkdir(tmpdir); + RuntimeTestUtils.mkdir(getTempDirPath()); String master = "lexer grammar M;\n" + @@ -110,19 +112,19 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "channels {CH_A, CH_B}\n" + "A : 'a' -> channel(CH_A);\n" + "B : 'b' -> channel(CH_B);\n"; - writeFile(tmpdir, "M.g4", master); + writeFile(getTempDirPath(), "M.g4", master); - String slave = + String slave = "lexer grammar S;\n" + "C : 'c';\n"; - writeFile(tmpdir, "S.g4", slave); + writeFile(getTempDirPath(), "S.g4", slave); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "M.g4", false, "-lib", getTempDirPath()); assertEquals(0, equeue.errors.size()); } - + @Test public void testImportMixedChannelsIntoLexerGrammar() throws Exception { - BaseRuntimeTest.mkdir(tmpdir); + RuntimeTestUtils.mkdir(getTempDirPath()); String master = "lexer grammar M;\n" + @@ -130,20 +132,20 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "channels {CH_A, CH_B}\n" + "A : 'a' -> channel(CH_A);\n" + "B : 'b' -> channel(CH_B);\n"; - writeFile(tmpdir, "M.g4", master); + writeFile(getTempDirPath(), "M.g4", master); - String slave = + String slave = "lexer grammar S;\n" + "channels {CH_C}\n" + "C : 'c' -> channel(CH_C);\n"; - writeFile(tmpdir, "S.g4", slave); + writeFile(getTempDirPath(), "S.g4", slave); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "M.g4", false, "-lib", getTempDirPath()); assertEquals(0, equeue.errors.size()); } @Test public void testImportClashingChannelsIntoLexerGrammar() throws Exception { - BaseRuntimeTest.mkdir(tmpdir); + RuntimeTestUtils.mkdir(getTempDirPath()); String master = "lexer grammar M;\n" + @@ -152,20 +154,20 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "A : 'a' -> channel(CH_A);\n" + "B : 'b' -> channel(CH_B);\n" + "C : 'C' -> channel(CH_C);\n"; - writeFile(tmpdir, "M.g4", master); + writeFile(getTempDirPath(), "M.g4", master); - String slave = + String slave = "lexer grammar S;\n" + "channels {CH_C}\n" + "C : 'c' -> channel(CH_C);\n"; - writeFile(tmpdir, "S.g4", slave); + writeFile(getTempDirPath(), "S.g4", slave); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "M.g4", false, "-lib", getTempDirPath()); assertEquals(0, equeue.errors.size()); - } - + } + @Test public void testMergeModesIntoLexerGrammar() throws Exception { - BaseRuntimeTest.mkdir(tmpdir); + RuntimeTestUtils.mkdir(getTempDirPath()); String master = "lexer grammar M;\n" + @@ -173,43 +175,43 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "A : 'a' -> pushMode(X);\n" + "mode X;\n" + "B : 'b';\n"; - writeFile(tmpdir, "M.g4", master); + writeFile(getTempDirPath(), "M.g4", master); - String slave = + String slave = "lexer grammar S;\n" + "D : 'd';\n" + "mode X;\n" + "C : 'c' -> popMode;\n"; - writeFile(tmpdir, "S.g4", slave); + writeFile(getTempDirPath(), "S.g4", slave); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "M.g4", false, "-lib", getTempDirPath()); assertEquals(0, equeue.errors.size()); } - + @Test public void testEmptyModesInLexerGrammar() throws Exception { - BaseRuntimeTest.mkdir(tmpdir); + RuntimeTestUtils.mkdir(getTempDirPath()); String master = "lexer grammar M;\n" + "import S;\n" + "A : 'a';\n" + - "C : 'e';\n" + + "C : 'e';\n" + "B : 'b';\n"; - writeFile(tmpdir, "M.g4", master); + writeFile(getTempDirPath(), "M.g4", master); - String slave = + String slave = "lexer grammar S;\n" + "D : 'd';\n" + "mode X;\n" + "C : 'c' -> popMode;\n"; - writeFile(tmpdir, "S.g4", slave); + writeFile(getTempDirPath(), "S.g4", slave); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "M.g4", false, "-lib", getTempDirPath()); assertEquals(0, equeue.errors.size()); } - + @Test public void testCombinedGrammarImportsModalLexerGrammar() throws Exception { - BaseRuntimeTest.mkdir(tmpdir); + RuntimeTestUtils.mkdir(getTempDirPath()); String master = "grammar M;\n" + @@ -217,16 +219,16 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "A : 'a';\n" + "B : 'b';\n" + "r : A B;\n"; - writeFile(tmpdir, "M.g4", master); + writeFile(getTempDirPath(), "M.g4", master); - String slave = + String slave = "lexer grammar S;\n" + "D : 'd';\n" + "mode X;\n" + "C : 'c' -> popMode;\n"; - writeFile(tmpdir, "S.g4", slave); + writeFile(getTempDirPath(), "S.g4", slave); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "M.g4", false, "-lib", getTempDirPath()); assertEquals(1, equeue.errors.size()); ANTLRMessage msg = equeue.errors.get(0); assertEquals(ErrorType.MODE_NOT_IN_LEXER, msg.getErrorType()); @@ -234,7 +236,7 @@ public class TestCompositeGrammars extends BaseJavaToolTest { assertEquals(3, msg.line); assertEquals(5, msg.charPosition); assertEquals("M.g4", new File(msg.fileName).getName()); - } + } @Test public void testDelegatesSeeSameTokenType() throws Exception { String slaveS = @@ -246,9 +248,9 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "tokens { C, B, A } // reverse order\n"+ "y : A ;\n"; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "S.g4", slaveS); - writeFile(tmpdir, "T.g4", slaveT); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "S.g4", slaveS); + writeFile(getTempDirPath(), "T.g4", slaveT); String master = "// The lexer will create rules to match letters a, b, c.\n"+ @@ -268,9 +270,9 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "A : 'a' ;\n"+ "C : 'c' ;\n"+ "WS : (' '|'\\n') -> skip ;\n"; - writeFile(tmpdir, "M.g4", master); + writeFile(getTempDirPath(), "M.g4", master); ErrorQueue equeue = new ErrorQueue(); - Grammar g = new Grammar(tmpdir+"/M.g4", master, equeue); + Grammar g = new Grammar(getTempDirPath()+"/M.g4", master, equeue); String expectedTokenIDToTypeMap = "{EOF=-1, B=1, A=2, C=3, WS=4}"; String expectedStringLiteralToTypeMap = "{'a'=2, 'b'=1, 'c'=3}"; String expectedTypeToTokenList = "[B, A, C, WS]"; @@ -284,13 +286,13 @@ public class TestCompositeGrammars extends BaseJavaToolTest { String slave = "parser grammar S;\n" + "a : 'a' | c;\n"; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "S.g4", slave); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "S.g4", slave); String master = "grammar M;\n" + "import S;\n"; - writeFile(tmpdir, "M.g4", master); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + writeFile(getTempDirPath(), "M.g4", master); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "M.g4", false, "-lib", getTempDirPath()); ANTLRMessage msg = equeue.errors.get(0); assertEquals(ErrorType.UNDEFINED_RULE_REF, msg.getErrorType()); assertEquals("c", msg.getArgs()[0]); @@ -303,9 +305,9 @@ public class TestCompositeGrammars extends BaseJavaToolTest { String slave = "parser grammar S;\n" + "a : B {System.out.println(\"S.a\");} ;\n"; - BaseRuntimeTest.mkdir(tmpdir); - String outdir = tmpdir + "/out"; - BaseRuntimeTest.mkdir(outdir); + RuntimeTestUtils.mkdir(getTempDirPath()); + String outdir = getTempDirPath() + "/out"; + RuntimeTestUtils.mkdir(outdir); writeFile(outdir, "S.g4", slave); String master = "grammar M;\n" + @@ -313,8 +315,8 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "s : a ;\n" + "B : 'b' ;" + // defines B from inherited token space "WS : (' '|'\\n') -> skip ;\n" ; - writeFile(tmpdir, "M.g4", master); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-o", outdir); + writeFile(getTempDirPath(), "M.g4", master); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "M.g4", false, "-o", outdir); assertEquals(ErrorType.CANNOT_FIND_IMPORTED_GRAMMAR, equeue.errors.get(0).getErrorType()); } @@ -322,9 +324,9 @@ public class TestCompositeGrammars extends BaseJavaToolTest { String slave = "parser grammar S;\n" + "a : B {System.out.println(\"S.a\");} ;\n"; - BaseRuntimeTest.mkdir(tmpdir); - String subdir = tmpdir + "/sub"; - BaseRuntimeTest.mkdir(subdir); + RuntimeTestUtils.mkdir(getTempDirPath()); + String subdir = getTempDirPath() + "/sub"; + RuntimeTestUtils.mkdir(subdir); writeFile(subdir, "S.g4", slave); String master = "grammar M;\n" + @@ -332,10 +334,10 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "s : a ;\n" + "B : 'b' ;" + // defines B from inherited token space "WS : (' '|'\\n') -> skip ;\n" ; - writeFile(tmpdir, "M.g4", master); - String outdir = tmpdir + "/out"; - BaseRuntimeTest.mkdir(outdir); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-o", outdir, "-lib", subdir); + writeFile(getTempDirPath(), "M.g4", master); + String outdir = getTempDirPath() + "/out"; + RuntimeTestUtils.mkdir(outdir); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "M.g4", false, "-o", outdir, "-lib", subdir); assertEquals(0, equeue.size()); } @@ -343,26 +345,26 @@ public class TestCompositeGrammars extends BaseJavaToolTest { String slave = "parser grammar S;\n" + "a : B {System.out.println(\"S.a\");} ;\n"; - BaseRuntimeTest.mkdir(tmpdir); - String subdir = tmpdir + "/sub"; - BaseRuntimeTest.mkdir(subdir); + RuntimeTestUtils.mkdir(getTempDirPath()); + String subdir = getTempDirPath() + "/sub"; + RuntimeTestUtils.mkdir(subdir); writeFile(subdir, "S.g4", slave); String parser = "parser grammar MParser;\n" + "import S;\n" + "options {tokenVocab=MLexer;}\n" + "s : a ;\n"; - writeFile(tmpdir, "MParser.g4", parser); + writeFile(getTempDirPath(), "MParser.g4", parser); String lexer = "lexer grammar MLexer;\n" + "B : 'b' ;" + // defines B from inherited token space "WS : (' '|'\\n') -> skip ;\n" ; - writeFile(tmpdir, "MLexer.g4", lexer); - String outdir = tmpdir + "/out"; - BaseRuntimeTest.mkdir(outdir); - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "MLexer.g4", false, "-o", outdir); + writeFile(getTempDirPath(), "MLexer.g4", lexer); + String outdir = getTempDirPath() + "/out"; + RuntimeTestUtils.mkdir(outdir); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "MLexer.g4", false, "-o", outdir); assertEquals(0, equeue.size()); - equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "MParser.g4", false, "-o", outdir, "-lib", subdir); + equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "MParser.g4", false, "-o", outdir, "-lib", subdir); assertEquals(0, equeue.size()); } @@ -373,16 +375,16 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "options {tokenVocab=whatever;}\n" + "tokens { A }\n" + "x : A {System.out.println(\"S.x\");} ;\n"; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "S.g4", slave); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "S.g4", slave); String master = "grammar M;\n" + "import S;\n" + "s : x ;\n" + "WS : (' '|'\\n') -> skip ;\n" ; - writeFile(tmpdir, "M.g4", master); - Grammar g = new Grammar(tmpdir+"/M.g4", master, equeue); + writeFile(getTempDirPath(), "M.g4", master); + Grammar g = new Grammar(getTempDirPath()+"/M.g4", master, equeue); Object expectedArg = "S"; ErrorType expectedMsgID = ErrorType.OPTIONS_IN_DELEGATE; @@ -399,16 +401,16 @@ public class TestCompositeGrammars extends BaseJavaToolTest { String slave = "parser grammar S;\n" + "options {toke\n"; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "S.g4", slave); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "S.g4", slave); String master = "grammar M;\n" + "import S;\n" + "s : x ;\n" + "WS : (' '|'\\n') -> skip ;\n" ; - writeFile(tmpdir, "M.g4", master); - /*Grammar g =*/ new Grammar(tmpdir+"/M.g4", master, equeue); + writeFile(getTempDirPath(), "M.g4", master); + /*Grammar g =*/ new Grammar(getTempDirPath()+"/M.g4", master, equeue); assertEquals(ErrorType.SYNTAX_ERROR, equeue.errors.get(0).getErrorType()); } @@ -419,21 +421,21 @@ public class TestCompositeGrammars extends BaseJavaToolTest { String slave = "parser grammar T;\n" + "a : T ;\n" ; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "T.g4", slave); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "T.g4", slave); String slave2 = "parser grammar S;\n" + "import T;\n" + "a : S ;\n" ; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "S.g4", slave2); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "S.g4", slave2); String master = "grammar M;\n" + "import S;\n" + "a : M ;\n" ; - writeFile(tmpdir, "M.g4", master); - Grammar g = new Grammar(tmpdir+"/M.g4", master, equeue); + writeFile(getTempDirPath(), "M.g4", master); + Grammar g = new Grammar(getTempDirPath()+"/M.g4", master, equeue); String expectedTokenIDToTypeMap = "{EOF=-1, M=1}"; // S and T aren't imported; overridden String expectedStringLiteralToTypeMap = "{}"; @@ -459,43 +461,43 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "parser grammar T;\n" + "tokens{T}\n" + "x : T ;\n" ; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "T.g4", slave); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "T.g4", slave); slave = "parser grammar S;\n" + "import T;\n" + "tokens{S}\n" + "y : S ;\n" ; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "S.g4", slave); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "S.g4", slave); slave = "parser grammar C;\n" + "tokens{C}\n" + "i : C ;\n" ; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "C.g4", slave); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "C.g4", slave); slave = "parser grammar B;\n" + "tokens{B}\n" + "j : B ;\n" ; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "B.g4", slave); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "B.g4", slave); slave = "parser grammar A;\n" + "import B,C;\n" + "tokens{A}\n" + "k : A ;\n" ; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "A.g4", slave); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "A.g4", slave); String master = "grammar M;\n" + "import S,A;\n" + "tokens{M}\n" + "a : M ;\n" ; - writeFile(tmpdir, "M.g4", master); - Grammar g = new Grammar(tmpdir+"/M.g4", master, equeue); + writeFile(getTempDirPath(), "M.g4", master); + Grammar g = new Grammar(getTempDirPath()+"/M.g4", master, equeue); assertEquals("[]", equeue.errors.toString()); assertEquals("[]", equeue.warnings.toString()); @@ -520,21 +522,21 @@ public class TestCompositeGrammars extends BaseJavaToolTest { String slave = "parser grammar T;\n" + "x : T ;\n" ; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "T.g4", slave); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "T.g4", slave); String slave2 = "parser grammar S;\n" + // A, B, C token type order "import T;\n" + "a : S ;\n" ; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "S.g4", slave2); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "S.g4", slave2); String master = "grammar M;\n" + "import S;\n" + "a : M x ;\n" ; // x MUST BE VISIBLE TO M - writeFile(tmpdir, "M.g4", master); - Grammar g = new Grammar(tmpdir+"/M.g4", master, equeue); + writeFile(getTempDirPath(), "M.g4", master); + Grammar g = new Grammar(getTempDirPath()+"/M.g4", master, equeue); String expectedTokenIDToTypeMap = "{EOF=-1, M=1, T=2}"; String expectedStringLiteralToTypeMap = "{}"; @@ -558,30 +560,30 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "T2: '2';\n" + "T3: '3';\n" + "T4: '4';\n" ; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "L.g4", gstr); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "L.g4", gstr); gstr = "parser grammar G1;\n" + "s: a | b;\n" + "a: T1;\n" + "b: T2;\n" ; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "G1.g4", gstr); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "G1.g4", gstr); gstr = "parser grammar G2;\n" + "import G1;\n" + "a: T3;\n" ; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "G2.g4", gstr); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "G2.g4", gstr); String G3str = "grammar G3;\n" + "import G2;\n" + "b: T4;\n" ; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "G3.g4", G3str); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "G3.g4", G3str); - Grammar g = new Grammar(tmpdir+"/G3.g4", G3str, equeue); + Grammar g = new Grammar(getTempDirPath()+"/G3.g4", G3str, equeue); String expectedTokenIDToTypeMap = "{EOF=-1, T4=1, T3=2}"; String expectedStringLiteralToTypeMap = "{}"; @@ -605,8 +607,8 @@ public class TestCompositeGrammars extends BaseJavaToolTest { String slave = "parser grammar S;\n" + "a : B {System.out.print(\"S.a\");} ;\n"; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "S.g4", slave); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "S.g4", slave); String master = "grammar M;\n" + "import S;\n" + @@ -614,7 +616,7 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "s : a ;\n" + "B : 'b' ;" + // defines B from inherited token space "WS : (' '|'\\n') -> skip ;\n" ; - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", master, false); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "M.g4", master, false); int expecting = 0; // should be ok assertEquals(expecting, equeue.errors.size()); } @@ -633,12 +635,12 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "grammar NewJava;\n" + "import Java;\n"; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "Java.g4", slave); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "Java.g4", slave); String found = execParser("NewJava.g4", master, "NewJavaParser", "NewJavaLexer", null, null, "compilationUnit", "package Foo;", debug); assertEquals(null, found); - assertNull(stderrDuringParse); + assertNull(getParseErrors()); } /** @@ -661,11 +663,11 @@ public class TestCompositeGrammars extends BaseJavaToolTest { "import Java;\n" + "s : e ;\n"; - BaseRuntimeTest.mkdir(tmpdir); - writeFile(tmpdir, "Java.g4", slave); + RuntimeTestUtils.mkdir(getTempDirPath()); + writeFile(getTempDirPath(), "Java.g4", slave); String found = execParser("T.g4", master, "TParser", "TLexer", null, null, "s", "a=b", debug); assertEquals(null, found); - assertNull(stderrDuringParse); + assertNull(getParseErrors()); } } diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestDollarParser.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestDollarParser.java index 6103e4423..91efcabe1 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestDollarParser.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestDollarParser.java @@ -28,7 +28,7 @@ public class TestDollarParser extends BaseJavaToolTest { String found = execParser("T.g4", grammar, "TParser", "TLexer", null, null, "a", "x", true); assertTrue(found.indexOf(this.getClass().getSimpleName())>=0); - assertNull(this.stderrDuringParse); + assertNull(getParseErrors()); } } diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestParserExec.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestParserExec.java index 14005c3c0..0d0265d82 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestParserExec.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestParserExec.java @@ -75,7 +75,7 @@ public class TestParserExec extends BaseJavaToolTest { "s1-INT->s2\n" + "s2-EOF->:s3=>1\n"; // Must point at accept state assertEquals(expecting, result); - assertNull(this.stderrDuringParse); + assertNull(getParseErrors()); } /** @@ -88,7 +88,7 @@ public class TestParserExec extends BaseJavaToolTest { String grammar = load("Psl.g4", "UTF-8"); String found = execParser("Psl.g4", grammar, "PslParser", "PslLexer", null, null, "floating_constant", " . 234", false); assertEquals(null, found); - assertEquals("line 1:6 rule floating_constant DEC:A floating-point constant cannot have internal white space\n", stderrDuringParse); + assertEquals("line 1:6 rule floating_constant DEC:A floating-point constant cannot have internal white space\n", getParseErrors()); } /** @@ -129,7 +129,7 @@ public class TestParserExec extends BaseJavaToolTest { String found = execParser("ModeTagsParser.g4", parserGrammar, "ModeTagsParser", "ModeTagsLexer", null, null, "file", "", false); assertEquals(null, found); - assertNull(stderrDuringParse); + assertNull(getParseErrors()); } /** @@ -158,6 +158,6 @@ public class TestParserExec extends BaseJavaToolTest { String found = execParser("Data.g4", grammar, "DataParser", "DataLexer", null, null, "file", input, false); assertEquals("6\n", found); - assertNull(stderrDuringParse); + assertNull(getParseErrors()); } } diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestParserProfiler.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestParserProfiler.java index a5cd58a74..fe89349e0 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestParserProfiler.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestParserProfiler.java @@ -223,7 +223,7 @@ public class TestParserProfiler extends BaseJavaToolTest { " {decision=1, contextSensitivities=0, errors=0, ambiguities=0, SLL_lookahead=6, " + "SLL_ATNTransitions=3, SLL_DFATransitions=3, LL_Fallback=0, LL_lookahead=0, LL_ATNTransitions=0}]\n"; assertEquals(expecting, found); - assertEquals(null, stderrDuringParse); + assertEquals(null, getParseErrors()); } public DecisionInfo[] interpAndGetDecisionInfo( diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestPerformance.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestPerformance.java index b5bd60662..823927a0d 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestPerformance.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestPerformance.java @@ -42,6 +42,7 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.antlr.v4.runtime.tree.TerminalNode; import org.antlr.v4.test.runtime.BaseRuntimeTest; import org.antlr.v4.test.runtime.ErrorQueue; +import org.antlr.v4.test.runtime.RuntimeTestUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -701,20 +702,20 @@ public class TestPerformance extends BaseJavaToolTest { builder.append(", Grammar=").append(USE_LR_GRAMMAR ? "LR" : "Standard"); builder.append(", ForceAtn=").append(FORCE_ATN); - builder.append(newline); + builder.append(NEW_LINE); builder.append("Op=Lex").append(RUN_PARSER ? "+Parse" : " only"); builder.append(", Strategy=").append(BAIL_ON_ERROR ? BailErrorStrategy.class.getSimpleName() : DefaultErrorStrategy.class.getSimpleName()); builder.append(", BuildParseTree=").append(BUILD_PARSE_TREES); builder.append(", WalkBlankListener=").append(BLANK_LISTENER); - builder.append(newline); + builder.append(NEW_LINE); builder.append("Lexer=").append(REUSE_LEXER ? "setInputStream" : "newInstance"); builder.append(", Parser=").append(REUSE_PARSER ? "setInputStream" : "newInstance"); builder.append(", AfterPass=").append(CLEAR_DFA ? "newInstance" : "setInputStream"); - builder.append(newline); + builder.append(NEW_LINE); return builder.toString(); } @@ -1135,7 +1136,7 @@ public class TestPerformance extends BaseJavaToolTest { protected ParserFactory getParserFactory(String lexerName, String parserName, String listenerName, final String entryPoint) { try { - ClassLoader loader = new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() }, ClassLoader.getSystemClassLoader()); + ClassLoader loader = new URLClassLoader(new URL[] { getTempTestDir().toURI().toURL() }, ClassLoader.getSystemClassLoader()); final Class lexerClass = loader.loadClass(lexerName).asSubclass(Lexer.class); final Class parserClass = loader.loadClass(parserName).asSubclass(Parser.class); final Class listenerClass = loader.loadClass(listenerName).asSubclass(ParseTreeListener.class); @@ -1952,7 +1953,7 @@ public class TestPerformance extends BaseJavaToolTest { "\n" + "rule_%d_%d : EOF;\n"; - BaseRuntimeTest.mkdir(tmpdir); + RuntimeTestUtils.mkdir(getTempDirPath()); long startTime = System.nanoTime(); @@ -1960,14 +1961,14 @@ public class TestPerformance extends BaseJavaToolTest { for (int level = 0; level < levels; level++) { String leafPrefix = level == levels - 1 ? "//" : ""; String grammar1 = String.format(grammarFormat, level, 1, leafPrefix, level + 1, level + 1, level, 1); - writeFile(tmpdir, "Level_" + level + "_1.g4", grammar1); + writeFile(getTempDirPath(), "Level_" + level + "_1.g4", grammar1); if (level > 0) { String grammar2 = String.format(grammarFormat, level, 2, leafPrefix, level + 1, level + 1, level, 1); - writeFile(tmpdir, "Level_" + level + "_2.g4", grammar2); + writeFile(getTempDirPath(), "Level_" + level + "_2.g4", grammar2); } } - ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "Level_0_1.g4", false); + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", "Level_0_1.g4", false); Assert.assertTrue(equeue.errors.isEmpty()); long endTime = System.nanoTime();