jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Merge branch 'master' into master

This commit is contained in:
Terence Parr 2021-04-08 09:18:40 -07:00 committed by GitHub
parent 3b3da29d45 b71f6894c7
commit 697074c3e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
524 changed files with 23229 additions and 13892 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
.appveyor/workflow.yml Normal file
View File

@ -0,0 +1,128 @@
environment:
matrix:
- job_name: java-tool-and-runtime
- job_name: csharp-runtime
job_depends_on: java-tool-and-runtime
- job_name: dart-runtime
job_depends_on: java-tool-and-runtime
- job_name: go-runtime
job_depends_on: java-tool-and-runtime
- job_name: javascript-runtime
job_depends_on: java-tool-and-runtime
- job_name: php-runtime
job_depends_on: java-tool-and-runtime
- job_name: python2-runtime
job_depends_on: java-tool-and-runtime
- job_name: python3-runtime
job_depends_on: java-tool-and-runtime
matrix:
fast_finish: false
version: '4.9.1-SNAPSHOT+AppVeyor.{build}'
cache:
- '%USERPROFILE%\.m2'
- '%USERPROFILE%\.nuget\packages -> **\project.json'
image: Visual Studio 2019
# not using MSBuild
build: off
for:
- matrix:
only:
- job_name: java-tool-and-runtime
build_script:
- mvn -q -DskipTests install --batch-mode
test_script:
- cd tool-testsuite
- mvn -q test
- cd ..\runtime-testsuite
- mvn -q -Dtest=java.* test
- matrix:
only:
- job_name: csharp-runtime
build_script:
- mvn -q -DskipTests install --batch-mode
- dotnet build runtime/CSharp/src/Antlr4.csproj -c Release
after_build:
- dotnet pack runtime/CSharp/src/Antlr4.csproj -c Release
test_script:
- cd runtime-testsuite
- mvn -q -Dtest=csharp.* test
artifacts:
- path: 'runtime\**\*.nupkg'
name: NuGet
- matrix:
only:
- job_name: dart-runtime
install:
- cinst -y dart-sdk --version=2.8.4
build_script:
- mvn -q -DskipTests install --batch-mode
test_script:
- cd runtime-testsuite
- mvn -q -Dtest=dart.* test -Dantlr-dart-dart="C:\tools\dart-sdk\bin\dart.exe" -Dantlr-dart-pub="C:\tools\dart-sdk\bin\pub.bat" -Dantlr-dart-dart2native="C:\tools\dart-sdk\bin\dart2native.bat"
- matrix:
only:
- job_name: go-runtime
build_script:
- mvn -q -DskipTests install --batch-mode
test_script:
- cd runtime-testsuite
- mvn -q -Dtest=go.* test
- matrix:
only:
- job_name: javascript-runtime
install:
- cinst nodejs.install
- node --version
- npm --version
- npm install -g yarn@v1.22.10
build_script:
- cd runtime\JavaScript\
- npm install
- npm link
- cd ..\..
- mvn -q -DskipTests install --batch-mode
test_script:
- cd runtime\JavaScript\
- yarn test
- cd ..\..
- cd runtime-testsuite
- mvn -q -Dtest=javascript.* test -Dantlr-javascript-npm="C:\Program Files\nodejs\npm.cmd" -Dantlr-javascript-nodejs="C:\Program Files\nodejs\node.exe"
- matrix:
only:
- job_name: php-runtime
install:
- git clone https://github.com/antlr/antlr-php-runtime.git
- mv antlr-php-runtime runtime/PHP
- cinst -y php --params "/InstallDir:C:\tools\php"
- cinst -y composer
build_script:
- mvn -q -DskipTests install --batch-mode
test_script:
- cd runtime-testsuite
- mvn -q -Dtest=php.* test -Dantlr-php-php="C:\tools\php\php.exe"
- matrix:
only:
- job_name: python2-runtime
build_script:
- mvn -q -DskipTests install --batch-mode
test_script:
- cd runtime-testsuite
- mvn -q -Dtest=python2.* test -Dantlr-python2-python="C:\Python27\python.exe"
- matrix:
only:
- job_name: python3-runtime
build_script:
- mvn -q -DskipTests install --batch-mode
test_script:
- cd runtime-testsuite
- mvn -q -Dtest=python3.* test -Dantlr-python3-python="C:\Python35\python.exe"

66
.circleci/config.yml Normal file
View File

@ -0,0 +1,66 @@
version: 2.1
jobs:
test_tool_and_runtime_java:
docker:
- image: cimg/openjdk:8.0
steps:
- checkout
- run:
name: build tool
command: mvn -B -V -DskipTests=true -Dmaven.javadoc.skip=true install
- run:
name: test runtime
command: |
cd runtime-testsuite
mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=java.* test
cd ..
- run:
name: test tool
command: |
cd tool-testsuite
mvn -q -Dparallel=methods -DthreadCount=4 test
cd ..
test_runtime:
parameters:
test-group:
description: The section
type: string
default: ALL
target:
description: The target
type: string
default: java
docker:
- image: cimg/openjdk:8.0
environment:
TARGET: << parameters.target >>
GROUP: << parameters.test-group >>
steps:
- checkout
- run:
name: Install << parameters.target >> pre-requisites
command: |
f=".circleci/scripts/install-linux-<< parameters.target >>.sh"; ! [ -x "$f" ] || "$f"
- run:
name: Build ANTLR4 tool
command: mvn -B -V -DskipTests=true -Dmaven.javadoc.skip=true install
- run:
name: Test << parameters.target >> runtime
command: |
.circleci/scripts/run-tests-<< parameters.target >>.sh
workflows:
build:
jobs:
- test_tool_and_runtime_java
- test_runtime:
matrix:
parameters:
target: [ dart, go, python2, python3, javascript, php ]
- test_runtime:
matrix:
parameters:
# target: [ cpp, dotnet, swift ]
target: [ cpp, dotnet ]
test-group: [ LEXER, PARSER, RECURSION ]

35
.circleci/scripts/install-linux-cpp.sh Executable file
View File

@ -0,0 +1,35 @@
#!/bin/bash
set -euo pipefail
echo "installing cpp SDK..."
sudo apt-get update -y
sudo apt-get install -y clang
sudo apt-get install -y cmake
sudo apt-get install -y pkg-config
sudo apt-get install -y uuid-dev
echo "done installing cpp SDK"
clang++ --version
cmake --version
echo "building cpp runtime..."
pushd "runtime/Cpp/"
echo $PWD
rc=0
if [ $rc == 0 ]; then
cmake . -DCMAKE_BUILD_TYPE=release
rc=$?
fi
if [ $rc == 0 ]; then
make -j 8
rc=$?
fi
popd
echo "done building cpp runtime"

14
.circleci/scripts/install-linux-dart.sh Executable file
View File

@ -0,0 +1,14 @@
#!/bin/bash
set -euo pipefail
echo "installing dart SDK..."
sudo apt-get update
sudo apt-get install apt-transport-https
sudo sh -c 'wget -qO- https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -'
sudo sh -c 'wget -qO- https://storage.googleapis.com/download.dartlang.org/linux/debian/dart_stable.list > /etc/apt/sources.list.d/dart_stable.list'
sudo apt-get update
sudo apt-get install dart=2.8.4-1
export PATH="$PATH:/usr/lib/dart/bin"
echo "done installing dart SDK"
sudo apt-get install -f

19
.circleci/scripts/install-linux-dotnet.sh Executable file
View File

@ -0,0 +1,19 @@
#!/bin/bash
set -euo pipefail
echo "installing .Net SDK..."
wget https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
sudo dpkg -i packages-microsoft-prod.deb
sudo apt-get update; \
sudo apt-get install -y apt-transport-https && \
sudo apt-get update && \
sudo apt-get install -y dotnet-sdk-3.1
export PATH=$PATH:~/.dotnet
echo "done installing .Net SDK"
# we need to build the runtime before test run, since we used "--no-dependencies"
# when we call dotnet cli for restore and build, in order to speed up
echo "building runtime..."
dotnet build -c Release -f netstandard2.0 runtime/CSharp/src/Antlr4.csproj
echo "done building runtime"

9
.circleci/scripts/install-linux-go.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/bash
set -euo pipefail
echo "installing go SDK..."
sudo apt update
sudo apt install golang-go
go version
echo "done installing go SDK"

21
.circleci/scripts/install-linux-javascript.sh Executable file
View File

@ -0,0 +1,21 @@
#!/bin/bash
set -euo pipefail
# use v14 and check
echo "installing nodejs..."
curl -sL https://deb.nodesource.com/setup_14.x | sudo -E bash -
sudo apt-get install -y nodejs
echo node version: $(node --version)
echo "done installing nodejs"
echo "installing yarn..."
sudo npm install -g yarn@v1.22.10
echo "done installing yarn"
echo "packaging javascript runtime..."
pushd runtime/JavaScript
sudo npm install
sudo npm link
popd
echo "done packaging javascript runtime"

24
.circleci/scripts/install-linux-libcurl3.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
echo "before patching"
ls -all /lib/x86_64-linux-gnu/ | grep libcurl
# This would fix missing CURL_OPENSSL_3
# use a dedicated temp dir in the user space
mkdir ~/libcurl3
cd ~/libcurl3
# fetch latest libcurl3
wget http://archive.ubuntu.com/ubuntu/pool/main/c/curl/libcurl3_7.47.0-1ubuntu2_amd64.deb
# extract data.tar.xz
ar x libcurl3* data.tar.xz
# extract all from data.tar.xz
tar xf data.tar.xz
# copy libcurl.so.3 where required
sudo cp -L ~/libcurl3/usr/lib/x86_64-linux-gnu/libcurl.so.4.4.0 /lib/x86_64-linux-gnu/libcurl.so.4.4.0
sudo ln -sf libcurl.so.4.4.0 /lib/x86_64-linux-gnu/libcurl.so.4
cd ..
# drop dedicated temp dir
sudo rm -rf ~/libcurl3
echo "after patching"
ls -all /lib/x86_64-linux-gnu/ | grep libcurl

18
.circleci/scripts/install-linux-php.sh Executable file
View File

@ -0,0 +1,18 @@
#!/bin/bash
set -euo pipefail
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
sudo apt-get update -qq
sudo apt update
sudo apt install php-all-dev
php -v
sudo apt install composer
git clone https://github.com/antlr/antlr-php-runtime.git runtime/PHP
composer install -d runtime/PHP
mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V

8
.circleci/scripts/install-linux-python2.sh Executable file
View File

@ -0,0 +1,8 @@
#!/bin/bash
set -euo pipefail
echo "installing python 2..."
sudo apt-get update -y
sudo apt-get install python2
echo "done installing python 2"

8
.circleci/scripts/install-linux-python3.sh Executable file
View File

@ -0,0 +1,8 @@
#!/bin/bash
set -euo pipefail
echo "installing python 3..."
sudo apt-get update -y
sudo apt-get install python3
echo "done installing python 3"

36
.circleci/scripts/install-linux-swift.sh Executable file
View File

@ -0,0 +1,36 @@
#!/bin/bash
set -euo pipefail
echo "installing swift SDK..."
.circleci/scripts/install-linux-libcurl3.sh
# see https://tecadmin.net/install-swift-ubuntu-1604-xenial/
sudo apt-get update -y
sudo apt-get install clang libicu-dev
sudo apt-get install libpython2.7 libpython2.7-dev
export SWIFT_VERSION=swift-5.3.2
echo "installing gpg key..."
wget -q -O - https://swift.org/keys/all-keys.asc | sudo gpg --import -
echo "downloading SDK gpg key..."
SWIFT_SDK=https://swift.org/builds/$SWIFT_VERSION-release/ubuntu1604/$SWIFT_VERSION-RELEASE/$SWIFT_VERSION-RELEASE-ubuntu16.04.tar.gz
echo $SWIFT_SDK
wget -q $SWIFT_SDK
sudo tar xzf $SWIFT_VERSION-RELEASE-ubuntu16.04.tar.gz
mv $SWIFT_VERSION-RELEASE-ubuntu16.04 $PWD/swift
export SWIFT_HOME=$PWD/swift/$SWIFT_VERSION-RELEASE-ubuntu16.04/usr/bin/
export PATH=$PWD/swift/usr/bin:$PATH
# This would fix a know linker issue mentioned in: # https://bugs.swift.org/browse/SR-2299
sudo ln -sf ld.gold /usr/bin/ld
# This would fix missing libtinfo.so.5
sudo apt install libncurses5
echo "done installing swift SDK..."
# check swift
swift --version
swift build --version

17
.circleci/scripts/run-tests-cpp.sh Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
set -euo pipefail
pushd runtime-testsuite
echo "running maven tests..."
if [ $GROUP == "LEXER" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=cpp.* test
elif [ $GROUP == "PARSER" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=cpp.* test
elif [ $GROUP == "RECURSION" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=cpp.* test
else
mvn -q -Dtest=cpp.* test
fi
popd

11
.circleci/scripts/run-tests-dart.sh Executable file
View File

@ -0,0 +1,11 @@
#!/bin/bash
set -euo pipefail
dart --version
pushd runtime-testsuite
echo "running maven tests..."
# mvn -q -Dparallel=classes -DthreadCount=4 -Dtest=dart.* test
mvn -q -Dtest=dart.* test
popd

16
.circleci/scripts/run-tests-dotnet.sh Executable file
View File

@ -0,0 +1,16 @@
#!/bin/bash
set -euo pipefail
pushd runtime-testsuite/
echo "running maven tests..."
if [ $GROUP == "LEXER" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=csharp.* test
elif [ $GROUP == "PARSER" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=csharp.* test
elif [ $GROUP == "RECURSION" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=csharp.* test
else
mvn -q -Dtest=csharp.* test
fi
popd

10
.circleci/scripts/run-tests-go.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
set -euo pipefail
go version
pushd runtime-testsuite
echo "running maven tests..."
mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=go.* test
popd

23
.circleci/scripts/run-tests-javascript.sh Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
set -euo pipefail
declare -i RESULT=0
pushd runtime/JavaScript
echo "running jest tests..."
yarn test
RESULT+=$?
popd
pushd runtime-testsuite
echo "running maven tests..."
mvn -q -Dtest=javascript.* test
RESULT+=$?
popd
exit $RESULT

11
.circleci/scripts/run-tests-php.sh Executable file
View File

@ -0,0 +1,11 @@
#!/bin/bash
set -euo pipefail
php -v
php_path=$(which php)
pushd runtime-testsuite
echo "running maven tests..."
mvn -q -DPHP_PATH="${php_path}" -Dparallel=methods -DthreadCount=4 -Dtest=php.* test
popd

24
.circleci/scripts/run-tests-python2.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
set -euo pipefail
python2 --version
pushd runtime/Python2/tests
echo "running native tests..."
python2 run.py
rc=$?
if [ $rc != 0 ]; then
echo "failed running native tests"
fi
popd
if [ $rc == 0 ]; then
pushd runtime-testsuite
echo "running maven tests..."
mvn -q -Dtest=python2.* test
rc=$?
popd
fi
# return $rc

24
.circleci/scripts/run-tests-python3.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
set -euo pipefail
python3 --version
pushd runtime/Python3/tests
echo "running native tests..."
python3 run.py
rc=$?
if [ $rc != 0 ]; then
echo "failed running native tests"
fi
popd
if [ $rc == 0 ]; then
pushd runtime-testsuite
echo "running maven tests..."
mvn -q -Dtest=python3.* test
rc=$?
popd
fi
# return $rc

27
.circleci/scripts/run-tests-swift.sh Executable file
View File

@ -0,0 +1,27 @@
#!/bin/bash
set -euo pipefail
pushd runtime/Swift
echo "running native tests..."
./boot.py --test
rc=$?
if [ $rc != 0 ]; then
echo "failed running native tests"
fi
popd
if [ $rc == 0 ]; then
pushd runtime-testsuite
echo "running maven tests..."
if [ $GROUP == "LEXER" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=swift.* test
elif [ $GROUP == "PARSER" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=swift.* test
elif [ $GROUP == "RECURSION" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=swift.* test
else
mvn -q -Dtest=swift.* test
fi
popd
fi

1112
.github/scripts/install-dotnet-on-osx.sh vendored Executable file
View File

File diff suppressed because it is too large Load Diff

30
.github/scripts/mac-action-runner-readme.txt vendored Normal file
View File

@ -0,0 +1,30 @@
Sharing the trouble of getting github action runners to work on a mac coming straight out of the factory, running Big Sur
XCode (you need XCode to build the Swift runtime):
- install XCode from the Mac App Store
- Launch it, this will force installation of components
- Go to Preferences -> Locations and select XCode as Command Line Tools
Brew (you need Brew to install maven):
- get the script from https://brew.sh
- once installed, run the following:
echo 'eval $(/opt/homebrew/bin/brew shellenv)' >> /Users/{user-account}/.zprofile
eval $(/opt/homebrew/bin/brew shellenv)
(you need to repeat these last steps for each user account)
Maven (supposedly installed by the github workflow, but it's convenient to have a global install for troubleshooting):
- brew install maven
JDK (we need a specific JDK):
- download openjdk8 from Oracle (later versions break the build due to some packages having disappeared)
- install it -> this will mess up your JAVA_HOME completely, pointing to /Library/Internet...
- fix the JAVA_HOME mess as follows:
sudo rm -fr /Library/Internet\ Plug-Ins/JavaAppletPlugin.plugin
sudo rm -fr /Library/PreferencePanes/JavaControlPanel.prefpane
C++:
- brew install cmake
C#:
- .github/scripts/install-dotnet-on-osx.sh
(you need to repeat this step for each user account)

15
.github/scripts/run-tests-cpp.sh vendored Executable file
View File

@ -0,0 +1,15 @@
#!/bin/bash
set -euo pipefail
cd runtime-testsuite/
if [ $GROUP == "LEXER" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=cpp.* test
elif [ $GROUP == "PARSER" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=cpp.* test
elif [ $GROUP == "RECURSION" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=cpp.* test
else
mvn -q -Dtest=cpp.* test
fi

23
.github/scripts/run-tests-dotnet.sh vendored Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
set -euo pipefail
export PATH=$PATH:~/.dotnet
# we need to build the runtime before test run, since we used "--no-dependencies"
# when we call dotnet cli for restore and build, in order to speed up
dotnet build -c Release -f netstandard2.0 runtime/CSharp/Antlr4.csproj
# run tests
cd runtime-testsuite/
if [ $GROUP == "LEXER" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest=csharp.* test
elif [ $GROUP == "PARSER" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest=csharp.* test
elif [ $GROUP == "RECURSION" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest=csharp.* test
else
mvn -q -Dtest=csharp.* test
fi

50
.github/scripts/run-tests-swift.sh vendored Executable file
View File

@ -0,0 +1,50 @@
#!/bin/bash
set -euo pipefail
# linux specific setup, those setup have to be
# here since environment variables doesn't pass
# across scripts
if [ $RUNNER_OS == "Linux" ]; then
export SWIFT_VERSION=swift-5.0.1
export SWIFT_HOME=$(pwd)/swift/$SWIFT_VERSION-RELEASE-ubuntu16.04/usr/bin/
export PATH=$SWIFT_HOME:$PATH
# download swift
mkdir swift
curl https://swift.org/builds/$SWIFT_VERSION-release/ubuntu1604/$SWIFT_VERSION-RELEASE/$SWIFT_VERSION-RELEASE-ubuntu16.04.tar.gz -s | tar xz -C swift &> /dev/null
fi
if [ -z "${JAVA_HOME}" ]
then
export JAVA_HOME="$(java -XshowSettings:properties -version 2>&1 |
grep 'java\.home' | awk '{ print $3 }')"
fi
echo "export JAVA_HOME=$JAVA_HOME"
# check swift
swift --version
swift build --version
# run swift tests
pushd runtime/Swift
./boot.py --test
rc=$?
popd
if [ $rc == 0 ]; then
# run java tests
cd runtime-testsuite/
if [ $GROUP == "LEXER" ]; then
mvn -e -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dtest="swift.*" test
elif [ $GROUP == "PARSER" ]; then
mvn -e -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dtest="swift.*" test
elif [ $GROUP == "RECURSION" ]; then
mvn -e -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dtest="swift.*" test
else
mvn -e -q -Dtest=swift.* test
fi
rc=$?
cat target/surefire-reports/*.dumpstream || true
fi
exit $rc

34
.github/workflows/macosx.yml vendored Normal file
View File

@ -0,0 +1,34 @@
name: MacOSX
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: [self-hosted, macOS, x64]
strategy:
fail-fast: false
matrix:
# TARGET: [swift, cpp, dotnet] disabling dotnet which is unstable on M1
TARGET: [swift, cpp]
GROUP: [LEXER, PARSER, RECURSION]
steps:
- uses: actions/checkout@v2
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
java-version: 1.8
- name: Set up Maven
uses: stCarolas/setup-maven@v4
with:
maven-version: 3.5.4
- name: Build tool with Maven
run: mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V
- name: Test with Maven
run: arch -x86_64 .github/scripts/run-tests-${{ matrix.TARGET }}.sh
env:
TARGET: ${{ matrix.TARGET }}
GROUP: ${{ matrix.GROUP }}

7
.gitignore vendored
View File

@ -3,7 +3,7 @@ target/
# ... but not code generation targets
!tool/src/org/antlr/v4/codegen/target/
# Node.js (npm and typings) cached dependencies
# Node.js (npm and typings) cached dependencies
node_modules/
typings/
@ -97,4 +97,7 @@ xcuserdata
# VSCode Java plugin temporary files
javac-services.0.log
javac-services.0.log.lck
test/
# Don't ignore python tests
!runtime/Python3/test/
Antlr4.sln

3
.gitmodules vendored
View File

@ -1,3 +0,0 @@
[submodule "runtime/PHP"]
path = runtime/PHP
url = https://github.com/antlr/antlr-php-runtime.git

193
.travis.yml
View File

@ -12,108 +12,12 @@ cache:
- $HOME/Library/Caches/Homebrew
stages:
- smoke-test
- main-test
# - smoke-test
# - main-test
- extended-test
matrix:
include:
- os: linux
dist: trusty
compiler: clang
jdk: openjdk8
env:
- TARGET=cpp
- CXX=g++-5
- GROUP=LEXER
stage: main-test
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-precise-3.7
packages:
- g++-5
- uuid-dev
- clang-3.7
- os: linux
dist: trusty
compiler: clang
jdk: openjdk8
env:
- TARGET=cpp
- CXX=g++-5
- GROUP=PARSER
stage: main-test
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-precise-3.7
packages:
- g++-5
- uuid-dev
- clang-3.7
- os: linux
dist: trusty
compiler: clang
jdk: openjdk8
env:
- TARGET=cpp
- CXX=g++-5
- GROUP=RECURSION
stage: main-test
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-precise-3.7
packages:
- g++-5
- uuid-dev
- clang-3.7
- os: osx
compiler: clang
osx_image: xcode10.2
env:
- TARGET=cpp
- GROUP=LEXER
stage: extended-test
- os: osx
compiler: clang
osx_image: xcode10.2
env:
- TARGET=cpp
- GROUP=PARSER
stage: extended-test
- os: osx
compiler: clang
osx_image: xcode10.2
env:
- TARGET=cpp
- GROUP=RECURSION
stage: extended-test
- os: osx
compiler: clang
osx_image: xcode10.2
env:
- TARGET=swift
- GROUP=LEXER
stage: main-test
- os: osx
compiler: clang
osx_image: xcode10.2
env:
- TARGET=swift
- GROUP=PARSER
stage: main-test
- os: osx
compiler: clang
osx_image: xcode10.2
env:
- TARGET=swift
- GROUP=RECURSION
stage: main-test
- os: linux
dist: xenial
compiler: clang
@ -121,93 +25,6 @@ matrix:
- TARGET=swift
- GROUP=ALL
stage: extended-test
- os: osx
osx_image: xcode10.2
env:
- TARGET=dotnet
- GROUP=LEXER
stage: extended-test
- os: osx
osx_image: xcode10.2
env:
- TARGET=dotnet
- GROUP=PARSER
stage: extended-test
- os: osx
osx_image: xcode10.2
env:
- TARGET=dotnet
- GROUP=RECURSION
stage: extended-test
- os: linux
dist: trusty
jdk: openjdk7
env: TARGET=java
stage: extended-test
- os: linux
jdk: openjdk8
env: TARGET=java
stage: smoke-test
- os: linux
jdk: openjdk8
env: TARGET=csharp
stage: main-test
- os: linux
jdk: openjdk8
env: TARGET=dart
stage: main-test
- os: linux
language: php
php:
- 7.2
jdk: openjdk8
env: TARGET=php
stage: main-test
- os: linux
jdk: openjdk8
dist: trusty
env:
- TARGET=dotnet
- GROUP=LEXER
stage: extended-test
- os: linux
jdk: openjdk8
dist: trusty
env:
- TARGET=dotnet
- GROUP=PARSER
stage: extended-test
- os: linux
jdk: openjdk8
dist: trusty
env:
- TARGET=dotnet
- GROUP=RECURSION
stage: extended-test
- os: linux
jdk: openjdk8
env: TARGET=python2
stage: main-test
- os: linux
jdk: openjdk8
env: TARGET=python3
addons:
apt:
sources:
- deadsnakes # source required so it finds the package definition below
packages:
- python3.7
stage: main-test
- os: linux
dist: trusty
jdk: openjdk8
env: TARGET=javascript
stage: main-test
- os: linux
dist: trusty
jdk: openjdk8
env: TARGET=go
stage: main-test
before_install:
- f="./.travis/before-install-$TRAVIS_OS_NAME-$TARGET.sh"; ! [ -x "$f" ] || "$f"
@ -215,7 +32,7 @@ before_install:
script:
- |
cd runtime-testsuite;
travis_wait 40 ../.travis/run-tests-$TARGET.sh;
rc=$?;
cat target/surefire-reports/*.dumpstream || true;
travis_wait 40 ../.travis/run-tests-$TARGET.sh
rc=$?
cat target/surefire-reports/*.dumpstream || true
exit $rc

8
.travis/before-install-linux-csharp.sh
View File

@ -1,8 +0,0 @@
#!/bin/bash
set -euo pipefail
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
echo "deb http://download.mono-project.com/repo/debian xenial main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list
sudo apt-get update -qq
sudo apt-get install -qq mono-complete

12
.travis/before-install-linux-dotnet.sh
View File

@ -2,9 +2,9 @@
set -euo pipefail
# install dotnet
sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ trusty main" > /etc/apt/sources.list.d/dotnetdev.list'
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 417A0893
sudo apt-get update
sudo apt-get --allow-unauthenticated install dotnet-dev-1.0.4
wget https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
sudo dpkg -i packages-microsoft-prod.deb
sudo apt-get update; \
sudo apt-get install -y apt-transport-https && \
sudo apt-get update && \
sudo apt-get install -y dotnet-sdk-3.1

7
.travis/before-install-linux-javascript.sh
View File

@ -2,7 +2,6 @@
set -euo pipefail
sudo apt-get update -qq
curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash -
sudo apt-get install -qq nodejs
node --version
# use v14 and check
echo node version: $(node --version)

1
.travis/before-install-linux-php.sh
View File

@ -7,4 +7,5 @@ sudo apt-get update -qq
php -v
git clone https://github.com/antlr/antlr-php-runtime.git
mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V

1134
.travis/before-install-osx-dotnet.sh
View File

File diff suppressed because it is too large Load Diff

5
.travis/run-tests-csharp.sh
View File

@ -1,5 +0,0 @@
#!/bin/bash
set -euo pipefail
mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=csharp.* test

13
.travis/run-tests-dotnet.sh
View File

@ -2,20 +2,21 @@
set -euo pipefail
export PATH=$PATH:/Users/travis/.dotnet
# we need to build the runtime before test run, since we used "--no-dependencies"
# when we call dotnet cli for restore and build, in order to speed up
dotnet restore ../runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj
dotnet build -c Release -f netstandard1.3 ../runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj
dotnet build -c Release -f netstandard2.0 ../runtime/CSharp/src/Antlr4.csproj
# call test
if [ $GROUP == "LEXER" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dparallel=classes -DthreadCount=4 -Dtest=csharp.* -Dantlr-csharp-netstandard=true test
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LexerTests" -Dparallel=classes -DthreadCount=4 -Dtest=csharp.* test
elif [ $GROUP == "PARSER" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dparallel=classes -DthreadCount=4 -Dtest=csharp.* -Dantlr-csharp-netstandard=true test
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.ParserTests" -Dparallel=classes -DthreadCount=4 -Dtest=csharp.* test
elif [ $GROUP == "RECURSION" ]; then
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dparallel=classes -DthreadCount=4 -Dtest=csharp.* -Dantlr-csharp-netstandard=true test
mvn -q -Dgroups="org.antlr.v4.test.runtime.category.LeftRecursionTests" -Dparallel=classes -DthreadCount=4 -Dtest=csharp.* test
else
mvn -q -Dparallel=classes -DthreadCount=4 -Dtest=csharp.* -Dantlr-csharp-netstandard=true test
mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=csharp.* test
fi

6
.travis/run-tests-javascript.sh
View File

@ -2,4 +2,8 @@
set -euo pipefail
mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=javascript.* test
cd ../runtime/JavaScript
npm install
npm link
cd ../../runtime-testsuite
mvn -q -Dparallel=methods -DthreadCount=1 -Dtest=javascript.* test

16
README.md
View File

@ -1,6 +1,14 @@
# ANTLR v4
[![Build Travis-CI Status](https://travis-ci.org/antlr/antlr4.svg?branch=master)](https://travis-ci.org/antlr/antlr4) [![Build AppVeyor Status](https://ci.appveyor.com/api/projects/status/5acpbx1pg7bhgh8v/branch/master?svg=true)](https://ci.appveyor.com/project/parrt/antlr4) [![Java 7+](https://img.shields.io/badge/java-7+-4c7e9f.svg)](http://java.oracle.com) [![License](https://img.shields.io/badge/license-BSD-blue.svg)](https://raw.githubusercontent.com/antlr/antlr4/master/LICENSE.txt)
[![Java 7+](https://img.shields.io/badge/java-7+-4c7e9f.svg)](http://java.oracle.com)
[![License](https://img.shields.io/badge/license-BSD-blue.svg)](https://raw.githubusercontent.com/antlr/antlr4/master/LICENSE.txt)
**Build status**
[![Github CI Build Status (MacOSX)](https://img.shields.io/github/workflow/status/antlr/antlr4/MacOSX?label=MacOSX)](https://github.com/antlr/antlr4/actions)
[![AppVeyor CI Build Status (Windows)](https://img.shields.io/appveyor/build/parrt/antlr4?label=Windows)](https://ci.appveyor.com/project/parrt/antlr4)
[![Circle CI Build Status (Linux)](https://img.shields.io/circleci/build/gh/antlr/antlr4/master?label=Linux)](https://app.circleci.com/pipelines/github/antlr/antlr4)
[![Travis-CI Build Status (Swift-Linux)](https://img.shields.io/travis/antlr/antlr4.svg?label=Linux-Swift&branch=master)](https://travis-ci.com/github/antlr/antlr4)
**ANTLR** (ANother Tool for Language Recognition) is a powerful parser generator for reading, processing, executing, or translating structured text or binary files. It's widely used to build languages, tools, and frameworks. From a grammar, ANTLR generates a parser that can build parse trees and also generates a listener interface (or visitor) that makes it easy to respond to the recognition of phrases of interest.
@ -13,8 +21,8 @@
* [Terence Parr](http://www.cs.usfca.edu/~parrt/), parrt@cs.usfca.edu
ANTLR project lead and supreme dictator for life
[University of San Francisco](http://www.usfca.edu/)
* [Sam Harwell](http://tunnelvisionlabs.com/) (Tool co-author, Java and C# target)
* Eric Vergnaud (Javascript, Python2, Python3 targets and significant work on C# target)
* [Sam Harwell](http://tunnelvisionlabs.com/) (Tool co-author, Java and original C# target)
* [Eric Vergnaud](https://github.com/ericvergnaud) (Javascript, Python2, Python3 targets and maintenance of C# target)
* [Peter Boyer](https://github.com/pboyer) (Go target)
* [Mike Lischke](http://www.soft-gems.net/) (C++ completed target)
* Dan McLaughlin (C++ initial target)
@ -32,7 +40,7 @@ ANTLR project lead and supreme dictator for life
* [Official site](http://www.antlr.org/)
* [Documentation](https://github.com/antlr/antlr4/blob/master/doc/index.md)
* [FAQ](https://github.com/antlr/antlr4/blob/master/doc/faq/index.md)
* [ANTLR code generation targets](https://github.com/antlr/antlr4/blob/master/doc/targets.md)<br>(Currently: Java, C#, Python2|3, JavaScript, Go, C++, Swift)
* [ANTLR code generation targets](https://github.com/antlr/antlr4/blob/master/doc/targets.md)<br>(Currently: Java, C#, Python2|3, JavaScript, Go, C++, Swift, Dart, PHP)
* [Java API](http://www.antlr.org/api/Java/index.html)
* [ANTLR v3](http://www.antlr3.org/)
* [v3 to v4 Migration, differences](https://github.com/antlr/antlr4/blob/master/doc/faq/general.md)

6
antlr4-maven-plugin/pom.xml
View File

@ -8,7 +8,7 @@
<parent>
<groupId>org.antlr</groupId>
<artifactId>antlr4-master</artifactId>
<version>4.8-2-SNAPSHOT</version>
<version>4.9.3-SNAPSHOT</version>
</parent>
<artifactId>antlr4-maven-plugin</artifactId>
<packaging>maven-plugin</packaging>
@ -64,7 +64,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<version>4.13.1</version>
<scope>test</scope>
</dependency>
<dependency>
@ -120,7 +120,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-plugin-plugin</artifactId>
<version>3.3</version>
<version>3.6.0</version>
<configuration>
<!-- see http://jira.codehaus.org/browse/MNG-5346 -->
<skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>

2
antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java
View File

@ -55,7 +55,7 @@ import java.util.Set;
name = "antlr4",
defaultPhase = LifecyclePhase.GENERATE_SOURCES,
requiresDependencyResolution = ResolutionScope.COMPILE,
requiresProject = true)
requiresProject = true, threadSafe = true)
public class Antlr4Mojo extends AbstractMojo {
// First, let's deal with the options that the ANTLR tool itself

2
antlr4-maven-plugin/src/test/projects/dependencyRemoved/pom.xml
View File

@ -17,7 +17,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<version>4.13.1</version>
<scope>test</scope>
</dependency>
</dependencies>

2
antlr4-maven-plugin/src/test/projects/importTokens/pom.xml
View File

@ -17,7 +17,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<version>4.13.1</version>
<scope>test</scope>
</dependency>
</dependencies>

2
antlr4-maven-plugin/src/test/projects/importsCustom/pom.xml
View File

@ -17,7 +17,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<version>4.13.1</version>
<scope>test</scope>
</dependency>
</dependencies>

2
antlr4-maven-plugin/src/test/projects/importsStandard/pom.xml
View File

@ -17,7 +17,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<version>4.13.1</version>
<scope>test</scope>
</dependency>
</dependencies>

22
appveyor.yml
View File

@ -1,22 +0,0 @@
version: '4.7.1-SNAPSHOT+AppVeyor.{build}'
cache:
- '%USERPROFILE%\.m2'
- '%USERPROFILE%\.nuget\packages -> **\project.json'
image: Visual Studio 2017
build: off
install:
- git submodule update --init --recursive
- cinst -y php --params "/InstallDir:C:\tools\php"
- cinst -y composer
- cinst -y dart-sdk --version=2.8.4
build_script:
- mvn -DskipTests install --batch-mode
- msbuild /target:restore /target:rebuild /property:Configuration=Release /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /verbosity:detailed runtime/CSharp/runtime/CSharp/Antlr4.dotnet.sln
- msbuild ./runtime-testsuite/target/classes/CSharp/runtime/CSharp/Antlr4.vs2013.sln /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /verbosity:detailed
after_build:
- msbuild /target:pack /property:Configuration=Release /verbosity:detailed runtime/CSharp/runtime/CSharp/Antlr4.dotnet.sln
test_script:
- mvn install -Dantlr-php-php="C:\tools\php\php.exe" -Dantlr-dart-dart="C:\tools\dart-sdk\bin\dart.exe" -Dantlr-dart-pub="C:\tools\dart-sdk\bin\pub.bat" -Dantlr-dart-dart2native="C:\tools\dart-sdk\bin\dart2native.bat" -Dantlr-python2-python="C:\Python27\python.exe" -Dantlr-python3-python="C:\Python35\python.exe" -Dantlr-javascript-nodejs="C:\Program Files (x86)\nodejs\node.exe" --batch-mode
artifacts:
- path: 'runtime\**\*.nupkg'
name: NuGet

64
build/Antlr4.Runtime.nuspec
View File

@ -1,64 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd">
<metadata>
<id>Antlr4.Runtime</id>
<version>0.0.0</version>
<authors>Sam Harwell, Terence Parr</authors>
<owners>Sam Harwell</owners>
<description>The runtime library for parsers generated by the C# target of ANTLR 4. This package supports projects targeting .NET 2.0 or newer, and built using Visual Studio 2008 or newer.</description>
<language>en-us</language>
<projectUrl>https://github.com/sharwell/antlr4cs</projectUrl>
<licenseUrl>https://raw.github.com/sharwell/antlr4cs/master/LICENSE.txt</licenseUrl>
<iconUrl>https://raw.github.com/antlr/website-antlr4/master/images/icons/antlr.png</iconUrl>
<copyright>Copyright © Sam Harwell 2014</copyright>
<releaseNotes>https://github.com/sharwell/antlr4cs/releases/v$version$</releaseNotes>
<requireLicenseAcceptance>true</requireLicenseAcceptance>
<tags>antlr antlr4 parsing</tags>
<title>ANTLR 4 Runtime</title>
<summary>The runtime library for parsers generated by the C# target of ANTLR 4.</summary>
</metadata>
<files>
<!-- Runtime Libraries -->
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net20\$Configuration$\Antlr4.Runtime.dll" target="lib\net20"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net20\$Configuration$\Antlr4.Runtime.pdb" target="lib\net20"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net20\$Configuration$\Antlr4.Runtime.xml" target="lib\net20"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net30\$Configuration$\Antlr4.Runtime.dll" target="lib\net30"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net30\$Configuration$\Antlr4.Runtime.pdb" target="lib\net30"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net30\$Configuration$\Antlr4.Runtime.xml" target="lib\net30"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net35-client\$Configuration$\Antlr4.Runtime.dll" target="lib\net35-client"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net35-client\$Configuration$\Antlr4.Runtime.pdb" target="lib\net35-client"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net35-client\$Configuration$\Antlr4.Runtime.xml" target="lib\net35-client"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net40-client\$Configuration$\Antlr4.Runtime.dll" target="lib\net40-client"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net40-client\$Configuration$\Antlr4.Runtime.pdb" target="lib\net40-client"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net40-client\$Configuration$\Antlr4.Runtime.xml" target="lib\net40-client"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net45\$Configuration$\Antlr4.Runtime.dll" target="lib\net45"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net45\$Configuration$\Antlr4.Runtime.pdb" target="lib\net45"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net45\$Configuration$\Antlr4.Runtime.xml" target="lib\net45"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net35-cf\$Configuration$\Antlr4.Runtime.dll" target="lib\net35-cf"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net35-cf\$Configuration$\Antlr4.Runtime.pdb" target="lib\net35-cf"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\net35-cf\$Configuration$\Antlr4.Runtime.xml" target="lib\net35-cf"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\portable-net40\$Configuration$\Antlr4.Runtime.dll" target="lib\portable-net4+sl5+netcore45+wpa81+wp8+MonoAndroid1+MonoTouch1"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\portable-net40\$Configuration$\Antlr4.Runtime.pdb" target="lib\portable-net4+sl5+netcore45+wpa81+wp8+MonoAndroid1+MonoTouch1"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\portable-net40\$Configuration$\Antlr4.Runtime.xml" target="lib\portable-net4+sl5+netcore45+wpa81+wp8+MonoAndroid1+MonoTouch1"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\portable-net45\$Configuration$\Antlr4.Runtime.dll" target="lib\portable-net45+netcore45+wpa81+wp8+MonoAndroid1+MonoTouch1"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\portable-net45\$Configuration$\Antlr4.Runtime.pdb" target="lib\portable-net45+netcore45+wpa81+wp8+MonoAndroid1+MonoTouch1"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\portable-net45\$Configuration$\Antlr4.Runtime.xml" target="lib\portable-net45+netcore45+wpa81+wp8+MonoAndroid1+MonoTouch1"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\netcore45\$Configuration$\Antlr4.Runtime.dll" target="lib\netcore45"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\netcore45\$Configuration$\Antlr4.Runtime.pdb" target="lib\netcore45"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\bin\netcore45\$Configuration$\Antlr4.Runtime.xml" target="lib\netcore45"/>
<!-- Source Code -->
<file exclude="..\runtime\CSharp\Antlr4.Runtime\obj\**\*.cs" src="..\runtime\CSharp\Antlr4.Runtime\**\*.cs" target="src"/>
<file src="..\runtime\CSharp\Antlr4.Runtime\**\$Configuration$\*Lexer.cs" target="src"/>
</files>
</package>

35
build/Antlr4.VS2008.nuspec
View File

@ -1,35 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd">
<metadata minClientVersion="2.7">
<id>Antlr4.VS2008</id>
<version>0.0.0</version>
<authors>Sam Harwell, Terence Parr</authors>
<owners>Sam Harwell</owners>
<description>The C# target of the ANTLR 4 parser generator for Visual Studio 2008 projects. This package supports projects targeting .NET 2.0 or newer, and built using Visual Studio 2008.</description>
<language>en-us</language>
<projectUrl>https://github.com/sharwell/antlr4cs</projectUrl>
<licenseUrl>https://raw.github.com/sharwell/antlr4cs/master/LICENSE.txt</licenseUrl>
<iconUrl>https://raw.github.com/antlr/website-antlr4/master/images/icons/antlr.png</iconUrl>
<copyright>Copyright © Sam Harwell 2014</copyright>
<releaseNotes>https://github.com/sharwell/antlr4cs/releases/v$version$</releaseNotes>
<requireLicenseAcceptance>true</requireLicenseAcceptance>
<developmentDependency>true</developmentDependency>
<tags>antlr antlr4 parsing</tags>
<title>ANTLR 4 (Visual Studio 2008)</title>
<summary>The C# target of the ANTLR 4 parser generator for Visual Studio 2008 projects.</summary>
<dependencies>
<dependency id="Antlr4.Runtime" version="$version$" />
</dependencies>
</metadata>
<files>
<!-- Tools -->
<file src="..\tool\target\antlr4-csharp-$CSharpToolVersion$-complete.jar" target="tools"/>
<!-- Build Configuration -->
<file src="..\runtime\CSharp\Antlr4BuildTasks\bin\net35\$Configuration$\Antlr4.net35.props" target="build\Antlr4.props"/>
<file src="..\runtime\CSharp\Antlr4BuildTasks\bin\net35\$Configuration$\Antlr4.net35.targets" target="build\Antlr4.targets"/>
<file src="..\runtime\CSharp\Antlr4BuildTasks\bin\net35\$Configuration$\Antlr4BuildTasks.net35.dll" target="build"/>
</files>
</package>

35
build/Antlr4.nuspec
View File

@ -1,35 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd">
<metadata minClientVersion="2.7">
<id>Antlr4</id>
<version>0.0.0</version>
<authors>Sam Harwell, Terence Parr</authors>
<owners>Sam Harwell</owners>
<description>The C# target of the ANTLR 4 parser generator for Visual Studio 2010+ projects. This package supports projects targeting .NET 2.0 or newer, and built using Visual Studio 2010 or newer.</description>
<language>en-us</language>
<projectUrl>https://github.com/sharwell/antlr4cs</projectUrl>
<licenseUrl>https://raw.github.com/sharwell/antlr4cs/master/LICENSE.txt</licenseUrl>
<iconUrl>https://raw.github.com/antlr/website-antlr4/master/images/icons/antlr.png</iconUrl>
<copyright>Copyright © Sam Harwell 2014</copyright>
<releaseNotes>https://github.com/sharwell/antlr4cs/releases/v$version$</releaseNotes>
<requireLicenseAcceptance>true</requireLicenseAcceptance>
<developmentDependency>true</developmentDependency>
<tags>antlr antlr4 parsing</tags>
<title>ANTLR 4</title>
<summary>The C# target of the ANTLR 4 parser generator for Visual Studio 2010+ projects.</summary>
<dependencies>
<dependency id="Antlr4.Runtime" version="$version$" />
</dependencies>
</metadata>
<files>
<!-- Tools -->
<file src="..\tool\target\antlr4-csharp-$CSharpToolVersion$-complete.jar" target="tools"/>
<!-- Build Configuration -->
<file src="..\runtime\CSharp\Antlr4BuildTasks\bin\net40\$Configuration$\Antlr4.net40.props" target="build\Antlr4.props"/>
<file src="..\runtime\CSharp\Antlr4BuildTasks\bin\net40\$Configuration$\Antlr4.net40.targets" target="build\Antlr4.targets"/>
<file src="..\runtime\CSharp\Antlr4BuildTasks\bin\net40\$Configuration$\Antlr4BuildTasks.net40.dll" target="build"/>
</files>
</package>

139
build/build.ps1
View File

@ -1,139 +0,0 @@
param (
[switch]$Debug,
[string]$VisualStudioVersion = "12.0",
[switch]$NoClean,
[string]$Java6Home,
[string]$MavenHome,
[string]$MavenRepo = "$($env:USERPROFILE)\.m2",
[switch]$SkipMaven,
[switch]$SkipKeyCheck
)
# build the solutions
$SolutionPath = "..\Runtime\CSharp\Antlr4.sln"
$CF35SolutionPath = "..\Runtime\CSharp\Antlr4.VS2008.sln"
# make sure the script was run from the expected path
if (!(Test-Path $SolutionPath)) {
echo "The script was run from an invalid working directory."
exit 1
}
. .\version.ps1
If ($Debug) {
$BuildConfig = 'Debug'
} Else {
$BuildConfig = 'Release'
}
If ($NoClean) {
$Target = 'build'
} Else {
$Target = 'rebuild'
}
If (-not $MavenHome) {
$MavenHome = $env:M2_HOME
}
$Java6RegKey = 'HKLM:\SOFTWARE\JavaSoft\Java Runtime Environment\1.6'
$Java6RegValue = 'JavaHome'
If (-not $Java6Home -and (Test-Path $Java6RegKey)) {
$JavaHomeKey = Get-Item -LiteralPath $Java6RegKey
If ($JavaHomeKey.GetValue($Java6RegValue, $null) -ne $null) {
$JavaHomeProperty = Get-ItemProperty $Java6RegKey $Java6RegValue
$Java6Home = $JavaHomeProperty.$Java6RegValue
}
}
# this is configured here for path checking, but also in the .props and .targets files
[xml]$pom = Get-Content "..\tool\pom.xml"
$CSharpToolVersionNodeInfo = Select-Xml "/mvn:project/mvn:version" -Namespace @{mvn='http://maven.apache.org/POM/4.0.0'} $pom
$CSharpToolVersion = $CSharpToolVersionNodeInfo.Node.InnerText.trim()
# build the main project
$msbuild = "$env:windir\Microsoft.NET\Framework64\v4.0.30319\msbuild.exe"
&$msbuild '/nologo' '/m' '/nr:false' "/t:$Target" "/p:Configuration=$BuildConfig" "/p:VisualStudioVersion=$VisualStudioVersion" $SolutionPath
if ($LASTEXITCODE -ne 0) {
$host.ui.WriteErrorLine('Build failed, aborting!')
exit $p.ExitCode
}
# build the compact framework project
$msbuild = "$env:windir\Microsoft.NET\Framework\v4.0.30319\msbuild.exe"
&$msbuild '/nologo' '/m' '/nr:false' '/t:rebuild' "/p:Configuration=$BuildConfig" $CF35SolutionPath
if ($LASTEXITCODE -ne 0) {
$host.ui.WriteErrorLine('.NET 3.5 Compact Framework Build failed, aborting!')
exit $p.ExitCode
}
if (-not (Test-Path 'nuget')) {
mkdir "nuget"
}
# Build the Java library using Maven
If (-not $SkipMaven) {
$OriginalPath = $PWD
cd '..\tool'
$MavenPath = "$MavenHome\bin\mvn.bat"
If (-not (Test-Path $MavenPath)) {
$host.ui.WriteErrorLine("Couldn't locate Maven binary: $MavenPath")
cd $OriginalPath
exit 1
}
If (-not (Test-Path $Java6Home)) {
$host.ui.WriteErrorLine("Couldn't locate Java 6 installation: $Java6Home")
cd $OriginalPath
exit 1
}
$MavenGoal = 'package'
&$MavenPath '-DskipTests=true' '--errors' '-e' '-Dgpg.useagent=true' "-Djava6.home=$Java6Home" '-Psonatype-oss-release' $MavenGoal
if ($LASTEXITCODE -ne 0) {
$host.ui.WriteErrorLine('Maven build of the C# Target custom Tool failed, aborting!')
cd $OriginalPath
exit $p.ExitCode
}
cd $OriginalPath
}
$JarPath = "..\tool\target\antlr4-csharp-$CSharpToolVersion-complete.jar"
if (!(Test-Path $JarPath)) {
$host.ui.WriteErrorLine("Couldn't locate the complete jar used for building C# parsers: $JarPath")
exit 1
}
# By default, do not create a NuGet package unless the expected strong name key files were used
if (-not $SkipKeyCheck) {
. .\keys.ps1
foreach ($pair in $Keys.GetEnumerator()) {
$assembly = Resolve-FullPath -Path "..\runtime\CSharp\Antlr4.Runtime\bin\$($pair.Key)\$BuildConfig\Antlr4.Runtime.dll"
# Run the actual check in a separate process or the current process will keep the assembly file locked
powershell -Command ".\check-key.ps1 -Assembly '$assembly' -ExpectedKey '$($pair.Value)' -Build '$($pair.Key)'"
if ($LASTEXITCODE -ne 0) {
Exit $p.ExitCode
}
}
}
$packages = @(
'Antlr4.Runtime'
'Antlr4'
'Antlr4.VS2008')
$nuget = '..\runtime\CSharp\.nuget\NuGet.exe'
ForEach ($package in $packages) {
If (-not (Test-Path ".\$package.nuspec")) {
$host.ui.WriteErrorLine("Couldn't locate NuGet package specification: $package")
exit 1
}
&$nuget 'pack' ".\$package.nuspec" '-OutputDirectory' 'nuget' '-Prop' "Configuration=$BuildConfig" '-Version' "$AntlrVersion" '-Prop' "M2_REPO=$M2_REPO" '-Prop' "CSharpToolVersion=$CSharpToolVersion" '-Symbols'
}

31
build/check-key.ps1
View File

@ -1,31 +0,0 @@
param(
[string]$Assembly,
[string]$ExpectedKey,
[string]$Build = $null
)
function Get-PublicKeyToken() {
param([string]$assembly = $null)
if ($assembly) {
$bytes = $null
$bytes = [System.Reflection.Assembly]::ReflectionOnlyLoadFrom($assembly).GetName().GetPublicKeyToken()
if ($bytes) {
$key = ""
for ($i=0; $i -lt $bytes.Length; $i++) {
$key += "{0:x2}" -f $bytes[$i]
}
$key
}
}
}
if (-not $Build) {
$Build = $Assembly
}
$actual = Get-PublicKeyToken -assembly $Assembly
if ($actual -ne $ExpectedKey) {
$host.ui.WriteErrorLine("Invalid publicKeyToken for '$Build'; expected '$ExpectedKey' but found '$actual'")
exit 1
}

17
build/keys.ps1
View File

@ -1,17 +0,0 @@
# Note: these values may only change during minor release
$Keys = @{
'net20' = '7983ae52036899ac'
'net30' = '7671200403f6656a'
'net35-cf' = '770a97458f51159e'
'net35-client' = '4307381ae04f9aa7'
'net40-client' = 'bb1075973a9370c4'
'net45' = 'edc21c04cf562012'
'netcore45' = 'e4e9019902d0b6e2'
'portable-net40' = '90bf14da8e1462b4'
'portable-net45' = '3d23c8e77559f391'
}
function Resolve-FullPath() {
param([string]$Path)
[System.IO.Path]::GetFullPath((Join-Path (pwd) $Path))
}

29
build/push.ps1
View File

@ -1,29 +0,0 @@
. .\version.ps1
If ($AntlrVersion.EndsWith('-dev')) {
$host.ui.WriteErrorLine("Cannot push development version '$AntlrVersion' to NuGet.")
Exit 1
}
$packages = @(
'Antlr4.Runtime'
'Antlr4'
'Antlr4.VS2008')
# Make sure all packages exist before pushing any packages
ForEach ($package in $packages) {
If (-not (Test-Path ".\nuget\$package.$AntlrVersion.nupkg")) {
$host.ui.WriteErrorLine("Couldn't locate NuGet package: $JarPath")
exit 1
}
If (-not (Test-Path ".\nuget\$package.$AntlrVersion.symbols.nupkg")) {
$host.ui.WriteErrorLine("Couldn't locate NuGet symbols package: $JarPath")
exit 1
}
}
$nuget = '..\runtime\CSharp\.nuget\NuGet.exe'
ForEach ($package in $packages) {
&$nuget 'push' ".\nuget\$package.$AntlrVersion.nupkg"
}

1
build/version.ps1
View File

@ -1 +0,0 @@
$AntlrVersion = "4.5.1"

44
contributors.txt
View File

@ -208,6 +208,7 @@ YYYY/MM/DD, github id, Full name, email
2018/07/03, jgoppert, James Goppert, james.goppert@gmail.com
2018/07/27, Maksim Novikov, mnovikov.work@gmail.com
2018/08/03, ENDOH takanao, djmchl@gmail.com
2018/10/08, xsIceman, Andreas Skaar, andreas.skaar@gmail.com
2018/10/18, edirgarcia, Edir García Lazo, edirgl@hotmail.com
2018/07/31, Lucas Henrqiue, lucashenrique580@gmail.com
2018/08/03, ENDOH takanao, djmchl@gmail.com
@ -218,37 +219,78 @@ YYYY/MM/DD, github id, Full name, email
2018/11/29, hannemann-tamas, Ralf Hannemann-Tamas, ralf.ht@gmail.com
2018/12/20, WalterCouto, Walter Couto, WalterCouto@users.noreply.github.com
2018/12/23, youkaichao, Kaichao You, youkaichao@gmail.com
2019/01/01, khoroshilov, Alexey Khoroshilov, khoroshilov@ispras.ru
2019/01/02, wkhemir, Wail Khemir, khemirwail@gmail.com
2019/01/16, kuegi, Markus Zancolo, markus.zancolo@roomle.com
2019/02/06, ralucado, Cristina Raluca Vijulie, ralucris.v[at]gmail[dot]com
2019/02/23, gedimitr, Gerasimos Dimitriadis, gedimitr@gmail.com
2019/03/13, base698, Justin Thomas, justin.thomas1@gmail.com
2019/03/18, carlodri, Carlo Dri, carlo.dri@gmail.com
2019/05/02, askingalot, Andy Collins, askingalot@gmail.com
2019/05/13, mapio, Massimo Santini, massimo.santini@gmail.com
2019/07/11, olowo726, Olof Wolgast, olof@baah.se
2019/07/16, abhijithneilabraham, Abhijith Neil Abraham, abhijithneilabrahampk@gmail.com
2019/07/26, Braavos96, Eric Hettiaratchi, erichettiaratchi@gmail.com
2019/08/02, thron7, Thomas Herchenroeder, thron7[at]users[dot]sourceforge[dot]net
2019/08/12, easonlin404, Eason Lin, easonlin404@gmail.com
2019/08/23, akaJes, Oleksandr Mamchyts, akaJes@gmail.com
2019/08/27, wurzelpeter, Markus Franke, markus[hyphen]franke[at]web[dot]de
2019/09/10, ImanHosseini, Iman Hosseini, hosseini.iman@yahoo.com
2019/09/03, João Henrique, johnnyonflame@hotmail.com
2019/09/10, neko1235, Ihar Mokharau, igor.mohorev@gmail.com
2019/09/10, yar3333, Yaroslav Sivakov, yar3333@gmail.com
2019/09/10, marcospassos, Marcos Passos, marcospassos.com@gmail.com
2019/09/10, amorimjuliana, Juliana Amorim, juu.amorim@gmail.com
2019/09/15, sullis, Sean Sullivan, github@seansullivan.com
2019/09/17, kaz, Kazuki Sawada, kazuki@6715.jp
2019/09/28, lmy269, Mingyang Liu, lmy040758@gmail.com
2019/10/29, tehbone, Tabari Alexander, tehbone@gmail.com
2019/10/31, a-square, Alexei Averchenko, lex.aver@gmail.com
2019/11/05, listba, Ben List, ben.list89@gmail.com
2019/11/11, foxeverl, Liu Xinfeng, liuxf1986[at]gmail[dot]com
2019/11/17, felixn, Felix Nieuwenhuizhen, felix@tdlrali.com
2019/11/18, mlilback, Mark Lilback, mark@lilback.com
2020/01/19, lingyv-li, Lingyu Li, lingyv.li@gmail.com
2020/02/02, carocad, Camilo Roca, carocad@unal.edu.co
2020/02/10, julibert, Julián Bermúdez Ortega, julibert.dev@gmail.com
2020/02/17, quantumsheep, Nathanael Demacon, nathanael.dmc@outlook.fr
2020/02/21, StochasticTinkr, Daniel Pitts, github@coloraura.com
2020/03/17, XsongyangX, Song Yang, songyang1218@gmail.com
2020/04/07, deniskyashif, Denis Kyashif, denis.kyashif@gmail.com
2020/04/08, lwehmeier, Leon Wehmeier, wehmeier@st.ovgu.de
2020/04/10, agrabski, Adam Grabski, adam.gr@outlook.com
2020/04/23, martinvw, Martin van Wingerden, martin@martinvw.nl
2020/04/30, TristonianJones, Tristan Swadell, tswadell@google.com
2020/05/06, iammosespaulr, Moses Paul R, iammosespaulr@gmail.com
2020/05/10, gomerser, Erik Gomersbach, gomerser@gomersba.ch
2020/05/22, keywan-ghadami-oxid, Keywan Ghadami, keywan.ghadami@oxid-esales.com
2020/05/25, graknol, Sindre van der Linden, graknol@gmail.com
2020/05/31, d-markey, David Markey, dmarkey@free.fr
2020/06/02, cohomology, Kilian Kilger, kkilger AT gmail.com
2020/06/04, IohannRabeson, Iohann Rabeson, iotaka6@gmail.com
2020/06/04, sigmasoldi3r, Pablo Blanco, pablobc.1995@gmail.com
2020/06/15, mattpaletta, Matthew Paletta, mattpaletta@gmail.com
2020/07/01, sha-N, Shan M Mathews, admin@bluestarqatar.com
2020/07/01, sha-N, Shan M Mathews, admin@bluestarqatar.com
2020/08/22, stevenjohnstone, Steven Johnstone, steven.james.johnstone@gmail.com
2020/09/06, ArthurSonzogni, Sonzogni Arthur, arthursonzogni@gmail.com
2020/09/10, Khailian, Arunav Sanyal, arunav.sanyal91@gmail.com
2020/09/12, Clcanny, Charles Ruan, a837940593@gmail.com
2020/09/15, rmcgregor1990, Robert McGregor, rmcgregor1990@gmail.com
2020/09/16, trenki2, Markus Trenkwalder, trenki2[at]gmx[dot]net
2020/10/08, Marti2203, Martin Mirchev, mirchevmartin2203@gmail.com
2020/10/16, adarshbhat, Adarsh Bhat, adarshbhat@users.noreply.github.com
2020/10/20, adamwojs, Adam Wójs, adam[at]wojs.pl
2020/10/24, cliid, Jiwu Jang, jiwujang@naver.com
2020/11/05, MichelHartmann, Michel Hartmann, MichelHartmann@users.noreply.github.com
2020/11/26, mr-c, Michael R. Crusoe, 1330696+mr-c@users.noreply.github.com
2020/12/01, maxence-lefebvre, Maxence Lefebvre, maxence-lefebvre@users.noreply.github.com
2020/12/03, electrum, David Phillips, david@acz.org
2021/01/25, l215884529, Qiheng Liu, 13607681+l215884529@users.noreply.github.com
2021/02/02, tsotnikov, Taras Sotnikov, taras.sotnikov@gmail.com
2021/02/10, jirislaby, Jiri Slaby, jirislaby@gmail.com
2021/02/21, namasikanam, Xingyu Xie, namasikanam@gmail.com
2021/02/27, khmarbaise, Karl Heinz Marbaise, github@soebes.com
2021/03/02, hackeris
2021/03/03, xTachyon, Damian Andrei, xTachyon@users.noreply.github.com
2021/04/08, bigerl, Alexander Bigerl, bigerl@mail.upb.de
2021/04/07, b1f6c1c4, Jinzheng Tu, b1f6c1c4@gmail.com

2
doc/actions.md
View File

@ -84,7 +84,7 @@ Using a rule label looks like this:
returnStat : 'return' e=expr {System.out.println("matched "+e.text);} ;
```
You can also use `$ followed by the name of the attribute to access the value associated with the currently executing rule. For example, `$start` is the starting token of the current rule.
You can also use `$` followed by the name of the attribute to access the value associated with the currently executing rule. For example, `$start` is the starting token of the current rule.
```
returnStat : 'return' expr {System.out.println("first token "+$start.getText());} ;

8
doc/antlr-project-testing.md
View File

@ -57,11 +57,9 @@ $ mvn install -DskipTests=true # make sure all artifacts are visible on this m
Now, make sure C# runtime is built and installed locally.
```bash
cd ~/antlr/code/antlr4/runtime/CSharp/runtime/CSharp
# kill previous ones manually as "xbuild /t:Clean" didn't seem to do it
find . -name '*.dll' -exec rm {} \;
# build
xbuild /p:Configuration=Release Antlr4.Runtime/Antlr4.Runtime.mono.csproj
cd ~/antlr/code/antlr4/runtime/CSharp/src
rm -rf `find . -name '{obj,bin}'`
dotnet build -c Release runtime/CSharp/src/Antlr4.csproj
```
C++ test rig automatically builds C++ runtime during tests. Others don't need a prebuilt lib.

16
doc/building-antlr.md
View File

@ -25,6 +25,22 @@ Checking connectivity... done.
Checking out files: 100% (1427/1427), done.
```
# Check your environment
If you are starting from a clean, minimum Ubuntu OS, check your environment.
```bash
$ sudo apt-get update
$ # Get Java
$ java > /dev/null 2>&1
$ if [[ "$?" != "0" ]]; then sudo apt install -y openjdk-11-jre-headless; fi
$ # Get Mvn
$ mvn > /dev/null 2>&1
$ if [[ "$?" != "0" ]]; then sudo apt install -y maven; fi
```
# Compile
```bash

6
doc/cpp-target.md
View File

@ -1,6 +1,6 @@
# C++
The C++ target supports all platforms that can either run MS Visual Studio 2013 (or newer), XCode 7 (or newer) or CMake (C++11 required). All build tools can either create static or dynamic libraries, both as 64bit or 32bit arch. Additionally, XCode can create an iOS library. Also see [Antlr4 for C++ with CMake: A practical example](http://blorente.me//Antlr-,-C++-and-CMake-Wait-what.html).
The C++ target supports all platforms that can either run MS Visual Studio 2013 (or newer), XCode 7 (or newer) or CMake (C++11 required). All build tools can either create static or dynamic libraries, both as 64bit or 32bit arch. Additionally, XCode can create an iOS library. Also see [Antlr4 for C++ with CMake: A practical example](http://blorente.me/beyond-the-loop/Antlr-cpp-cmake/).
## How to create a C++ lexer or parser?
This is pretty much the same as creating a Java lexer or parser, except you need to specify the language target, for example:
@ -65,7 +65,7 @@ int main(int argc, const char* argv[]) {
tree::ParseTree *tree = parser.key();
TreeShapeListener listener;
tree::ParseTreeWalker::DEFAULT->walk(&listener, tree);
tree::ParseTreeWalker::DEFAULT.walk(&listener, tree);
return 0;
}
@ -74,7 +74,7 @@ int main(int argc, const char* argv[]) {
This example assumes your grammar contains a parser rule named `key` for which the enterKey function was generated.
## Specialities of this ANTLR target
## Special cases for this ANTLR target
There are a couple of things that only the C++ ANTLR target has to deal with. They are described here.

4
doc/csharp-target.md
View File

@ -36,7 +36,7 @@ using Antlr4.Runtime.Tree;
public void MyParseMethod() {
String input = "your text to parse here";
ICharStream stream = CharStreams.fromstring(input);
ICharStream stream = CharStreams.fromString(input);
ITokenSource lexer = new MyGrammarLexer(stream);
ITokenStream tokens = new CommonTokenStream(lexer);
MyGrammarParser parser = new MyGrammarParser(tokens);
@ -86,7 +86,7 @@ In order to execute this listener, you would simply add the following lines to t
...
IParseTree tree = parser.StartRule() - only repeated here for reference
KeyPrinter printer = new KeyPrinter();
ParseTreeWalker.DEFAULT.walk(printer, tree);
ParseTreeWalker.Default.Walk(printer, tree);
```
Further information can be found from The Definitive ANTLR Reference book.

4
doc/faq/index.md
View File

@ -38,8 +38,8 @@ This is the main landing page for the ANTLR 4 FAQ. The links below will take you
## Translation
* [ASTs vs parse trees](parse-trees.md)
* [Decoupling input walking from output generation](parse-trees.md)
* [ASTs vs parse trees](translation.md)
* [Decoupling input walking from output generation](translation.md)
## Actions and semantic predicates

4
doc/faq/parse-trees.md
View File

@ -50,7 +50,7 @@ For writing a compiler, either generate [LLVM-type static-single-assignment](htt
### XPath
XPath works great when you need to find specific nodes, possibly in certain contexts. The context is limited to the parents on the way to the root of the tree. For example, if you want to find all ID nodes, use path `//ID`. If you want all variable declarations, you might use path `//vardecl`. If you only want fields declarations, then you can use some context information via path `/classdef/vardecl`, which would only find vardecls that our children of class definitions. You can merge the results of multiple XPath `findAll()`s simulating a set union for XPath. The only caveat is that the order from the original tree is not preserved when you union multiple `findAll()` sets.
XPath works great when you need to find specific nodes, possibly in certain contexts. The context is limited to the parents on the way to the root of the tree. For example, if you want to find all ID nodes, use path `//ID`. If you want all variable declarations, you might use path `//vardecl`. If you only want fields declarations, then you can use some context information via path `/classdef/vardecl`, which would only find vardecls that are children of class definitions. You can merge the results of multiple XPath `findAll()`s simulating a set union for XPath. The only caveat is that the order from the original tree is not preserved when you union multiple `findAll()` sets.
### Tree pattern matching
@ -70,4 +70,4 @@ scopeStack.peek().define(new VariableSymbol("foo"))
That way each listener function does not have to compute its appropriate scope.
Examples: [DefScopesAndSymbols.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/DefScopesAndSymbols.java) and [SetScopeListener.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/SetScopeListener.java) and [VerifyListener.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/VerifyListener.java)
Examples: [DefScopesAndSymbols.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/DefScopesAndSymbols.java) and [SetScopeListener.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/SetScopeListener.java) and [VerifyListener.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/VerifyListener.java)

31
doc/getting-started.md
View File

@ -6,46 +6,46 @@ Hi and welcome to the version 4 release of ANTLR! It's named after the fearless
ANTLR is really two things: a tool that translates your grammar to a parser/lexer in Java (or other target language) and the runtime needed by the generated parsers/lexers. Even if you are using the ANTLR Intellij plug-in or ANTLRWorks to run the ANTLR tool, the generated code will still need the runtime library.
The first thing you should do is probably download and install a development tool plug-in. Even if you only use such tools for editing, they are great. Then, follow the instructions below to get the runtime environment available to your system to run generated parsers/lexers. In what follows, I talk about antlr-4.7.1-complete.jar, which has the tool and the runtime and any other support libraries (e.g., ANTLR v4 is written in v3).
The first thing you should do is probably download and install a development tool plug-in. Even if you only use such tools for editing, they are great. Then, follow the instructions below to get the runtime environment available to your system to run generated parsers/lexers. In what follows, I talk about antlr-4.9-complete.jar, which has the tool and the runtime and any other support libraries (e.g., ANTLR v4 is written in v3).
If you are going to integrate ANTLR into your existing build system using mvn, ant, or want to get ANTLR into your IDE such as eclipse or intellij, see Integrating ANTLR into Development Systems.
If you are going to integrate ANTLR into your existing build system using mvn, ant, or want to get ANTLR into your IDE such as eclipse or intellij, see [Integrating ANTLR into Development Systems](https://github.com/antlr/antlr4/blob/master/doc/IDEs.md).
### UNIX
0. Install Java (version 1.6 or higher)
0. Install Java (version 1.7 or higher)
1. Download
```
$ cd /usr/local/lib
$ curl -O https://www.antlr.org/download/antlr-4.7.1-complete.jar
$ curl -O https://www.antlr.org/download/antlr-4.9-complete.jar
```
Or just download in browser from website:
[https://www.antlr.org/download.html](https://www.antlr.org/download.html)
and put it somewhere rational like `/usr/local/lib`.
2. Add `antlr-4.7.1-complete.jar` to your `CLASSPATH`:
2. Add `antlr-4.9-complete.jar` to your `CLASSPATH`:
```
$ export CLASSPATH=".:/usr/local/lib/antlr-4.7.1-complete.jar:$CLASSPATH"
$ export CLASSPATH=".:/usr/local/lib/antlr-4.9-complete.jar:$CLASSPATH"
```
It's also a good idea to put this in your `.bash_profile` or whatever your startup script is.
3. Create aliases for the ANTLR Tool, and `TestRig`.
```
$ alias antlr4='java -Xmx500M -cp "/usr/local/lib/antlr-4.7.1-complete.jar:$CLASSPATH" org.antlr.v4.Tool'
$ alias grun='java -Xmx500M -cp "/usr/local/lib/antlr-4.7.1-complete.jar:$CLASSPATH" org.antlr.v4.gui.TestRig'
$ alias antlr4='java -Xmx500M -cp "/usr/local/lib/antlr-4.9-complete.jar:$CLASSPATH" org.antlr.v4.Tool'
$ alias grun='java -Xmx500M -cp "/usr/local/lib/antlr-4.9-complete.jar:$CLASSPATH" org.antlr.v4.gui.TestRig'
```
### WINDOWS
(*Thanks to Graham Wideman*)
0. Install Java (version 1.6 or higher)
1. Download antlr-4.7.1-complete.jar (or whatever version) from [https://www.antlr.org/download/](https://www.antlr.org/download/)
0. Install Java (version 1.7 or higher)
1. Download antlr-4.9-complete.jar (or whatever version) from [https://www.antlr.org/download/](https://www.antlr.org/download/)
Save to your directory for 3rd party Java libraries, say `C:\Javalib`
2. Add `antlr-4.7.1-complete.jar` to CLASSPATH, either:
2. Add `antlr-4.9-complete.jar` to CLASSPATH, either:
* Permanently: Using System Properties dialog > Environment variables > Create or append to `CLASSPATH` variable
* Temporarily, at command line:
```
SET CLASSPATH=.;C:\Javalib\antlr-4.7.1-complete.jar;%CLASSPATH%
SET CLASSPATH=.;C:\Javalib\antlr-4.9-complete.jar;%CLASSPATH%
```
3. Create short convenient commands for the ANTLR Tool, and TestRig, using batch files or doskey commands:
* Batch files (in directory in system PATH) antlr4.bat and grun.bat
@ -71,7 +71,7 @@ Either launch org.antlr.v4.Tool directly:
```
$ java org.antlr.v4.Tool
ANTLR Parser Generator Version 4.7.1
ANTLR Parser Generator Version 4.9
-o ___ specify output directory where all output is generated
-lib ___ specify location of .tokens files
...
@ -80,8 +80,8 @@ ANTLR Parser Generator Version 4.7.1
or use -jar option on java:
```
$ java -jar /usr/local/lib/antlr-4.7.1-complete.jar
ANTLR Parser Generator Version 4.7.1
$ java -jar /usr/local/lib/antlr-4.9-complete.jar
ANTLR Parser Generator Version 4.9
-o ___ specify output directory where all output is generated
-lib ___ specify location of .tokens files
...
@ -138,3 +138,4 @@ The book has lots and lots of examples that should be useful too. You can downlo
Also, there is a large collection of grammars for v4 at github:
[https://github.com/antlr/grammars-v4](https://github.com/antlr/grammars-v4)
/

2
doc/go-target.md
View File

@ -65,7 +65,7 @@ Another common option to the ANTLR tool is `-visitor`, which generates a parse t
We'll write a small main func to call the generated parser/lexer (assuming they are separate). This one writes out the encountered `ParseTreeContext`'s. Suppose the gen'ed parser code is in the `parser` directory relative to this code:
```
```golang
package main
import (

2
doc/java-target.md
View File

@ -139,7 +139,7 @@ Edit the pom.xml file. Now we need to extensively modify the pom.xml file. The f
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<version>4.13.1</version>
</dependency>
</dependencies>
<build>

99
doc/javascript-target.md
View File

@ -11,11 +11,18 @@ In practice, this target has been extensively tested against:
* Chrome 39.0.2171
* Explorer 11.0.3
The tests were conducted using Selenium. No issue was found, so you should find that the runtime works pretty much against any recent JavaScript engine.
The above tests were conducted using Selenium. No issue was found, so you should find that the runtime works pretty much against any recent JavaScript engine.
## Is NodeJS supported?
The runtime has also been extensively tested against Node.js 10 LTS. No issue was found.
The runtime has also been extensively tested against Node.js 14 LTS. No issue was found.
NodeJS together with a packaging tool is now the preferred development path, developers are encouraged to follow it.
## What about modules?
Starting with version 8.1, Antlr4 JavaScript runtime follows esm semantics (see https://tc39.es/ecma262/#sec-modules for details)
Generated lexers, parsers, listeners and visitors also follow this new standard.
If you have used previous versions of the runtime, you will need to migrate and make your parser a module.
## How to create a JavaScript lexer or parser?
@ -61,7 +68,10 @@ The steps to create your parsing code are the following:
You are now ready to bundle your parsing code as follows:
- following webpack specs, create a webpack.config file
- in the `webpack.config` file, exclude node.js only modules using: `node: { module: "empty", net: "empty", fs: "empty" }`
- For Webpack version 5,
- in the `webpack.config` file, exclude node.js only modules using: `resolve: { fallback: { fs: false } }`
- For older versions of Webpack,
- in the `webpack.config` file, exclude node.js only modules using: `node: { module: "empty", net: "empty", fs: "empty" }`
- from the cmd line, navigate to the directory containing webpack.config and type: webpack
This will produce a single js file containing all your parsing code. Easy to include in your web pages!
@ -80,18 +90,18 @@ Let's suppose that your grammar is named, as above, "MyGrammar". Let's suppose t
Now a fully functioning script might look like the following:
```javascript
var antlr4 = require('antlr4');
var MyGrammarLexer = require('./MyGrammarLexer').MyGrammarLexer;
var MyGrammarParser = require('./MyGrammarParser').MyGrammarParser;
var MyGrammarListener = require('./MyGrammarListener').MyGrammarListener;
import antlr4 from 'antlr4';
import MyGrammarLexer from './MyGrammarLexer.js';
import MyGrammarParser from './MyGrammarParser.js';
import MyGrammarListener from './MyGrammarListener.js';
var input = "your text to parse here"
var chars = new antlr4.InputStream(input);
var lexer = new MyGrammarLexer(chars);
var tokens = new antlr4.CommonTokenStream(lexer);
var parser = new MyGrammarParser(tokens);
const input = "your text to parse here"
const chars = new antlr4.InputStream(input);
const lexer = new MyGrammarLexer(chars);
const tokens = new antlr4.CommonTokenStream(lexer);
const parser = new MyGrammarParser(tokens);
parser.buildParseTrees = true;
var tree = parser.MyStartRule();
const tree = parser.MyStartRule();
```
This program will work. But it won't be useful unless you do one of the following:
@ -105,19 +115,19 @@ This program will work. But it won't be useful unless you do one of the followin
## How do I create and run a visitor?
```javascript
// test.js
var antlr4 = require('antlr4');
var MyGrammarLexer = require('./QueryLexer').QueryLexer;
var MyGrammarParser = require('./QueryParser').QueryParser;
var MyGrammarListener = require('./QueryListener').QueryListener;
import antlr4 from 'antlr4';
import MyGrammarLexer from './QueryLexer.js';
import MyGrammarParser from './QueryParser.js';
import MyGrammarListener from './QueryListener.js';
var input = "field = 123 AND items in (1,2,3)"
var chars = new antlr4.InputStream(input);
var lexer = new MyGrammarLexer(chars);
var tokens = new antlr4.CommonTokenStream(lexer);
var parser = new MyGrammarParser(tokens);
const input = "field = 123 AND items in (1,2,3)"
const chars = new antlr4.InputStream(input);
const lexer = new MyGrammarLexer(chars);
const tokens = new antlr4.CommonTokenStream(lexer);
const parser = new MyGrammarParser(tokens);
parser.buildParseTrees = true;
var tree = parser.query();
const tree = parser.query();
class Visitor {
visitChildren(ctx) {
@ -145,40 +155,37 @@ tree.accept(new Visitor());
Let's suppose your MyGrammar grammar comprises 2 rules: "key" and "value". The antlr4 tool will have generated the following listener:
```javascript
MyGrammarListener = function(ParseTreeListener) {
// some code here
}
// some code here
MyGrammarListener.prototype.enterKey = function(ctx) {};
MyGrammarListener.prototype.exitKey = function(ctx) {};
MyGrammarListener.prototype.enterValue = function(ctx) {};
MyGrammarListener.prototype.exitValue = function(ctx) {};
class MyGrammarListener extends ParseTreeListener {
constructor() {
super();
}
enterKey(ctx) {}
exitKey(ctx) {}
enterValue(ctx) {}
exitValue(ctx) {}
}
```
In order to provide custom behavior, you might want to create the following class:
```javascript
var KeyPrinter = function() {
MyGrammarListener.call(this); // inherit default listener
return this;
};
class KeyPrinter extends MyGrammarListener {
// continue inheriting default listener
KeyPrinter.prototype = Object.create(MyGrammarListener.prototype);
KeyPrinter.prototype.constructor = KeyPrinter;
// override default listener behavior
KeyPrinter.prototype.exitKey = function(ctx) {
console.log("Oh, a key!");
};
// override default listener behavior
exitKey(ctx) {
console.log("Oh, a key!");
}
}
```
In order to execute this listener, you would simply add the following lines to the above code:
```javascript
...
tree = parser.StartRule() // only repeated here for reference
var printer = new KeyPrinter();
...
tree = parser.StartRule() // only repeated here for reference
const printer = new KeyPrinter();
antlr4.tree.ParseTreeWalker.DEFAULT.walk(printer, tree);
```

11
doc/lexer-rules.md
View File

@ -25,7 +25,7 @@ fragment DIGIT : [0-9] ; // not a token by itself
## Lexical Modes
Modes allow you to group lexical rules by context, such as inside and outside of XML tags. It’s like having multiple sublexers, one for context. The lexer can only return tokens matched by entering a rule in the current mode. Lexers start out in the so-called default mode. All rules are considered to be within the default mode unless you specify a mode command. Modes are not allowed within combined grammars, just lexer grammars. (See grammar `XMLLexer` from [Tokenizing XML](http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-reference).)
Modes allow you to group lexical rules by context, such as inside and outside of XML tags. It’s like having multiple sublexers, one for each context. The lexer can only return tokens matched by entering a rule in the current mode. Lexers start out in the so-called default mode. All rules are considered to be within the default mode unless you specify a mode command. Modes are not allowed within combined grammars, just lexer grammars. (See grammar `XMLLexer` from [Tokenizing XML](http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-reference).)
```
rules in default mode
@ -58,7 +58,7 @@ Match that character or sequence of characters. E.g., ’while’ or ’=’.</t
<tr>
<td>[char set]</td><td>
<p>Match one of the characters specified in the character set. Interpret <tt>x-y</tt> as the set of characters between range <tt>x</tt> and <tt>y</tt>, inclusively. The following escaped characters are interpreted as single special characters: <tt>\n</tt>, <tt>\r</tt>, <tt>\b</tt>, <tt>\t</tt>, <tt>\f</tt>, <tt>\uXXXX</tt>, and <tt>\u{XXXXXX}</tt>. To get <tt>]</tt>, <tt>\</tt>, or <tt>-</tt> you must escape them with <tt>\</tt>.</p>
<p>Match one of the characters specified in the character set. Interpret <tt>x-y</tt> as the set of characters between range <tt>x</tt> and <tt>y</tt>, inclusively. The following escaped characters are interpreted as single special characters: <tt>\n</tt>, <tt>\r</tt>, <tt>\b</tt>, <tt>\t</tt>, <tt>\f</tt>, <tt>\uXXXX</tt>, and <tt>\u{XXXXXX}</tt>. To get <tt>]</tt> or <tt>\</tt> you must escape them with <tt>\</tt>. To get <tt>-</tt> you must escape it with <tt>\</tt> too, except for the case when <tt>-</tt> is the first or last character in the set.</p>
<p>You can also include all characters matching Unicode properties (general category, boolean, or enumerated including scripts and blocks) with <tt>\p{PropertyName}</tt> or <tt>\p{EnumProperty=Value}</tt>. (You can invert the test with <tt>\P{PropertyName}</tt> or <tt>\P{EnumProperty=Value}</tt>).</p>
@ -90,6 +90,8 @@ UNICODE_ID : [\p{Alpha}\p{General_Category=Other_Letter}] [\p{Alnum}\p{General_C
EMOJI : [\u{1F4A9}\u{1F926}] ; // note Unicode code points > U+FFFF
DASHBRACK : [\-\]]+ ; // match - or ] one or more times
DASH : [---] ; // match a single -, i.e., "any character" between - and - (note first and last - not escaped)
</pre>
</td>
</tr>
@ -123,7 +125,7 @@ ESC : '\\' . ; // match any escaped \x character
<tr>
<td>{«action»}</td><td>
Lexer actions can appear anywhere as of 4.2, not just at the end of the outermost alternative. The lexer executes the actions at the appropriate input position, according to the placement of the action within the rule. To execute a single action for a role that has multiple alternatives, you can enclose the alts in parentheses and put the action afterwards:
Lexer actions can appear anywhere as of 4.2, not just at the end of the outermost alternative. The lexer executes the actions at the appropriate input position, according to the placement of the action within the rule. To execute a single action for a rule that has multiple alternatives, you can enclose the alts in parentheses and put the action afterwards:
<pre>
END : ('endif'|'end') {System.out.println("found an end");} ;
@ -244,7 +246,8 @@ The mode commands alter the mode stack and hence the mode of the lexer. The 'mor
```
// Default "mode": Everything OUTSIDE of a tag
COMMENT : '<!--' .*? '-->' ;
CDATA : '<![CDATA[' .*? ']]>' ;OPEN : '<' -> pushMode(INSIDE) ;
CDATA : '<![CDATA[' .*? ']]>' ;
OPEN : '<' -> pushMode(INSIDE) ;
...
XMLDeclOpen : '<?xml' S -> pushMode(INSIDE) ;
SPECIAL_OPEN: '<?' Name -> more, pushMode(PROC_INSTR) ;

8
doc/lexicon.md
View File

@ -26,8 +26,8 @@ The Javadoc comments are hidden from the parser and are ignored at the moment.
Token names always start with a capital letter and so do lexer rules as defined by Java’s `Character.isUpperCase` method. Parser rule names always start with a lowercase letter (those that fail `Character.isUpperCase`). The initial character can be followed by uppercase and lowercase letters, digits, and underscores. Here are some sample names:
```
ID, LPAREN, RIGHT_CURLY // token names/rules
expr, simpleDeclarator, d2, header_file // rule names
ID, LPAREN, RIGHT_CURLY // token names/lexer rules
expr, simpleDeclarator, d2, header_file // parser rule names
```
Like Java, ANTLR accepts Unicode characters in ANTLR names:
@ -79,7 +79,7 @@ These more or less correspond to `isJavaIdentifierPart` and `isJavaIdentifierSta
## Literals
ANTLR does not distinguish between character and string literals as most languages do. All literal strings one or more characters in length are enclosed in single quotes such as `’;’`, `’if’`, `’>=’`, and `’\’` (refers to the one-character string containing the single quote character). Literals never contain regular expressions.
ANTLR does not distinguish between character and string literals as most languages do. All literal strings one or more characters in length are enclosed in single quotes such as `’;’`, `’if’`, `’>=’`, and `’\’’` (refers to the one-character string containing the single quote character). Literals never contain regular expressions.
Literals can contain Unicode escape sequences of the form `’\uXXXX’` (for Unicode code points up to `’U+FFFF’`) or `’\u{XXXXXX}’` (for all Unicode code points), where `’XXXX’` is the hexadecimal Unicode code point value.
@ -96,7 +96,7 @@ The recognizers that ANTLR generates assume a character vocabulary containing al
## Actions
Actions are code blocks written in the target language. You can use actions in a number of places within a grammar, but the syntax is always the same: arbitrary text surrounded by curly braces. You don’t need to escape a closing curly character if it’s in a string or comment: `"}"` or `/*}*/`. If the curlies are balanced, you also don’t need to escape }: `{...}`. Otherwise, escape extra curlies with a backslash: `\{` or `\}`. The action text should conform to the target language as specified with thelanguage option.
Actions are code blocks written in the target language. You can use actions in a number of places within a grammar, but the syntax is always the same: arbitrary text surrounded by curly braces. You don’t need to escape a closing curly character if it’s in a string or comment: `"}"` or `/*}*/`. If the curlies are balanced, you also don’t need to escape }: `{...}`. Otherwise, escape extra curlies with a backslash: `\{` or `\}`. The action text should conform to the target language as specified with the language option.
Embedded code can appear in: `@header` and `@members` named actions, parser and lexer rules, exception catching specifications, attribute sections for parser rules (return values, arguments, and locals), and some rule element options (currently predicates).

4
doc/php-target.md
View File

@ -15,7 +15,7 @@ generated by ANTLR4. The runtime provides a common set of tools for using your p
Install the runtime with Composer:
```bash
composer install antlr/antlr4
composer require antlr/antlr4-php-runtime
```
#### 3. Generate your parser
@ -108,4 +108,4 @@ The expected output is:
{"a":1}
"a":1
1
```
```

168
doc/releasing-antlr.md
View File

@ -9,41 +9,41 @@ Create a pre-release or full release at github; [Example 4.5-rc-1](https://githu
Wack any existing tag as mvn will create one and it fails if already there.
```
$ git tag -d 4.8
$ git push origin :refs/tags/4.8
$ git push upstream :refs/tags/4.8
$ git tag -d 4.9
$ git push origin :refs/tags/4.9
$ git push upstream :refs/tags/4.9
```
### Create release candidate tag
```bash
$ git tag -a 4.8-rc1 -m 'heading towards 4.8'
$ git push origin 4.8-rc1
$ git push upstream 4.8-rc1
$ git tag -a 4.9-rc1 -m 'heading towards 4.9'
$ git push origin 4.9-rc1
$ git push upstream 4.9-rc1
```
## Update submodules
## Copy PHP runtime over
Make sure you tell git to pull in the submodule (for every clone you do of antlr4):
Bump version to 4.9 in `~/antlr/code/antlr-php-runtime/src/RuntimeMetaData.php` in separate repository and commit plus push.
```bash
git submodule init
```
cd ~/antlr/code/antlr-php-runtime/src
... vi RuntimeMetaData.php ...
git commit -a -m "Update PHP Runtime to latest version"
```
Also bump version to 4.8 in `runtime/PHP/src/RuntimeMetaData.php`.
them back over in the ANTLR repo:
Update the runtime submodules by running the following command:
```bash
git submodule update --recursive
git submodule update --remote --merge # might only need this last one but do both
```
Make sure these changes go back to antlr4 repo:
```bash
git add runtime/PHP
git commit -m "Update PHP Runtime to latest version"
cd runtime
rm -rf PHP
mkdir PHP
cp ~/antlr/code/antlr-php-runtime/LICENSE PHP
cp ~/antlr/code/antlr-php-runtime/README.md PHP
cp ~/antlr/code/antlr-php-runtime/composer.json PHP
cp ~/antlr/code/antlr-php-runtime/phpcs.xml.dist PHP
cp ~/antlr/code/antlr-php-runtime/phpstan.neon.dist PHP
cp -r ~/antlr/code/antlr-php-runtime/src PHP
```
## Bump version
@ -55,10 +55,11 @@ Edit the repository looking for 4.5 or whatever and update it. Bump version in t
* runtime/Python2/src/antlr4/Recognizer.py
* runtime/Python3/setup.py
* runtime/Python3/src/antlr4/Recognizer.py
* runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs
* runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj
* runtime/CSharp/src/Antlr4.csproj
* runtime/PHP/src/RuntimeMetaData.php
* runtime/JavaScript/package.json
* runtime/JavaScript/src/antlr4/Recognizer.js
* runtime/JavaScript/package-lock.json
* runtime/Cpp/VERSION
* runtime/Cpp/runtime/src/RuntimeMetaData.cpp
* runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake
@ -67,6 +68,11 @@ Edit the repository looking for 4.5 or whatever and update it. Bump version in t
* runtime/Swift/Antlr4/org/antlr/v4/runtime/RuntimeMetaData.swift
* runtime/Dart/lib/src/runtime_meta_data.dart
* runtime/Dart/pubspec.yaml
* runtime/Swift/Tests/Antlr4Tests/RuntimeMetaDataTests.swift
* runtime/Swift/Sources/Antlr4/RuntimeMetaData.swift
* runtime/CSharp/src/Tree/Xpath/XPathLexer.cs
* runtime/CSharp/src/README.md
* runtime/CSharp/src/Properties/AssemblyInfo.cs
* tool/src/org/antlr/v4/codegen/target/GoTarget.java
* tool/src/org/antlr/v4/codegen/target/CppTarget.java
* tool/src/org/antlr/v4/codegen/target/CSharpTarget.java
@ -74,13 +80,20 @@ Edit the repository looking for 4.5 or whatever and update it. Bump version in t
* tool/src/org/antlr/v4/codegen/target/Python2Target.java
* tool/src/org/antlr/v4/codegen/target/Python3Target.java
* tool/src/org/antlr/v4/codegen/target/SwiftTarget.java
* tool/src/org/antlr/v4/codegen/target/PHPTarget.java
* tool/src/org/antlr/v4/codegen/Target.java
* tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg
Here is a simple script to display any line from the critical files with, say, `4.5` in it:
Here is a simple script to display any line from the critical files with, say, `4.9` in it:
```bash
find tool runtime -type f -exec grep -l '4\.6' {} \;
mvn clean
rm -rf runtime/CSharp/src/bin
rm -rf runtime/CSharp/src/obj
rm -rf runtime/Cpp/runtime/build
rm -rf runtime/gen
rm -rf runtime/JavaScript/dist
find tool runtime -type f -exec grep -l '4\.9' {} \;
```
Commit to repository.
@ -136,23 +149,24 @@ Here is the file template
## Maven deploy snapshot
The goal is to get a snapshot, such as `4.8-SNAPSHOT`, to the staging server: [antlr4 tool](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4) and [antlr4 java runtime](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-runtime).
The goal is to get a snapshot, such as `4.9-SNAPSHOT`, to the staging server: [antlr4 tool](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4) and [antlr4 java runtime](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-runtime).
Do this:
```bash
$ mvn install -DskipTests # seems required to get the jar files visible to maven
$ mvn deploy -DskipTests
...
[INFO] --- maven-deploy-plugin:2.7:deploy (default-deploy) @ antlr4-tool-testsuite ---
Downloading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.8-SNAPSHOT/maven-metadata.xml
Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.8-SNAPSHOT/antlr4-tool-testsuite-4.8-20161211.173752-1.jar
Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.8-SNAPSHOT/antlr4-tool-testsuite-4.8-20161211.173752-1.jar (3 KB at 3.4 KB/sec)
Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.8-SNAPSHOT/antlr4-tool-testsuite-4.8-20161211.173752-1.pom
Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.8-SNAPSHOT/antlr4-tool-testsuite-4.8-20161211.173752-1.pom (3 KB at 6.5 KB/sec)
Downloading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.9-SNAPSHOT/maven-metadata.xml
Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.9-SNAPSHOT/antlr4-tool-testsuite-4.9-20161211.173752-1.jar
Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.9-SNAPSHOT/antlr4-tool-testsuite-4.9-20161211.173752-1.jar (3 KB at 3.4 KB/sec)
Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.9-SNAPSHOT/antlr4-tool-testsuite-4.9-20161211.173752-1.pom
Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.9-SNAPSHOT/antlr4-tool-testsuite-4.9-20161211.173752-1.pom (3 KB at 6.5 KB/sec)
Downloading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml
Downloaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml (371 B at 1.4 KB/sec)
Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.8-SNAPSHOT/maven-metadata.xml
Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.8-SNAPSHOT/maven-metadata.xml (774 B at 1.8 KB/sec)
Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.9-SNAPSHOT/maven-metadata.xml
Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.9-SNAPSHOT/maven-metadata.xml (774 B at 1.8 KB/sec)
Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml
Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml (388 B at 0.9 KB/sec)
[INFO] ------------------------------------------------------------------------
@ -198,10 +212,10 @@ export GPG_TTY=$(tty)
Side note to set jdk 1.7 on os x:
```bash
alias java='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/java'
alias javac='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/javac'
alias javadoc='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/javadoc'
alias jar='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/jar'
alias java="`/usr/libexec/java_home -v 1.7`/bin/java"
alias javac="`/usr/libexec/java_home -v 1.7`/bin/javac"
alias javadoc="`/usr/libexec/java_home -v 1.7`/bin/javadoc"
alias jar="`/usr/libexec/java_home -v 1.7`/bin/jar"
export JAVA_HOME=`/usr/libexec/java_home -v 1.7`
```
@ -222,18 +236,18 @@ It will start out by asking you the version number:
```
...
What is the release version for "ANTLR 4"? (org.antlr:antlr4-master) 4.8: : 4.8
What is the release version for "ANTLR 4 Runtime"? (org.antlr:antlr4-runtime) 4.8: :
What is the release version for "ANTLR 4 Tool"? (org.antlr:antlr4) 4.8: :
What is the release version for "ANTLR 4 Maven plugin"? (org.antlr:antlr4-maven-plugin) 4.8: :
What is the release version for "ANTLR 4 Runtime Test Generator"? (org.antlr:antlr4-runtime-testsuite) 4.8: :
What is the release version for "ANTLR 4 Tool Tests"? (org.antlr:antlr4-tool-testsuite) 4.8: :
What is SCM release tag or label for "ANTLR 4"? (org.antlr:antlr4-master) antlr4-master-4.8: : 4.8
What is the new development version for "ANTLR 4"? (org.antlr:antlr4-master) 4.8.1-SNAPSHOT:
What is the release version for "ANTLR 4"? (org.antlr:antlr4-master) 4.9: : 4.9
What is the release version for "ANTLR 4 Runtime"? (org.antlr:antlr4-runtime) 4.9: :
What is the release version for "ANTLR 4 Tool"? (org.antlr:antlr4) 4.9: :
What is the release version for "ANTLR 4 Maven plugin"? (org.antlr:antlr4-maven-plugin) 4.9: :
What is the release version for "ANTLR 4 Runtime Test Generator"? (org.antlr:antlr4-runtime-testsuite) 4.9: :
What is the release version for "ANTLR 4 Tool Tests"? (org.antlr:antlr4-tool-testsuite) 4.9: :
What is SCM release tag or label for "ANTLR 4"? (org.antlr:antlr4-master) antlr4-master-4.9: : 4.9
What is the new development version for "ANTLR 4"? (org.antlr:antlr4-master) 4.9.1-SNAPSHOT:
...
```
Maven will go through your pom.xml files to update versions from 4.8-SNAPSHOT to 4.8 for release and then to 4.8.1-SNAPSHOT after release, which is done with:
Maven will go through your pom.xml files to update versions from 4.9-SNAPSHOT to 4.9 for release and then to 4.9.1-SNAPSHOT after release, which is done with:
```bash
mvn release:perform -Darguments="-DskipTests"
@ -247,18 +261,18 @@ Now, go here:
and on the left click "Staging Repositories". You click the staging repo and close it, then you refresh, click it and release it. It's done when you see it here:
&nbsp;&nbsp;&nbsp;&nbsp;[https://oss.sonatype.org/service/local/repositories/releases/content/org/antlr/antlr4-runtime/4.8-1/antlr4-runtime-4.8-1.jar](https://oss.sonatype.org/service/local/repositories/releases/content/org/antlr/antlr4-runtime/4.8-1/antlr4-runtime-4.8-1.jar)
&nbsp;&nbsp;&nbsp;&nbsp;[https://oss.sonatype.org/service/local/repositories/releases/content/org/antlr/antlr4-runtime/4.9-1/antlr4-runtime-4.9-1.jar](https://oss.sonatype.org/service/local/repositories/releases/content/org/antlr/antlr4-runtime/4.9-1/antlr4-runtime-4.9-1.jar)
All releases should be here: https://repo1.maven.org/maven2/org/antlr/antlr4-runtime/
Copy the jars to antlr.org site and update download/index.html
```bash
cp ~/.m2/repository/org/antlr/antlr4-runtime/4.8/antlr4-runtime-4.8.jar ~/antlr/sites/website-antlr4/download/antlr-runtime-4.8.jar
cp ~/.m2/repository/org/antlr/antlr4/4.8/antlr4-4.8-complete.jar ~/antlr/sites/website-antlr4/download/antlr-4.8-complete.jar
cp ~/.m2/repository/org/antlr/antlr4-runtime/4.9/antlr4-runtime-4.9.jar ~/antlr/sites/website-antlr4/download/antlr-runtime-4.9.jar
cp ~/.m2/repository/org/antlr/antlr4/4.9/antlr4-4.9-complete.jar ~/antlr/sites/website-antlr4/download/antlr-4.9-complete.jar
cd ~/antlr/sites/website-antlr4/download
git add antlr-4.8-complete.jar
git add antlr-runtime-4.8.jar
git add antlr-4.9-complete.jar
git add antlr-runtime-4.9.jar
```
Update on site:
@ -270,7 +284,7 @@ Update on site:
* scripts/topnav.js
```
git commit -a -m 'add 4.8 jars'
git commit -a -m 'add 4.9 jars'
git push origin gh-pages
```
@ -287,15 +301,18 @@ cd runtime/JavaScript
```bash
cd runtime/JavaScript
npm update
npm install
npm run build
npm login
npm publish antlr4
npm publish # don't put antlr4 on there or it will try to push the old version for some reason
```
Move target to website
Move (and zip) target to website:
```bash
npm run build
cp /dist/antlr4.js ~/antlr/sites/website-antlr4/download
cd src
zip -r ~/antlr/sites/website-antlr4/download/antlr-javascript-runtime-4.9.zip .
```
### CSharp
@ -308,10 +325,17 @@ Now we have [appveyor create artifact](https://ci.appveyor.com/project/parrt/ant
Of course you need Mono and `nuget` to be installed. On mac:
- .NET build tools - can be loaded from [here](https://www.visualstudio.com/downloads/)
- .NET build tools - can be loaded from [here](https://www.visualstudio.com/downloads/) (I need dotnet 5 and 3.1 versions)
- nuget - download [nuget.exe](https://www.nuget.org/downloads)
- dotnet - follow [the instructions here](https://www.microsoft.com/net/core)
From @kvanTTT: Install `dotnet` on any platform (see https://dotnet.microsoft.com/download) and run the following command on any OS (Win, Linux, macOS):
* building: `dotnet build runtime/CSharp/src/Antlr4.csproj -c Release`
Output `.dll` will be in `runtime/CSharp/src/bin/Release/netstandard2.0` or in `runtime/CSharp/src/bin/Release/netstandard2.1`
* packing: `dotnet pack runtime/CSharp/src/Antlr4.csproj -c Release`
Output `.nupkg` will be in `runtime/CSharp/src/bin/Release/Antlr4.Runtime.Standard.4.9.1.nupkg`
Alternatively, you can install Visual Studio 2017 and make sure to check boxes with .NET Core SDK.
You also need to enable .NET Framework 3.5 support in Windows "Programs and Features".
@ -336,7 +360,7 @@ Copyright (C) Microsoft Corporation. All rights reserved.
Restore completed in 427.62 ms for C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\Antlr4.Runtime.dotnet.csproj.
Antlr4.Runtime.dotnet -> C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\netstandard1.3\Antlr4.Runtime.Standard.dll
Antlr4.Runtime.dotnet -> C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\net35\Antlr4.Runtime.Standard.dll
Successfully created package 'C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\Antlr4.Runtime.Standard.4.8.2.nupkg'.
Successfully created package 'C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\Antlr4.Runtime.Standard.4.9.2.nupkg'.
```
**Publishing to NuGet**
@ -412,7 +436,7 @@ On a Mac (with XCode 7+ installed):
```bash
cd runtime/Cpp
./deploy-macos.sh
cp antlr4-cpp-runtime-macos.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.8-macos.zip
cp antlr4-cpp-runtime-macos.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.9-macos.zip
```
On any Mac or Linux machine:
@ -420,7 +444,7 @@ On any Mac or Linux machine:
```bash
cd runtime/Cpp
./deploy-source.sh
cp antlr4-cpp-runtime-source.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.8-source.zip
cp antlr4-cpp-runtime-source.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.9-source.zip
```
On a Windows machine the build scripts checks if VS 2017 and/or VS 2019 are installed and builds binaries for each, if found. This script requires 7z to be installed (http://7-zip.org then do `set PATH=%PATH%;C:\Program Files\7-Zip\` from DOS not powershell).
@ -428,7 +452,7 @@ On a Windows machine the build scripts checks if VS 2017 and/or VS 2019 are inst
```bash
cd runtime/Cpp
deploy-windows.cmd Community
cp antlr4-cpp-runtime-vs2019.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.8-vs2019.zip
cp antlr4-cpp-runtime-vs2019.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.9-vs2019.zip
```
Move target to website (**_rename to a specific ANTLR version first if needed_**):
@ -436,9 +460,9 @@ Move target to website (**_rename to a specific ANTLR version first if needed_**
```bash
pushd ~/antlr/sites/website-antlr4/download
# vi index.html
git add antlr4cpp-runtime-4.8-macos.zip
git add antlr4cpp-runtime-4.8-windows.zip
git add antlr4cpp-runtime-4.8-source.zip
git add antlr4-cpp-runtime-4.9-macos.zip
git add antlr4-cpp-runtime-4.9-windows.zip
git add antlr4-cpp-runtime-4.9-source.zip
git commit -a -m 'update C++ runtime'
git push origin gh-pages
popd
@ -446,6 +470,8 @@ popd
### Dart
Install Dart SDK from https://dart.dev/get-dart
Push to pub.dev
```bash
@ -459,7 +485,13 @@ Otherwise enter `N` to ignore the warning.
## Update javadoc for runtime and tool
First, gen javadoc:
Above build should make latest in
```
~/.m2/repository/org/antlr/antlr4-runtime/4.9/antlr4-runtime-4.9
```
but you can regen (watch pom version!):
```bash
$ cd antlr4
@ -473,9 +505,9 @@ cd ~/antlr/sites/website-antlr4/api
git checkout gh-pages
git pull origin gh-pages
cd Java
jar xvf ~/.m2/repository/org/antlr/antlr4-runtime/4.8/antlr4-runtime-4.8-javadoc.jar
jar xvf ~/.m2/repository/org/antlr/antlr4-runtime/4.9/antlr4-runtime-4.9-javadoc.jar
cd ../JavaTool
jar xvf ~/.m2/repository/org/antlr/antlr4/4.8/antlr4-4.8-javadoc.jar
jar xvf ~/.m2/repository/org/antlr/antlr4/4.9/antlr4-4.9-javadoc.jar
git commit -a -m 'freshen api doc'
git push origin gh-pages
```

9
doc/tree-matching.md
View File

@ -52,15 +52,6 @@ m.setDelimiters("<<", ">>", "$"); // $ is the escape character
This would allow pattern `<<ID>> = <<expr>> ;$<< ick $>>` to be interpreted as elements: `ID`, ` = `, `expr`, and ` ;<< ick >>`.
```java
String xpath = "//blockStatement/*";
String treePattern = "int <Identifier> = <expression>;";
ParseTreePattern p =
parser.compileParseTreePattern(treePattern,
JavaParser.RULE_localVariableDeclarationStatement);
List<ParseTreeMatch> matches = p.findAll(tree, xpath);
```
### Pattern labels
The tree pattern matcher tracks the nodes in the tree at matches against the tags in a tree pattern. That way we can use the `get()` and `getAll()` methods to retrieve components of the matched subtree. For example, for pattern `<ID>`, `get("ID")` returns the node matched for that `ID`. If more than one node matched the specified token or rule tag, only the first match is returned. If there is no node associated with the label, this returns null.

4
pom.xml
View File

@ -13,7 +13,7 @@
</parent>
<groupId>org.antlr</groupId>
<artifactId>antlr4-master</artifactId>
<version>4.8-2-SNAPSHOT</version>
<version>4.9.3-SNAPSHOT</version>
<packaging>pom</packaging>
<name>ANTLR 4</name>
@ -151,7 +151,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.0</version>
<version>3.8.1</version>
<configuration>
<source>${maven.compiler.source}</source>
<target>${maven.compiler.target}</target>

2
runtime-testsuite/annotations/pom.xml
View File

@ -9,7 +9,7 @@
<parent>
<groupId>org.antlr</groupId>
<artifactId>antlr4-master</artifactId>
<version>4.8-2-SNAPSHOT</version>
<version>4.9.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>antlr4-runtime-test-annotations</artifactId>

4
runtime-testsuite/pom.xml
View File

@ -10,7 +10,7 @@
<parent>
<groupId>org.antlr</groupId>
<artifactId>antlr4-master</artifactId>
<version>4.8-2-SNAPSHOT</version>
<version>4.9.3-SNAPSHOT</version>
</parent>
<artifactId>antlr4-runtime-testsuite</artifactId>
<name>ANTLR 4 Runtime Tests (2nd generation)</name>
@ -56,7 +56,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<version>4.13.1</version>
<scope>test</scope>
</dependency>
<dependency>

2
runtime-testsuite/processors/pom.xml
View File

@ -9,7 +9,7 @@
<parent>
<groupId>org.antlr</groupId>
<artifactId>antlr4-master</artifactId>
<version>4.8-2-SNAPSHOT</version>
<version>4.9.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>antlr4-runtime-test-annotation-processors</artifactId>

26
runtime-testsuite/processors/src/org/antlr/v4/test/runtime/CommentHasStringValueProcessor.java
View File

@ -6,10 +6,11 @@
package org.antlr.v4.test.runtime;
import com.sun.tools.javac.main.JavaCompiler;
import com.sun.tools.javac.model.JavacElements;
import com.sun.tools.javac.processing.JavacProcessingEnvironment;
import com.sun.tools.javac.tree.JCTree;
import com.sun.tools.javac.tree.TreeMaker;
import com.sun.tools.javac.util.Context;
import com.sun.tools.javac.util.List;
import javax.annotation.processing.AbstractProcessor;
@ -20,6 +21,7 @@ import javax.annotation.processing.SupportedSourceVersion;
import javax.lang.model.SourceVersion;
import javax.lang.model.element.Element;
import javax.lang.model.element.TypeElement;
import java.lang.reflect.Field;
import java.util.Set;
/**
@ -38,6 +40,7 @@ import java.util.Set;
@SupportedAnnotationTypes({"org.antlr.v4.test.runtime.CommentHasStringValue"})
@SupportedSourceVersion(SourceVersion.RELEASE_7)
public class CommentHasStringValueProcessor extends AbstractProcessor {
protected JavacElements utilities;
protected TreeMaker treeMaker;
@ -46,9 +49,21 @@ public class CommentHasStringValueProcessor extends AbstractProcessor {
super.init(processingEnv);
// Messager messager = processingEnv.getMessager();
// messager.printMessage(Diagnostic.Kind.NOTE, "WOW INIT--------------------");
JavacProcessingEnvironment javacProcessingEnv = (JavacProcessingEnvironment) processingEnv;
utilities = javacProcessingEnv.getElementUtils();
treeMaker = TreeMaker.instance(javacProcessingEnv.getContext());
utilities = (JavacElements)processingEnv.getElementUtils();
treeMaker = TreeMaker.instance(extractContext(utilities));
}
private static Context extractContext(JavacElements utilities) {
try {
Field compilerField = JavacElements.class.getDeclaredField("javaCompiler");
compilerField.setAccessible(true);
JavaCompiler compiler = (JavaCompiler)compilerField.get(utilities);
Field contextField = JavaCompiler.class.getDeclaredField("context");
contextField.setAccessible(true);
return (Context)contextField.get(compiler);
} catch (NoSuchFieldException | IllegalAccessException e) {
throw new IllegalStateException(e);
}
}
@Override
@ -66,8 +81,7 @@ public class CommentHasStringValueProcessor extends AbstractProcessor {
else if ( elementTree instanceof JCTree.JCMethodDecl ) {
JCTree.JCStatement[] statements = new JCTree.JCStatement[1];
statements[0] = treeMaker.Return(literal);
JCTree.JCBlock body = treeMaker.Block(0, List.from(statements));
((JCTree.JCMethodDecl)elementTree).body = body;
((JCTree.JCMethodDecl)elementTree).body = treeMaker.Block(0, List.from(statements));
}
}
return true;

2
runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Chrome.test.stg
View File

@ -74,7 +74,7 @@ LANotEquals(i, v) ::= <%this._input.LA(<i>)!=<v>%>
TokenStartColumnEquals(i) ::= <%this._tokenStartColumn===<i>%>
ImportListener(X) ::= <<var <X>Listener = require('./<X>Listener').<X>Listener;>>
ImportListener(X) ::= ""
GetExpectedTokenNames() ::= "this.getExpectedTokens().toString(this.literalNames)"

6
runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Firefox.test.stg
View File

@ -76,11 +76,7 @@ LANotEquals(i, v) ::= <%this._input.LA(<i>)!=<v>%>
TokenStartColumnEquals(i) ::= <%this._tokenStartColumn===<i>%>
ImportListener(X) ::= <<
@parser::header {
var <X>Listener = require('./<X>Listener').<X>Listener;
}
>>
ImportListener(X) ::= ""
GetExpectedTokenNames() ::= "this.getExpectedTokens().toString(this.literalNames)"

6
runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Node.test.stg
View File

@ -74,11 +74,7 @@ LANotEquals(i, v) ::= <%this._input.LA(<i>)!=<v>%>
TokenStartColumnEquals(i) ::= <%this._tokenStartColumn===<i>%>
ImportListener(X) ::= <<
@parser::header {
var <X>Listener = require('./<X>Listener').<X>Listener;
}
>>
ImportListener(X) ::= ""
GetExpectedTokenNames() ::= "this.getExpectedTokens().toString(this.literalNames)"

6
runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Safari.test.stg
View File

@ -74,11 +74,7 @@ LANotEquals(i, v) ::= <%this._input.LA(<i>)!=<v>%>
TokenStartColumnEquals(i) ::= <%this._tokenStartColumn===<i>%>
ImportListener(X) ::= <<
@parser::header {
var <X>Listener = require('./<X>Listener').<X>Listener;
}
>>
ImportListener(X) ::= ""
GetExpectedTokenNames() ::= "this.getExpectedTokens().toString(this.literalNames)"

99
runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java
View File

@ -11,9 +11,7 @@ import org.antlr.v4.runtime.misc.Pair;
import org.antlr.v4.runtime.misc.Utils;
import org.antlr.v4.tool.ANTLRMessage;
import org.antlr.v4.tool.DefaultToolListener;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.*;
import org.junit.rules.TestRule;
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;
@ -44,31 +42,58 @@ import static org.junit.Assume.assumeFalse;
* @since 4.6.
*/
public abstract class BaseRuntimeTest {
public final static String[] Targets = {
"Cpp",
"Java",
"Go",
"CSharp",
"Python2", "Python3",
"PHP",
"Dart",
"Go",
"Java",
"Node",
"Dart"
"PHP",
"Python2", "Python3",
"Swift"
};
static {
// Add heartbeat thread to gen minimal output for travis, appveyor to
// avoid timeout.
@BeforeClass
public static void startHeartbeatToAvoidTimeout() {
if (isTravisCI() || isAppVeyorCI())
startHeartbeat();
}
@AfterClass
public static void stopHeartbeat() {
heartbeat = false;
}
private static boolean isAppVeyorCI() {
// see https://www.appveyor.com/docs/environment-variables/
String s = System.getenv("APPVEYOR");
return s!=null && "true".equals(s.toLowerCase());
}
private static boolean isTravisCI() {
// see https://docs.travis-ci.com/user/environment-variables/#default-environment-variables
String s = System.getenv("TRAVIS");
return s!=null && "true".equals(s.toLowerCase());
}
static boolean heartbeat = false;
private static void startHeartbeat() {
// Add heartbeat thread to gen minimal output for travis, appveyor to avoid timeout.
Thread t = new Thread("heartbeat") {
@Override
public void run() {
while (true) {
System.out.print('.');
heartbeat = true;
while (heartbeat) {
try {
Thread.sleep(5000);
}
catch (Exception e) {
//noinspection BusyWait
Thread.sleep(10000);
} catch (Exception e) {
e.printStackTrace();
}
System.out.print('.');
}
}
};
@ -86,19 +111,21 @@ public abstract class BaseRuntimeTest {
this.delegate = delegate;
}
public static void mkdir(String dir) {
File f = new File(dir);
f.mkdirs();
}
@Before
public void setUp() throws Exception {
// From http://junit.sourceforge.net/javadoc/org/junit/Assume.html
// "The default JUnit runner treats tests with failing assumptions as ignored"
assumeFalse(descriptor.ignore(descriptor.getTarget()));
assumeFalse(checkIgnored());
delegate.testSetUp();
}
public boolean checkIgnored() {
boolean ignored = !TestContext.isSupportedTarget(descriptor.getTarget()) || descriptor.ignore(descriptor.getTarget());
if(ignored)
System.out.println("Ignore " + descriptor);
return ignored;
}
@Rule
public final TestRule testWatcher = new TestWatcher() {
@Override
@ -110,22 +137,24 @@ public abstract class BaseRuntimeTest {
@Test
public void testOne() throws Exception {
// System.out.println(descriptor.getTestName());
// System.out.println(delegate.getTmpDir());
if ( descriptor.ignore(descriptor.getTarget()) ) {
System.out.printf("Ignore "+descriptor);
if (descriptor.ignore(descriptor.getTarget()) ) {
System.out.println("Ignore " + descriptor);
return;
}
if ( descriptor.getTestType().contains("Parser") ) {
delegate.beforeTest(descriptor);
if (descriptor.getTestType().contains("Parser") ) {
testParser(descriptor);
}
else {
testLexer(descriptor);
}
delegate.afterTest(descriptor);
}
public void testParser(RuntimeTestDescriptor descriptor) throws Exception {
mkdir(delegate.getTmpDir());
RuntimeTestUtils.mkdir(delegate.getTempParserDirPath());
Pair<String, String> pair = descriptor.getGrammar();
@ -142,7 +171,7 @@ public abstract class BaseRuntimeTest {
g.registerRenderer(String.class, new StringRenderer());
g.importTemplates(targetTemplates);
ST grammarST = new ST(g, spair.b);
writeFile(delegate.getTmpDir(), spair.a+".g4", grammarST.render());
writeFile(delegate.getTempParserDirPath(), spair.a+".g4", grammarST.render());
}
}
@ -167,7 +196,7 @@ public abstract class BaseRuntimeTest {
}
public void testLexer(RuntimeTestDescriptor descriptor) throws Exception {
mkdir(delegate.getTmpDir());
RuntimeTestUtils.mkdir(delegate.getTempParserDirPath());
Pair<String, String> pair = descriptor.getGrammar();
@ -184,7 +213,7 @@ public abstract class BaseRuntimeTest {
g.registerRenderer(String.class, new StringRenderer());
g.importTemplates(targetTemplates);
ST grammarST = new ST(g, spair.b);
writeFile(delegate.getTmpDir(), spair.a+".g4", grammarST.render());
writeFile(delegate.getTempParserDirPath(), spair.a+".g4", grammarST.render());
}
}
@ -208,7 +237,7 @@ public abstract class BaseRuntimeTest {
boolean defaultListener,
String... extraOptions)
{
mkdir(workdir);
RuntimeTestUtils.mkdir(workdir);
writeFile(workdir, grammarFileName, grammarStr);
return antlrOnString(workdir, targetName, grammarFileName, defaultListener, extraOptions);
}
@ -273,6 +302,8 @@ public abstract class BaseRuntimeTest {
// ---- support ----
public static RuntimeTestDescriptor[] getRuntimeTestDescriptors(Class<?> clazz, String targetName) {
if(!TestContext.isSupportedTarget(targetName))
return new RuntimeTestDescriptor[0];
Class<?>[] nestedClasses = clazz.getClasses();
List<RuntimeTestDescriptor> descriptors = new ArrayList<RuntimeTestDescriptor>();
for (Class<?> nestedClass : nestedClasses) {
@ -280,8 +311,10 @@ public abstract class BaseRuntimeTest {
if ( RuntimeTestDescriptor.class.isAssignableFrom(nestedClass) && !Modifier.isAbstract(modifiers) ) {
try {
RuntimeTestDescriptor d = (RuntimeTestDescriptor) nestedClass.newInstance();
d.setTarget(targetName);
descriptors.add(d);
if(!d.ignore(targetName)) {
d.setTarget(targetName);
descriptors.add(d);
}
} catch (Exception e) {
e.printStackTrace(System.err);
}

238
runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTestSupport.java Normal file
View File

@ -0,0 +1,238 @@
package org.antlr.v4.test.runtime;
import org.antlr.v4.Tool;
import org.antlr.v4.automata.LexerATNFactory;
import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNDeserializer;
import org.antlr.v4.runtime.atn.ATNSerializer;
import org.antlr.v4.semantics.SemanticPipeline;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar;
import org.junit.rules.TestRule;
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.util.Locale;
import java.util.logging.Logger;
import static org.junit.Assert.assertEquals;
@SuppressWarnings("ResultOfMethodCallIgnored")
public abstract class BaseRuntimeTestSupport implements RuntimeTestSupport {
// -J-Dorg.antlr.v4.test.BaseTest.level=FINE
protected static final Logger logger = Logger.getLogger(BaseRuntimeTestSupport.class.getName());
public static final String NEW_LINE = System.getProperty("line.separator");
public static final String PATH_SEP = System.getProperty("path.separator");
private File tempTestDir = null;
/** If error during parser execution, store stderr here; can't return
* stdout and stderr. This doesn't trap errors from running antlr.
*/
private String parseErrors;
/** Errors found while running antlr */
private StringBuilder antlrToolErrors;
@org.junit.Rule
public final TestRule testWatcher = new TestWatcher() {
@Override
protected void succeeded(Description description) {
testSucceeded(description);
}
};
protected void testSucceeded(Description description) {
// remove tmpdir if no error.
eraseTempDir();
}
@Override
public File getTempParserDir() {
return getTempTestDir();
}
@Override
public String getTempParserDirPath() {
return getTempParserDir() == null ? null : getTempParserDir().getAbsolutePath();
}
@Override
public final File getTempTestDir() {
return tempTestDir;
}
@Override
public final String getTempDirPath() {
return tempTestDir ==null ? null : tempTestDir.getAbsolutePath();
}
public void setParseErrors(String errors) {
this.parseErrors = errors;
}
public String getParseErrors() {
return parseErrors;
}
public String getANTLRToolErrors() {
if ( antlrToolErrors.length()==0 ) {
return null;
}
return antlrToolErrors.toString();
}
protected abstract String getPropertyPrefix();
@Override
public void testSetUp() throws Exception {
createTempDir();
antlrToolErrors = new StringBuilder();
}
private void createTempDir() {
// new output dir for each test
String propName = getPropertyPrefix() + "-test-dir";
String prop = System.getProperty(propName);
if(prop!=null && prop.length()>0) {
tempTestDir = new File(prop);
} else {
String dirName = getClass().getSimpleName() + "-" + Thread.currentThread().getName() + "-" + System.currentTimeMillis();
tempTestDir = new File(System.getProperty("java.io.tmpdir"), dirName);
}
}
@Override
public void testTearDown() throws Exception {
}
@Override
public void beforeTest(RuntimeTestDescriptor descriptor) {
}
@Override
public void afterTest(RuntimeTestDescriptor descriptor) {
}
public void eraseTempDir() {
if(shouldEraseTempDir()) {
eraseDirectory(getTempTestDir());
}
}
protected boolean shouldEraseTempDir() {
if(tempTestDir == null)
return false;
String propName = getPropertyPrefix() + "-erase-test-dir";
String prop = System.getProperty(propName);
if (prop != null && prop.length() > 0)
return Boolean.getBoolean(prop);
else
return true;
}
public static void eraseDirectory(File dir) {
if ( dir.exists() ) {
eraseFilesInDir(dir);
dir.delete();
}
}
public static void eraseFilesInDir(File dir) {
String[] files = dir.list();
for(int i = 0; files!=null && i < files.length; i++) {
try {
eraseFile(dir, files[i]);
} catch(IOException e) {
logger.info(e.getMessage());
}
}
}
private static void eraseFile(File dir, String name) throws IOException {
File file = new File(dir,name);
if(Files.isSymbolicLink((file.toPath())))
Files.delete(file.toPath());
else if(file.isDirectory()) {
// work around issue where Files.isSymbolicLink returns false on Windows for node/antlr4 linked package
if("antlr4".equals(name))
; // logger.warning("antlr4 not seen as a symlink");
else
eraseDirectory(file);
} else
file.delete();
}
private static String detectedOS;
public static String getOS() {
if (detectedOS == null) {
String os = System.getProperty("os.name", "generic").toLowerCase(Locale.ENGLISH);
if (os.contains("mac") || os.contains("darwin")) {
detectedOS = "mac";
}
else if (os.contains("win")) {
detectedOS = "windows";
}
else if (os.contains("nux")) {
detectedOS = "linux";
}
else {
detectedOS = "unknown";
}
}
return detectedOS;
}
public static boolean isWindows() {
return getOS().equalsIgnoreCase("windows");
}
protected ATN createATN(Grammar g, boolean useSerializer) {
if ( g.atn==null ) {
semanticProcess(g);
assertEquals(0, g.tool.getNumErrors());
ParserATNFactory f = g.isLexer() ? new LexerATNFactory((LexerGrammar) g) : new ParserATNFactory(g);
g.atn = f.createATN();
assertEquals(0, g.tool.getNumErrors());
}
ATN atn = g.atn;
if ( useSerializer ) {
char[] serialized = ATNSerializer.getSerializedAsChars(atn);
return new ATNDeserializer().deserialize(serialized);
}
return atn;
}
protected void semanticProcess(Grammar g) {
if ( g.ast!=null && !g.ast.hasErrors ) {
// System.out.println(g.ast.toStringTree());
Tool antlr = new Tool();
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any)
for (Grammar imp : g.getImportedGrammars()) {
antlr.processNonCombinedGrammar(imp, false);
}
}
}
}
}

91
runtime-testsuite/test/org/antlr/v4/test/runtime/MockIntTokenStream.java Normal file
View File

@ -0,0 +1,91 @@
package org.antlr.v4.test.runtime;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.runtime.misc.Interval;
public class MockIntTokenStream implements TokenStream {
public IntegerList types;
int p=0;
public MockIntTokenStream(IntegerList types) { this.types = types; }
@Override
public void consume() { p++; }
@Override
public int LA(int i) { return LT(i).getType(); }
@Override
public int mark() {
return index();
}
@Override
public int index() { return p; }
@Override
public void release(int marker) {
seek(marker);
}
@Override
public void seek(int index) {
p = index;
}
@Override
public int size() {
return types.size();
}
@Override
public String getSourceName() {
return UNKNOWN_SOURCE_NAME;
}
@Override
public Token LT(int i) {
CommonToken t;
int rawIndex = p + i - 1;
if ( rawIndex>=types.size() ) t = new CommonToken(Token.EOF);
else t = new CommonToken(types.get(rawIndex));
t.setTokenIndex(rawIndex);
return t;
}
@Override
public Token get(int i) {
return new org.antlr.v4.runtime.CommonToken(types.get(i));
}
@Override
public TokenSource getTokenSource() {
return null;
}
@Override
public String getText() {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Interval interval) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(RuleContext ctx) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Token start, Token stop) {
throw new UnsupportedOperationException("can't give strings");
}
}

20
runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestSupport.java
View File

@ -6,6 +6,8 @@
package org.antlr.v4.test.runtime;
import java.io.File;
/** This interface describes functionality needed to execute a runtime test.
* Unfortunately the Base*Test.java files are big junk drawers. This is
* an attempt to make it more obvious what new target implementers have to
@ -14,13 +16,22 @@ package org.antlr.v4.test.runtime;
* @since 4.6
*/
public interface RuntimeTestSupport {
void testSetUp() throws Exception;
void testTearDown() throws Exception;
// dir containing grammar input and output
File getTempParserDir();
String getTempParserDirPath();
// dir containing test input and output
File getTempTestDir();
String getTempDirPath();
void eraseTempDir();
String getTmpDir();
void testSetUp() throws Exception;
void testTearDown() throws Exception;
void beforeTest(RuntimeTestDescriptor descriptor);
void afterTest(RuntimeTestDescriptor descriptor);
String getStdout();
String getParseErrors();
String getANTLRToolErrors();
@ -39,4 +50,5 @@ public interface RuntimeTestSupport {
String startRuleName,
String input,
boolean showDiagnosticErrors);
}

89
runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestUtils.java Normal file
View File

@ -0,0 +1,89 @@
package org.antlr.v4.test.runtime;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.tool.LexerGrammar;
import java.io.*;
import java.util.*;
public abstract class RuntimeTestUtils {
/** Sort a list */
public static <T extends Comparable<? super T>> List<T> sort(List<T> data) {
List<T> dup = new ArrayList<T>(data);
dup.addAll(data);
Collections.sort(dup);
return dup;
}
/** Return map sorted by key */
public static <K extends Comparable<? super K>,V> LinkedHashMap<K,V> sort(Map<K,V> data) {
LinkedHashMap<K,V> dup = new LinkedHashMap<K, V>();
List<K> keys = new ArrayList<K>(data.keySet());
Collections.sort(keys);
for (K k : keys) {
dup.put(k, data.get(k));
}
return dup;
}
public static List<String> getTokenTypes(LexerGrammar lg,
ATN atn,
CharStream input) {
LexerATNSimulator interp = new LexerATNSimulator(atn, new DFA[]{new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE))}, null);
List<String> tokenTypes = new ArrayList<String>();
int ttype;
boolean hitEOF = false;
do {
if ( hitEOF ) {
tokenTypes.add("EOF");
break;
}
int t = input.LA(1);
ttype = interp.match(input, Lexer.DEFAULT_MODE);
if ( ttype==Token.EOF ) {
tokenTypes.add("EOF");
}
else {
tokenTypes.add(lg.typeToTokenList.get(ttype));
}
if ( t== IntStream.EOF ) {
hitEOF = true;
}
} while ( ttype!=Token.EOF );
return tokenTypes;
}
public static IntegerList getTokenTypesViaATN(String input, LexerATNSimulator lexerATN) {
ANTLRInputStream in = new ANTLRInputStream(input);
IntegerList tokenTypes = new IntegerList();
int ttype;
do {
ttype = lexerATN.match(in, Lexer.DEFAULT_MODE);
tokenTypes.add(ttype);
} while ( ttype!= Token.EOF );
return tokenTypes;
}
public static void copyFile(File source, File dest) throws IOException {
InputStream is = new FileInputStream(source);
OutputStream os = new FileOutputStream(dest);
byte[] buf = new byte[4 << 10];
int l;
while ((l = is.read(buf)) > -1) {
os.write(buf, 0, l);
}
is.close();
os.close();
}
public static void mkdir(String dir) {
File f = new File(dir);
f.mkdirs();
}
}

33
runtime-testsuite/test/org/antlr/v4/test/runtime/TestContext.java Normal file
View File

@ -0,0 +1,33 @@
package org.antlr.v4.test.runtime;
public abstract class TestContext {
public static boolean isTravisCI() {
return "true".equals(String.valueOf(System.getenv("TRAVIS")).toLowerCase());
}
public static boolean isGitHubCI() {
return "true".equals(String.valueOf(System.getenv("CI")).toLowerCase());
}
public static boolean isAppVeyorCI() {
return "true".equals(String.valueOf(System.getenv("APPVEYOR")).toLowerCase());
}
public static boolean isCircleCI() {
return "true".equals(String.valueOf(System.getenv("CIRCLECI")).toLowerCase());
}
public static boolean isCI() {
return isAppVeyorCI() || isCircleCI() || isGitHubCI() || isTravisCI();
}
@SuppressWarnings("BooleanMethodIsAlwaysInverted")
public static boolean isSupportedTarget(String target) {
if(isAppVeyorCI())
return !target.matches("Swift");
else
return true;
}
}

733
runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java
View File

@ -5,64 +5,19 @@
*/
package org.antlr.v4.test.runtime.cpp;
import org.antlr.v4.Tool;
import org.antlr.v4.automata.ATNFactory;
import org.antlr.v4.automata.ATNPrinter;
import org.antlr.v4.automata.LexerATNFactory;
import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.IntStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenSource;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.WritableToken;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNDeserializer;
import org.antlr.v4.runtime.atn.ATNSerializer;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.semantics.SemanticPipeline;
import org.antlr.v4.test.runtime.ErrorQueue;
import org.antlr.v4.test.runtime.RuntimeTestSupport;
import org.antlr.v4.test.runtime.StreamVacuum;
import org.antlr.v4.tool.ANTLRMessage;
import org.antlr.v4.tool.DOTGenerator;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.GrammarSemanticsMessage;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.test.runtime.*;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupString;
import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
@ -72,248 +27,12 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
public class BaseCppTest implements RuntimeTestSupport {
// -J-Dorg.antlr.v4.test.BaseTest.level=FINE
// private static final Logger LOGGER = Logger.getLogger(BaseTest.class.getName());
public static final String newline = System.getProperty("line.separator");
public static final String pathSep = System.getProperty("path.separator");
public class BaseCppTest extends BaseRuntimeTestSupport implements RuntimeTestSupport {
public String tmpdir = null;
/** If error during parser execution, store stderr here; can't return
* stdout and stderr. This doesn't trap errors from running antlr.
*/
protected String stderrDuringParse;
/** Errors found while running antlr */
protected StringBuilder antlrToolErrors;
private String getPropertyPrefix() {
protected String getPropertyPrefix() {
return "antlr-" + getLanguage().toLowerCase();
}
@Override
public void testSetUp() throws Exception {
// new output dir for each test
String propName = getPropertyPrefix() + "-test-dir";
String prop = System.getProperty(propName);
if(prop!=null && prop.length()>0) {
tmpdir = prop;
}
else {
tmpdir = new File(System.getProperty("java.io.tmpdir"),
getClass().getSimpleName()+"-"+Thread.currentThread().getName()+"-"+System.currentTimeMillis()).getAbsolutePath();
}
antlrToolErrors = new StringBuilder();
}
@Override
public void testTearDown() throws Exception {
}
@Override
public String getTmpDir() {
return tmpdir;
}
@Override
public String getStdout() {
return null;
}
@Override
public String getParseErrors() {
return stderrDuringParse;
}
@Override
public String getANTLRToolErrors() {
if ( antlrToolErrors.length()==0 ) {
return null;
}
return antlrToolErrors.toString();
}
protected org.antlr.v4.Tool newTool(String[] args) {
Tool tool = new Tool(args);
return tool;
}
protected Tool newTool() {
org.antlr.v4.Tool tool = new Tool(new String[] {"-o", tmpdir});
return tool;
}
protected ATN createATN(Grammar g, boolean useSerializer) {
if ( g.atn==null ) {
semanticProcess(g);
assertEquals(0, g.tool.getNumErrors());
ParserATNFactory f;
if ( g.isLexer() ) {
f = new LexerATNFactory((LexerGrammar)g);
}
else {
f = new ParserATNFactory(g);
}
g.atn = f.createATN();
assertEquals(0, g.tool.getNumErrors());
}
ATN atn = g.atn;
if (useSerializer) {
char[] serialized = ATNSerializer.getSerializedAsChars(atn);
return new ATNDeserializer().deserialize(serialized);
}
return atn;
}
protected void semanticProcess(Grammar g) {
if ( g.ast!=null && !g.ast.hasErrors ) {
System.out.println(g.ast.toStringTree());
Tool antlr = new Tool();
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any)
for (Grammar imp : g.getImportedGrammars()) {
antlr.processNonCombinedGrammar(imp, false);
}
}
}
}
public DFA createDFA(Grammar g, DecisionState s) {
// PredictionDFAFactory conv = new PredictionDFAFactory(g, s);
// DFA dfa = conv.createDFA();
// conv.issueAmbiguityWarnings();
// System.out.print("DFA="+dfa);
// return dfa;
return null;
}
// public void minimizeDFA(DFA dfa) {
// DFAMinimizer dmin = new DFAMinimizer(dfa);
// dfa.minimized = dmin.minimize();
// }
IntegerList getTypesFromString(Grammar g, String expecting) {
IntegerList expectingTokenTypes = new IntegerList();
if ( expecting!=null && !expecting.trim().isEmpty() ) {
for (String tname : expecting.replace(" ", "").split(",")) {
int ttype = g.getTokenType(tname);
expectingTokenTypes.add(ttype);
}
}
return expectingTokenTypes;
}
public IntegerList getTokenTypesViaATN(String input, LexerATNSimulator lexerATN) {
ANTLRInputStream in = new ANTLRInputStream(input);
IntegerList tokenTypes = new IntegerList();
int ttype;
do {
ttype = lexerATN.match(in, Lexer.DEFAULT_MODE);
tokenTypes.add(ttype);
} while ( ttype!= Token.EOF );
return tokenTypes;
}
public List<String> getTokenTypes(LexerGrammar lg,
ATN atn,
CharStream input)
{
LexerATNSimulator interp = new LexerATNSimulator(atn,new DFA[] { new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE)) },null);
List<String> tokenTypes = new ArrayList<String>();
int ttype;
boolean hitEOF = false;
do {
if ( hitEOF ) {
tokenTypes.add("EOF");
break;
}
int t = input.LA(1);
ttype = interp.match(input, Lexer.DEFAULT_MODE);
if ( ttype == Token.EOF ) {
tokenTypes.add("EOF");
}
else {
tokenTypes.add(lg.typeToTokenList.get(ttype));
}
if ( t== IntStream.EOF ) {
hitEOF = true;
}
} while ( ttype!=Token.EOF );
return tokenTypes;
}
List<ANTLRMessage> checkRuleDFA(String gtext, String ruleName, String expecting)
throws Exception
{
ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(gtext, equeue);
ATN atn = createATN(g, false);
ATNState s = atn.ruleToStartState[g.getRule(ruleName).index];
if ( s==null ) {
System.err.println("no such rule: "+ruleName);
return null;
}
ATNState t = s.transition(0).target;
if ( !(t instanceof DecisionState) ) {
System.out.println(ruleName+" has no decision");
return null;
}
DecisionState blk = (DecisionState)t;
checkRuleDFA(g, blk, expecting);
return equeue.all;
}
List<ANTLRMessage> checkRuleDFA(String gtext, int decision, String expecting)
throws Exception
{
ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(gtext, equeue);
ATN atn = createATN(g, false);
DecisionState blk = atn.decisionToState.get(decision);
checkRuleDFA(g, blk, expecting);
return equeue.all;
}
void checkRuleDFA(Grammar g, DecisionState blk, String expecting)
throws Exception
{
DFA dfa = createDFA(g, blk);
String result = null;
if ( dfa!=null ) result = dfa.toString();
assertEquals(expecting, result);
}
List<ANTLRMessage> checkLexerDFA(String gtext, String expecting)
throws Exception
{
return checkLexerDFA(gtext, LexerGrammar.DEFAULT_MODE_NAME, expecting);
}
List<ANTLRMessage> checkLexerDFA(String gtext, String modeName, String expecting)
throws Exception
{
ErrorQueue equeue = new ErrorQueue();
LexerGrammar g = new LexerGrammar(gtext, equeue);
g.atn = createATN(g, false);
// LexerATNToDFAConverter conv = new LexerATNToDFAConverter(g);
// DFA dfa = conv.createDFA(modeName);
// g.setLookaheadDFA(0, dfa); // only one decision to worry about
//
// String result = null;
// if ( dfa!=null ) result = dfa.toString();
// assertEquals(expecting, result);
//
// return equeue.all;
return null;
}
protected String getLanguage() {
return "Cpp";
}
@ -338,44 +57,13 @@ public class BaseCppTest implements RuntimeTestSupport {
null,
lexerName,"-no-listener");
assertTrue(success);
writeFile(tmpdir, "input", input);
writeFile(getTempDirPath(), "input", input);
writeLexerTestFile(lexerName, showDFA);
String output = execModule("Test.cpp");
return output;
}
public ParseTree execStartRule(String startRuleName, Parser parser)
throws IllegalAccessException, InvocationTargetException,
NoSuchMethodException
{
Method startRule = null;
Object[] args = null;
try {
startRule = parser.getClass().getMethod(startRuleName);
}
catch (NoSuchMethodException nsme) {
// try with int _p arg for recursive func
startRule = parser.getClass().getMethod(startRuleName, int.class);
args = new Integer[] {0};
}
ParseTree result = (ParseTree)startRule.invoke(parser, args);
// System.out.println("parse tree = "+result.toStringTree(parser));
return result;
}
// protected String execParser(String grammarFileName,
// String grammarStr,
// String parserName,
// String lexerName,
// String listenerName,
// String visitorName,
// String startRuleName,
// String input,
// boolean debug) {
// return execParser(grammarFileName, grammarStr, parserName, lexerName,
// listenerName, visitorName, startRuleName, input, debug);
// }
//
@Override
public String execParser(String grammarFileName,
String grammarStr,
@ -393,7 +81,7 @@ public class BaseCppTest implements RuntimeTestSupport {
lexerName,
"-visitor");
assertTrue(success);
writeFile(tmpdir, "input", input);
writeFile(getTempDirPath(), "input", input);
rawBuildRecognizerTestFile(parserName,
lexerName,
listenerName,
@ -423,7 +111,7 @@ public class BaseCppTest implements RuntimeTestSupport {
String... extraOptions)
{
ErrorQueue equeue =
antlrOnString(getTmpDir(), "Cpp", grammarFileName, grammarStr, defaultListener, extraOptions);
antlrOnString(getTempDirPath(), "Cpp", grammarFileName, grammarStr, defaultListener, extraOptions);
if (!equeue.errors.isEmpty()) {
return false;
}
@ -457,7 +145,7 @@ public class BaseCppTest implements RuntimeTestSupport {
boolean debug,
boolean trace)
{
this.stderrDuringParse = null;
setParseErrors(null);
if ( parserName==null ) {
writeLexerTestFile(lexerName, false);
}
@ -476,26 +164,6 @@ public class BaseCppTest implements RuntimeTestSupport {
}
private static String detectedOS;
public static String getOS() {
if (detectedOS == null) {
String os = System.getProperty("os.name", "generic").toLowerCase(Locale.ENGLISH);
if ((os.indexOf("mac") >= 0) || (os.indexOf("darwin") >= 0)) {
detectedOS = "mac";
}
else if (os.indexOf("win") >= 0) {
detectedOS = "windows";
}
else if (os.indexOf("nux") >= 0) {
detectedOS = "linux";
}
else {
detectedOS = "unknown";
}
}
return detectedOS;
}
public List<String> allCppFiles(String path) {
ArrayList<String> files = new ArrayList<String>();
File folder = new File(path);
@ -510,7 +178,7 @@ public class BaseCppTest implements RuntimeTestSupport {
}
private String runProcess(ProcessBuilder builder, String description, boolean showStderr) throws Exception {
// System.out.println("BUILDER: "+builder.command());
// System.out.println("BUILDER: " + builder.command() + " @ " + builder.directory().toString());
Process process = builder.start();
StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
@ -521,16 +189,16 @@ public class BaseCppTest implements RuntimeTestSupport {
stderrVacuum.join();
String output = stdoutVacuum.toString();
if ( stderrVacuum.toString().length()>0 ) {
this.stderrDuringParse = stderrVacuum.toString();
if ( showStderr ) System.err.println(this.stderrDuringParse);
setParseErrors(stderrVacuum.toString());
if ( showStderr ) System.err.println(getParseErrors());
}
if (errcode != 0) {
String err = "execution of '"+description+"' failed with error code: "+errcode;
if ( this.stderrDuringParse!=null ) {
this.stderrDuringParse += err;
if ( getParseErrors()!=null ) {
setParseErrors(getParseErrors() + err);
}
else {
this.stderrDuringParse = err;
setParseErrors(err);
}
}
@ -597,15 +265,15 @@ public class BaseCppTest implements RuntimeTestSupport {
public String execModule(String fileName) {
String runtimePath = locateRuntime();
String includePath = runtimePath + "/runtime/src";
String binPath = new File(new File(tmpdir), "a.out").getAbsolutePath();
String inputPath = new File(new File(tmpdir), "input").getAbsolutePath();
String binPath = new File(getTempTestDir(), "a.out").getAbsolutePath();
String inputPath = new File(getTempTestDir(), "input").getAbsolutePath();
// Build runtime using cmake once.
synchronized (runtimeBuiltOnce) {
if ( !runtimeBuiltOnce ) {
try {
String command[] = {"clang++", "--version"};
String output = runCommand(command, tmpdir, "printing compiler version", false);
String output = runCommand(command, getTempDirPath(), "printing compiler version", false);
System.out.println("Compiler version is: "+output);
}
catch (Exception e) {
@ -625,7 +293,7 @@ public class BaseCppTest implements RuntimeTestSupport {
String libExtension = (getOS().equals("mac")) ? "dylib" : "so";
try {
String command[] = { "ln", "-s", runtimePath + "/dist/libantlr4-runtime." + libExtension };
if (runCommand(command, tmpdir, "sym linking C++ runtime", true) == null)
if (runCommand(command, getTempDirPath(), "sym linking C++ runtime", true) == null)
return null;
}
catch (Exception e) {
@ -636,8 +304,8 @@ public class BaseCppTest implements RuntimeTestSupport {
try {
List<String> command2 = new ArrayList<String>(Arrays.asList("clang++", "-std=c++11", "-I", includePath, "-L.", "-lantlr4-runtime", "-o", "a.out"));
command2.addAll(allCppFiles(tmpdir));
if (runCommand(command2.toArray(new String[0]), tmpdir, "building test binary", true) == null) {
command2.addAll(allCppFiles(getTempDirPath()));
if (runCommand(command2.toArray(new String[0]), getTempDirPath(), "building test binary", true) == null) {
return null;
}
}
@ -648,10 +316,10 @@ public class BaseCppTest implements RuntimeTestSupport {
}
// Now run the newly minted binary. Reset the error output, as we could have got compiler warnings which are not relevant here.
this.stderrDuringParse = null;
setParseErrors(null);
try {
ProcessBuilder builder = new ProcessBuilder(binPath, inputPath);
builder.directory(new File(tmpdir));
builder.directory(getTempTestDir());
Map<String, String> env = builder.environment();
env.put("LD_PRELOAD", runtimePath + "/dist/libantlr4-runtime." + libExtension);
String output = runProcess(builder, "running test binary", false);
@ -688,156 +356,11 @@ public class BaseCppTest implements RuntimeTestSupport {
p = Paths.get(runtimeURL.toURI()).toFile().toString();
}
catch (URISyntaxException use) {
p = "Can't find runtime";
p = "Can't find runtime at " + runtimeURL.toString();
}
return p;
}
List<ANTLRMessage> getMessagesOfType(List<ANTLRMessage> msgs, Class<? extends ANTLRMessage> c) {
List<ANTLRMessage> filtered = new ArrayList<ANTLRMessage>();
for (ANTLRMessage m : msgs) {
if ( m.getClass() == c ) filtered.add(m);
}
return filtered;
}
void checkRuleATN(Grammar g, String ruleName, String expecting) {
ParserATNFactory f = new ParserATNFactory(g);
ATN atn = f.createATN();
DOTGenerator dot = new DOTGenerator(g);
System.out.println(dot.getDOT(atn.ruleToStartState[g.getRule(ruleName).index]));
Rule r = g.getRule(ruleName);
ATNState startState = atn.ruleToStartState[r.index];
ATNPrinter serializer = new ATNPrinter(g, startState);
String result = serializer.asString();
//System.out.print(result);
assertEquals(expecting, result);
}
public void testActions(String templates, String actionName, String action, String expected) throws org.antlr.runtime.RecognitionException {
int lp = templates.indexOf('(');
String name = templates.substring(0, lp);
STGroup group = new STGroupString(templates);
ST st = group.getInstanceOf(name);
st.add(actionName, action);
String grammar = st.render();
ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(grammar, equeue);
if ( g.ast!=null && !g.ast.hasErrors ) {
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
ATNFactory factory = new ParserATNFactory(g);
if ( g.isLexer() ) factory = new LexerATNFactory((LexerGrammar)g);
g.atn = factory.createATN();
CodeGenerator gen = new CodeGenerator(g);
ST outputFileST = gen.generateParser();
String output = outputFileST.render();
//System.out.println(output);
String b = "#" + actionName + "#";
int start = output.indexOf(b);
String e = "#end-" + actionName + "#";
int end = output.indexOf(e);
String snippet = output.substring(start+b.length(),end);
assertEquals(expected, snippet);
}
if ( equeue.size()>0 ) {
System.err.println(equeue.toString());
}
}
protected void checkGrammarSemanticsError(ErrorQueue equeue,
GrammarSemanticsMessage expectedMessage)
throws Exception
{
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
ANTLRMessage m = equeue.errors.get(i);
if (m.getErrorType()==expectedMessage.getErrorType() ) {
foundMsg = m;
}
}
assertNotNull("no error; "+expectedMessage.getErrorType()+" expected", foundMsg);
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs()));
if ( equeue.size()!=1 ) {
System.err.println(equeue);
}
}
protected void checkGrammarSemanticsWarning(ErrorQueue equeue,
GrammarSemanticsMessage expectedMessage)
throws Exception
{
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.warnings.size(); i++) {
ANTLRMessage m = equeue.warnings.get(i);
if (m.getErrorType()==expectedMessage.getErrorType() ) {
foundMsg = m;
}
}
assertNotNull("no error; "+expectedMessage.getErrorType()+" expected", foundMsg);
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs()));
if ( equeue.size()!=1 ) {
System.err.println(equeue);
}
}
protected void checkError(ErrorQueue equeue,
ANTLRMessage expectedMessage)
throws Exception
{
//System.out.println("errors="+equeue);
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
ANTLRMessage m = equeue.errors.get(i);
if (m.getErrorType()==expectedMessage.getErrorType() ) {
foundMsg = m;
}
}
assertTrue("no error; "+expectedMessage.getErrorType()+" expected", !equeue.errors.isEmpty());
assertTrue("too many errors; "+equeue.errors, equeue.errors.size()<=1);
assertNotNull("couldn't find expected error: "+expectedMessage.getErrorType(), foundMsg);
/*
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
*/
assertArrayEquals(expectedMessage.getArgs(), foundMsg.getArgs());
}
public static class FilteringTokenStream extends CommonTokenStream {
public FilteringTokenStream(TokenSource src) { super(src); }
Set<Integer> hide = new HashSet<Integer>();
@Override
protected boolean sync(int i) {
if (!super.sync(i)) {
return false;
}
Token t = get(i);
if ( hide.contains(t.getType()) ) {
((WritableToken)t).setChannel(Token.HIDDEN_CHANNEL);
}
return true;
}
public void setTokenTypeChannel(int ttype, int channel) {
hide.add(ttype);
}
}
protected void mkdir(String dir) {
File f = new File(dir);
f.mkdirs();
}
protected void writeParserTestFile(String parserName, String lexerName,
String listenerName, String visitorName,
String parserStartRuleName, boolean debug, boolean trace) {
@ -871,7 +394,8 @@ public class BaseCppTest implements RuntimeTestSupport {
+ "\n"
+ "\n"
+ "int main(int argc, const char* argv[]) {\n"
+ " ANTLRFileStream input(argv[1]);\n"
+ " ANTLRFileStream input;\n"
+ " input.loadFromFile(argv[1]);\n"
+ " <lexerName> lexer(&input);\n"
+ " CommonTokenStream tokens(&lexer);\n"
+ "<createParser>"
@ -898,7 +422,7 @@ public class BaseCppTest implements RuntimeTestSupport {
outputFileST.add("listenerName", listenerName);
outputFileST.add("visitorName", visitorName);
outputFileST.add("parserStartRuleName", parserStartRuleName);
writeFile(tmpdir, "Test.cpp", outputFileST.render());
writeFile(getTempDirPath(), "Test.cpp", outputFileST.render());
}
protected void writeLexerTestFile(String lexerName, boolean showDFA) {
@ -913,7 +437,8 @@ public class BaseCppTest implements RuntimeTestSupport {
+ "using namespace antlr4;\n"
+ "\n"
+ "int main(int argc, const char* argv[]) {\n"
+ " ANTLRFileStream input(argv[1]);\n"
+ " ANTLRFileStream input;\n"
+ " input.loadFromFile(argv[1]);\n"
+ " <lexerName> lexer(&input);\n"
+ " CommonTokenStream tokens(&lexer);\n"
+ " tokens.fill();\n"
@ -923,208 +448,8 @@ public class BaseCppTest implements RuntimeTestSupport {
+ " return 0;\n"
+ "}\n");
outputFileST.add("lexerName", lexerName);
writeFile(tmpdir, "Test.cpp", outputFileST.render());
writeFile(getTempDirPath(), "Test.cpp", outputFileST.render());
}
public void writeRecognizer(String parserName, String lexerName,
String listenerName, String visitorName,
String parserStartRuleName, boolean debug, boolean trace) {
if ( parserName==null ) {
writeLexerTestFile(lexerName, debug);
}
else {
writeParserTestFile(parserName,
lexerName,
listenerName,
visitorName,
parserStartRuleName,
debug,
trace);
}
}
protected void eraseFiles(final String filesEndingWith) {
File tmpdirF = new File(tmpdir);
String[] files = tmpdirF.list();
for(int i = 0; files!=null && i < files.length; i++) {
if ( files[i].endsWith(filesEndingWith) ) {
new File(tmpdir+"/"+files[i]).delete();
}
}
}
protected void eraseFiles(File dir) {
String[] files = dir.list();
for(int i = 0; files!=null && i < files.length; i++) {
new File(dir,files[i]).delete();
}
}
@Override
public void eraseTempDir() {
boolean doErase = true;
String propName = getPropertyPrefix() + "-erase-test-dir";
String prop = System.getProperty(propName);
if(prop!=null && prop.length()>0)
doErase = Boolean.getBoolean(prop);
if(doErase) {
File tmpdirF = new File(tmpdir);
if ( tmpdirF.exists() ) {
eraseFiles(tmpdirF);
tmpdirF.delete();
}
}
}
public String getFirstLineOfException() {
if ( this.stderrDuringParse ==null ) {
return null;
}
String[] lines = this.stderrDuringParse.split("\n");
String prefix="Exception in thread \"main\" ";
return lines[0].substring(prefix.length(),lines[0].length());
}
/**
* When looking at a result set that consists of a Map/HashTable
* we cannot rely on the output order, as the hashing algorithm or other aspects
* of the implementation may be different on different JDKs or platforms. Hence
* we take the Map, convert the keys to a List, sort them and Stringify the Map, which is a
* bit of a hack, but guarantees that we get the same order on all systems. We assume that
* the keys are strings.
*
* @param m The Map that contains keys we wish to return in sorted order
* @return A string that represents all the keys in sorted order.
*/
public <K, V> String sortMapToString(Map<K, V> m) {
// Pass in crap, and get nothing back
//
if (m == null) {
return null;
}
System.out.println("Map toString looks like: " + m.toString());
// Sort the keys in the Map
//
TreeMap<K, V> nset = new TreeMap<K, V>(m);
System.out.println("Tree map looks like: " + nset.toString());
return nset.toString();
}
public List<String> realElements(List<String> elements) {
return elements.subList(Token.MIN_USER_TOKEN_TYPE, elements.size());
}
public void assertNotNullOrEmpty(String message, String text) {
assertNotNull(message, text);
assertFalse(message, text.isEmpty());
}
public void assertNotNullOrEmpty(String text) {
assertNotNull(text);
assertFalse(text.isEmpty());
}
public static class IntTokenStream implements TokenStream {
IntegerList types;
int p=0;
public IntTokenStream(IntegerList types) { this.types = types; }
@Override
public void consume() { p++; }
@Override
public int LA(int i) { return LT(i).getType(); }
@Override
public int mark() {
return index();
}
@Override
public int index() { return p; }
@Override
public void release(int marker) {
seek(marker);
}
@Override
public void seek(int index) {
p = index;
}
@Override
public int size() {
return types.size();
}
@Override
public String getSourceName() {
return null;
}
@Override
public Token LT(int i) {
CommonToken t;
int rawIndex = p + i - 1;
if ( rawIndex>=types.size() ) t = new CommonToken(Token.EOF);
else t = new CommonToken(types.get(rawIndex));
t.setTokenIndex(rawIndex);
return t;
}
@Override
public Token get(int i) {
return new org.antlr.v4.runtime.CommonToken(types.get(i));
}
@Override
public TokenSource getTokenSource() {
return null;
}
@Override
public String getText() {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Interval interval) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(RuleContext ctx) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Token start, Token stop) {
throw new UnsupportedOperationException("can't give strings");
}
}
/** Sort a list */
public <T extends Comparable<? super T>> List<T> sort(List<T> data) {
List<T> dup = new ArrayList<T>();
dup.addAll(data);
Collections.sort(dup);
return dup;
}
/** Return map sorted by key */
public <K extends Comparable<? super K>,V> LinkedHashMap<K,V> sort(Map<K,V> data) {
LinkedHashMap<K,V> dup = new LinkedHashMap<K, V>();
List<K> keys = new ArrayList<K>();
keys.addAll(data.keySet());
Collections.sort(keys);
for (K k : keys) {
dup.put(k, data.get(k));
}
return dup;
}
}

6
runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/Antlr4.Test.dotnet.csproj → runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/Antlr4.Test.csproj
View File

@ -1,13 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp1.0</TargetFramework>
<TargetFramework>netcoreapp3.1</TargetFramework>
<NoWarn>$(NoWarn);CS3021</NoWarn>
<AssemblyName>Test</AssemblyName>
<OutputType>Exe</OutputType>
<PackageId>Antlr4.Test.dotnet</PackageId>
<RuntimeFrameworkVersion>1.1.1</RuntimeFrameworkVersion>
<PackageTargetFallback>$(PackageTargetFallback);dnxcore50</PackageTargetFallback>
<PackageId>Antlr4.Test</PackageId>
<GenerateAssemblyTitleAttribute>false</GenerateAssemblyTitleAttribute>
<GenerateAssemblyDescriptionAttribute>false</GenerateAssemblyDescriptionAttribute>
<GenerateAssemblyConfigurationAttribute>false</GenerateAssemblyConfigurationAttribute>

51
runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/Antlr4.Test.mono.csproj
View File

@ -1,51 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003" DefaultTargets="Build" ToolsVersion="4.0">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{EDC70A11-C4C1-4209-93A6-CCE2B19E8E95}</ProjectGuid>
<OutputType>Exe</OutputType>
<RootNamespace>Antlr4.Test.mono</RootNamespace>
<AssemblyName>Test</AssemblyName>
<StartupObject>Test</StartupObject>
<TargetFrameworkVersion>v3.5</TargetFrameworkVersion>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug</OutputPath>
<DefineConstants>DEBUG;</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<Externalconsole>true</Externalconsole>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<Optimize>true</Optimize>
<OutputPath>bin\Release</OutputPath>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<Externalconsole>true</Externalconsole>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="Antlr4.Runtime.mono.csproj">
<Project>{E1A46D9D-66CB-46E8-93B0-7FC87299ABEF}</Project>
<Name>Antlr4.Runtime.mono</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<Compile Include="AssemblyInfo.cs" />
<Compile Include="Test.cs" />
<Compile Include="L.cs" />
</ItemGroup>
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
<ItemGroup>
<None Include="App.config">
<LogicalName>Test.exe.config</LogicalName>
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

44
runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/Antlr4.Test.vs2013.csproj
View File

@ -1,44 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?><Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003" DefaultTargets="Build" ToolsVersion="4.0">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{EDC70A11-C4C1-4209-93A6-CCE2B19E8E95}</ProjectGuid>
<OutputType>Exe</OutputType>
<RootNamespace>Antlr4.Test.mono</RootNamespace>
<AssemblyName>Test</AssemblyName>
<StartupObject>Test</StartupObject>
<TargetFrameworkVersion>v3.5</TargetFrameworkVersion>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug</OutputPath>
<DefineConstants>DEBUG;</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<Externalconsole>true</Externalconsole>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<Optimize>true</Optimize>
<OutputPath>bin\Release</OutputPath>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<Externalconsole>true</Externalconsole>
</PropertyGroup>
<ItemGroup>
<Reference Include="System"/>
</ItemGroup>
<ItemGroup>
<Compile Include="AssemblyInfo.cs"/>
<Compile Include="Test.cs"/>
<Compile Include="L.cs"/>
</ItemGroup>
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets"/>
<ItemGroup>
<ProjectReference Include="Antlr4.Runtime.vs2013.csproj">
<Project>{E1A46D9D-66CB-46E8-93B0-7FC87299ABEF}</Project>
<Name>Antlr4.Runtime.vs2013</Name>
</ProjectReference>
</ItemGroup>
</Project>

7
runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/App.config
View File

@ -1,7 +0,0 @@
<?xml version="1.0" encoding="UTF-8" ?>
<configuration>
<startup>
<supportedRuntime version="v2.0.50727"/>
<supportedRuntime version="v4.0"/>
</startup>
</configuration>

28
runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/AssemblyInfo.cs
View File

@ -1,28 +0,0 @@
using System;
using System.Reflection;
using System.Runtime.CompilerServices;
// Information about this assembly is defined by the following attributes.
// Change them to the values specific to your project.
[assembly: AssemblyTitle ("Antlr4.Test.mono")]
[assembly: AssemblyDescription ("")]
[assembly: AssemblyConfiguration ("")]
[assembly: AssemblyCompany ("")]
[assembly: AssemblyProduct ("")]
[assembly: AssemblyCopyright ("ericvergnaud")]
[assembly: AssemblyTrademark ("")]
[assembly: AssemblyCulture ("")]
[assembly: CLSCompliant (true)]
// The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}".
// The form "{Major}.{Minor}.*" will automatically update the build and revision,
// and "{Major}.{Minor}.{Build}.*" will update just the revision.
[assembly: AssemblyVersion ("1.0.*")]
// The following attributes are used to specify the signing key for the assembly,
// if desired. See the Mono documentation for more information about signing.
//[assembly: AssemblyDelaySign(false)]
//[assembly: AssemblyKeyFile("")]

931
runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java
View File

File diff suppressed because it is too large Load Diff

841
runtime-testsuite/test/org/antlr/v4/test/runtime/dart/BaseDartTest.java
View File

@ -6,39 +6,14 @@
package org.antlr.v4.test.runtime.dart;
import org.antlr.v4.Tool;
import org.antlr.v4.analysis.AnalysisPipeline;
import org.antlr.v4.automata.ATNFactory;
import org.antlr.v4.automata.ATNPrinter;
import org.antlr.v4.automata.LexerATNFactory;
import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.Pair;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.semantics.SemanticPipeline;
import org.antlr.v4.test.runtime.BaseRuntimeTest;
import org.antlr.v4.test.runtime.ErrorQueue;
import org.antlr.v4.test.runtime.RuntimeTestSupport;
import org.antlr.v4.test.runtime.StreamVacuum;
import org.antlr.v4.test.runtime.*;
import org.antlr.v4.test.runtime.descriptors.LexerExecDescriptors;
import org.antlr.v4.test.runtime.descriptors.PerformanceDescriptors;
import org.antlr.v4.tool.*;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupString;
import java.io.*;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.*;
import static junit.framework.TestCase.*;
@ -47,333 +22,19 @@ import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
import static org.junit.Assert.assertArrayEquals;
public class BaseDartTest implements RuntimeTestSupport {
public class BaseDartTest extends BaseRuntimeTestSupport implements RuntimeTestSupport {
private static final List<String> AOT_COMPILE_TESTS = Arrays.asList(
new PerformanceDescriptors.DropLoopEntryBranchInLRRule_4().input,
new LexerExecDescriptors.LargeLexer().input
);
public static final String newline = System.getProperty("line.separator");
public static final String pathSep = System.getProperty("path.separator");
/**
* When the {@code antlr.preserve-test-dir} runtime property is set to
* {@code true}, the temporary directories created by the test run will not
* be removed at the end of the test run, even for tests that completed
* successfully.
* <p>
* <p>
* The default behavior (used in all other cases) is removing the temporary
* directories for all tests which completed successfully, and preserving
* the directories for tests which failed.</p>
*/
public static final boolean PRESERVE_TEST_DIR = Boolean.parseBoolean(System.getProperty("antlr.preserve-test-dir", "false"));
/**
* The base test directory is the directory where generated files get placed
* during unit test execution.
* <p>
* <p>
* The default value for this property is the {@code java.io.tmpdir} system
* property, and can be overridden by setting the
* {@code antlr.java-test-dir} property to a custom location. Note that the
* {@code antlr.java-test-dir} property directly affects the
* {@link #CREATE_PER_TEST_DIRECTORIES} value as well.</p>
*/
public static final String BASE_TEST_DIR;
/**
* When {@code true}, a temporary directory will be created for each test
* executed during the test run.
* <p>
* <p>
* This value is {@code true} when the {@code antlr.java-test-dir} system
* property is set, and otherwise {@code false}.</p>
*/
public static final boolean CREATE_PER_TEST_DIRECTORIES;
static {
String baseTestDir = System.getProperty("antlr.dart-test-dir");
boolean perTestDirectories = false;
if (baseTestDir == null || baseTestDir.isEmpty()) {
baseTestDir = System.getProperty("java.io.tmpdir");
perTestDirectories = true;
}
if (!new File(baseTestDir).isDirectory()) {
throw new UnsupportedOperationException("The specified base test directory does not exist: " + baseTestDir);
}
BASE_TEST_DIR = baseTestDir;
CREATE_PER_TEST_DIRECTORIES = perTestDirectories;
}
/**
* Build up the full classpath we need, including the surefire path (if present)
*/
public static final String CLASSPATH = System.getProperty("java.class.path");
public String tmpdir = null;
/**
* If error during parser execution, store stderr here; can't return
* stdout and stderr. This doesn't trap errors from running antlr.
*/
protected String stderrDuringParse;
/**
* Errors found while running antlr
*/
protected StringBuilder antlrToolErrors;
private static String cacheDartPackages;
private String getPropertyPrefix() {
public String getPropertyPrefix() {
return "antlr-dart";
}
@Override
public void testSetUp() throws Exception {
if (CREATE_PER_TEST_DIRECTORIES) {
// new output dir for each test
String threadName = Thread.currentThread().getName();
String testDirectory = getClass().getSimpleName() + "-" + threadName + "-" + System.nanoTime();
tmpdir = new File(BASE_TEST_DIR, testDirectory).getAbsolutePath();
} else {
tmpdir = new File(BASE_TEST_DIR).getAbsolutePath();
if (!PRESERVE_TEST_DIR && new File(tmpdir).exists()) {
eraseFiles();
}
}
antlrToolErrors = new StringBuilder();
}
@Override
public void testTearDown() throws Exception {
}
@Override
public String getTmpDir() {
return tmpdir;
}
@Override
public String getStdout() {
return null;
}
@Override
public String getParseErrors() {
return stderrDuringParse;
}
@Override
public String getANTLRToolErrors() {
if (antlrToolErrors.length() == 0) {
return null;
}
return antlrToolErrors.toString();
}
protected Tool newTool(String[] args) {
Tool tool = new Tool(args);
return tool;
}
protected ATN createATN(Grammar g, boolean useSerializer) {
if (g.atn == null) {
semanticProcess(g);
assertEquals(0, g.tool.getNumErrors());
ParserATNFactory f;
if (g.isLexer()) {
f = new LexerATNFactory((LexerGrammar) g);
} else {
f = new ParserATNFactory(g);
}
g.atn = f.createATN();
assertEquals(0, g.tool.getNumErrors());
}
ATN atn = g.atn;
if (useSerializer) {
char[] serialized = ATNSerializer.getSerializedAsChars(atn);
return new ATNDeserializer().deserialize(serialized);
}
return atn;
}
protected void semanticProcess(Grammar g) {
if (g.ast != null && !g.ast.hasErrors) {
// System.out.println(g.ast.toStringTree());
Tool antlr = new Tool();
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
if (g.getImportedGrammars() != null) { // process imported grammars (if any)
for (Grammar imp : g.getImportedGrammars()) {
antlr.processNonCombinedGrammar(imp, false);
}
}
}
}
public DFA createDFA(Grammar g, DecisionState s) {
// PredictionDFAFactory conv = new PredictionDFAFactory(g, s);
// DFA dfa = conv.createDFA();
// conv.issueAmbiguityWarnings();
// System.out.print("DFA="+dfa);
// return dfa;
return null;
}
// public void minimizeDFA(DFA dfa) {
// DFAMinimizer dmin = new DFAMinimizer(dfa);
// dfa.minimized = dmin.minimize();
// }
IntegerList getTypesFromString(Grammar g, String expecting) {
IntegerList expectingTokenTypes = new IntegerList();
if (expecting != null && !expecting.trim().isEmpty()) {
for (String tname : expecting.replace(" ", "").split(",")) {
int ttype = g.getTokenType(tname);
expectingTokenTypes.add(ttype);
}
}
return expectingTokenTypes;
}
public IntegerList getTokenTypesViaATN(String input, LexerATNSimulator lexerATN) {
ANTLRInputStream in = new ANTLRInputStream(input);
IntegerList tokenTypes = new IntegerList();
int ttype;
do {
ttype = lexerATN.match(in, Lexer.DEFAULT_MODE);
tokenTypes.add(ttype);
} while (ttype != Token.EOF);
return tokenTypes;
}
public List<String> getTokenTypes(LexerGrammar lg,
ATN atn,
CharStream input) {
LexerATNSimulator interp = new LexerATNSimulator(atn, new DFA[]{new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE))}, null);
List<String> tokenTypes = new ArrayList<String>();
int ttype;
boolean hitEOF = false;
do {
if (hitEOF) {
tokenTypes.add("EOF");
break;
}
int t = input.LA(1);
ttype = interp.match(input, Lexer.DEFAULT_MODE);
if (ttype == Token.EOF) {
tokenTypes.add("EOF");
} else {
tokenTypes.add(lg.typeToTokenList.get(ttype));
}
if (t == IntStream.EOF) {
hitEOF = true;
}
} while (ttype != Token.EOF);
return tokenTypes;
}
List<ANTLRMessage> checkRuleDFA(String gtext, String ruleName, String expecting)
throws Exception {
ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(gtext, equeue);
ATN atn = createATN(g, false);
ATNState s = atn.ruleToStartState[g.getRule(ruleName).index];
if (s == null) {
System.err.println("no such rule: " + ruleName);
return null;
}
ATNState t = s.transition(0).target;
if (!(t instanceof DecisionState)) {
System.out.println(ruleName + " has no decision");
return null;
}
DecisionState blk = (DecisionState) t;
checkRuleDFA(g, blk, expecting);
return equeue.all;
}
List<ANTLRMessage> checkRuleDFA(String gtext, int decision, String expecting)
throws Exception {
ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(gtext, equeue);
ATN atn = createATN(g, false);
DecisionState blk = atn.decisionToState.get(decision);
checkRuleDFA(g, blk, expecting);
return equeue.all;
}
void checkRuleDFA(Grammar g, DecisionState blk, String expecting)
throws Exception {
DFA dfa = createDFA(g, blk);
String result = null;
if (dfa != null) result = dfa.toString();
assertEquals(expecting, result);
}
List<ANTLRMessage> checkLexerDFA(String gtext, String expecting)
throws Exception {
return checkLexerDFA(gtext, LexerGrammar.DEFAULT_MODE_NAME, expecting);
}
List<ANTLRMessage> checkLexerDFA(String gtext, String modeName, String expecting)
throws Exception {
ErrorQueue equeue = new ErrorQueue();
LexerGrammar g = new LexerGrammar(gtext, equeue);
g.atn = createATN(g, false);
// LexerATNToDFAConverter conv = new LexerATNToDFAConverter(g);
// DFA dfa = conv.createDFA(modeName);
// g.setLookaheadDFA(0, dfa); // only one decision to worry about
//
// String result = null;
// if ( dfa!=null ) result = dfa.toString();
// assertEquals(expecting, result);
//
// return equeue.all;
return null;
}
protected String load(String fileName, String encoding)
throws IOException {
if (fileName == null) {
return null;
}
String fullFileName = getClass().getPackage().getName().replace('.', '/') + '/' + fileName;
int size = 65000;
InputStreamReader isr;
InputStream fis = getClass().getClassLoader().getResourceAsStream(fullFileName);
if (encoding != null) {
isr = new InputStreamReader(fis, encoding);
} else {
isr = new InputStreamReader(fis);
}
try {
char[] data = new char[size];
int n = isr.read(data);
return new String(data, 0, n);
} finally {
isr.close();
}
}
protected String execLexer(String grammarFileName,
String grammarStr,
String lexerName,
String input) {
return execLexer(grammarFileName, grammarStr, lexerName, input, false);
}
@Override
public String execLexer(String grammarFileName,
String grammarStr,
@ -385,71 +46,12 @@ public class BaseDartTest implements RuntimeTestSupport {
null,
lexerName);
assertTrue(success);
writeFile(tmpdir, "input", input);
writeFile(getTempDirPath(), "input", input);
writeLexerTestFile(lexerName, showDFA);
String output = execClass("Test", AOT_COMPILE_TESTS.contains(input));
return output;
}
public ParseTree execParser(String startRuleName, String input,
String parserName, String lexerName)
throws Exception {
Pair<Parser, Lexer> pl = getParserAndLexer(input, parserName, lexerName);
Parser parser = pl.a;
return execStartRule(startRuleName, parser);
}
public ParseTree execStartRule(String startRuleName, Parser parser)
throws IllegalAccessException, InvocationTargetException,
NoSuchMethodException {
Method startRule = null;
Object[] args = null;
try {
startRule = parser.getClass().getMethod(startRuleName);
} catch (NoSuchMethodException nsme) {
// try with int _p arg for recursive func
startRule = parser.getClass().getMethod(startRuleName, int.class);
args = new Integer[]{0};
}
ParseTree result = (ParseTree) startRule.invoke(parser, args);
// System.out.println("parse tree = "+result.toStringTree(parser));
return result;
}
public Pair<Parser, Lexer> getParserAndLexer(String input,
String parserName, String lexerName)
throws Exception {
final Class<? extends Lexer> lexerClass = loadLexerClassFromTempDir(lexerName);
final Class<? extends Parser> parserClass = loadParserClassFromTempDir(parserName);
ANTLRInputStream in = new ANTLRInputStream(new StringReader(input));
Class<? extends Lexer> c = lexerClass.asSubclass(Lexer.class);
Constructor<? extends Lexer> ctor = c.getConstructor(CharStream.class);
Lexer lexer = ctor.newInstance(in);
Class<? extends Parser> pc = parserClass.asSubclass(Parser.class);
Constructor<? extends Parser> pctor = pc.getConstructor(TokenStream.class);
CommonTokenStream tokens = new CommonTokenStream(lexer);
Parser parser = pctor.newInstance(tokens);
return new Pair<Parser, Lexer>(parser, lexer);
}
public Class<?> loadClassFromTempDir(String name) throws Exception {
ClassLoader loader =
new URLClassLoader(new URL[]{new File(tmpdir).toURI().toURL()},
ClassLoader.getSystemClassLoader());
return loader.loadClass(name);
}
public Class<? extends Lexer> loadLexerClassFromTempDir(String name) throws Exception {
return loadClassFromTempDir(name).asSubclass(Lexer.class);
}
public Class<? extends Parser> loadParserClassFromTempDir(String name) throws Exception {
return loadClassFromTempDir(name).asSubclass(Parser.class);
}
@Override
public String execParser(String grammarFileName,
String grammarStr,
@ -480,7 +82,7 @@ public class BaseDartTest implements RuntimeTestSupport {
lexerName,
"-visitor");
assertTrue(success);
writeFile(tmpdir, "input", input);
writeFile(getTempDirPath(), "input", input);
return rawExecRecognizer(parserName,
lexerName,
startRuleName,
@ -510,7 +112,7 @@ public class BaseDartTest implements RuntimeTestSupport {
boolean defaultListener,
String... extraOptions) {
ErrorQueue equeue =
BaseRuntimeTest.antlrOnString(getTmpDir(), "Dart", grammarFileName, grammarStr, defaultListener, extraOptions);
BaseRuntimeTest.antlrOnString(getTempDirPath(), "Dart", grammarFileName, grammarStr, defaultListener, extraOptions);
if (!equeue.errors.isEmpty()) {
return false;
}
@ -534,17 +136,29 @@ public class BaseDartTest implements RuntimeTestSupport {
}
String runtime = locateRuntime();
writeFile(tmpdir, "pubspec.yaml",
writeFile(getTempDirPath(), "pubspec.yaml",
"name: \"test\"\n" +
"dependencies:\n" +
" antlr4:\n" +
" path: " + runtime + "\n");
if (cacheDartPackages == null) {
try {
Process process = Runtime.getRuntime().exec(new String[]{locatePub(), "get"}, null, new File(tmpdir));
final Process process = Runtime.getRuntime().exec(new String[]{locatePub(), "get"}, null, getTempTestDir());
StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
stderrVacuum.start();
Timer timer = new Timer();
timer.schedule(new TimerTask() {
@Override
public void run() {
try {
process.destroy();
} catch(Exception e) {
e.printStackTrace(System.err);
}
}
}, 30_000);
process.waitFor();
timer.cancel();
stderrVacuum.join();
String stderrDuringPubGet = stderrVacuum.toString();
if (!stderrDuringPubGet.isEmpty()) {
@ -554,9 +168,9 @@ public class BaseDartTest implements RuntimeTestSupport {
e.printStackTrace();
return false;
}
cacheDartPackages = readFile(tmpdir, ".packages");
cacheDartPackages = readFile(getTempDirPath(), ".packages");
} else {
writeFile(tmpdir, ".packages", cacheDartPackages);
writeFile(getTempDirPath(), ".packages", cacheDartPackages);
}
return true; // allIsWell: no compile
}
@ -567,7 +181,7 @@ public class BaseDartTest implements RuntimeTestSupport {
boolean debug,
boolean profile,
boolean aotCompile) {
this.stderrDuringParse = null;
setParseErrors(null);
if (parserName == null) {
writeLexerTestFile(lexerName, false);
} else {
@ -590,11 +204,23 @@ public class BaseDartTest implements RuntimeTestSupport {
};
String cmdLine = Utils.join(args, " ");
System.err.println("Compile: " + cmdLine);
Process process =
Runtime.getRuntime().exec(args, null, new File(tmpdir));
final Process process =
Runtime.getRuntime().exec(args, null, getTempTestDir());
StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
stderrVacuum.start();
Timer timer = new Timer();
timer.schedule(new TimerTask() {
@Override
public void run() {
try {
process.destroy();
} catch(Exception e) {
e.printStackTrace(System.err);
}
}
}, 30_000);
int result = process.waitFor();
timer.cancel();
if (result != 0) {
stderrVacuum.join();
System.err.print("Error compiling dart file: " + stderrVacuum.toString());
@ -604,23 +230,35 @@ public class BaseDartTest implements RuntimeTestSupport {
String[] args;
if (compile) {
args = new String[]{
new File(tmpdir, className).getAbsolutePath(), new File(tmpdir, "input").getAbsolutePath()
new File(getTempTestDir(), className).getAbsolutePath(), new File(getTempTestDir(), "input").getAbsolutePath()
};
} else {
args = new String[]{
locateDart(),
className + ".dart", new File(tmpdir, "input").getAbsolutePath()
className + ".dart", new File(getTempTestDir(), "input").getAbsolutePath()
};
}
//String cmdLine = Utils.join(args, " ");
//System.err.println("execParser: " + cmdLine);
Process process =
Runtime.getRuntime().exec(args, null, new File(tmpdir));
final Process process =
Runtime.getRuntime().exec(args, null, getTempTestDir());
StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
stdoutVacuum.start();
stderrVacuum.start();
Timer timer = new Timer();
timer.schedule(new TimerTask() {
@Override
public void run() {
try {
process.destroy();
} catch(Exception e) {
e.printStackTrace(System.err);
}
}
}, 30_000);
process.waitFor();
timer.cancel();
stdoutVacuum.join();
stderrVacuum.join();
String output = stdoutVacuum.toString();
@ -628,7 +266,7 @@ public class BaseDartTest implements RuntimeTestSupport {
output = null;
}
if (stderrVacuum.toString().length() > 0) {
this.stderrDuringParse = stderrVacuum.toString();
setParseErrors(stderrVacuum.toString());
}
return output;
} catch (Exception e) {
@ -731,187 +369,6 @@ public class BaseDartTest implements RuntimeTestSupport {
return runtimeSrc.getPath();
}
private boolean isWindows() {
return System.getProperty("os.name").toLowerCase().contains("windows");
}
// void ambig(List<Message> msgs, int[] expectedAmbigAlts, String expectedAmbigInput)
// throws Exception
// {
// ambig(msgs, 0, expectedAmbigAlts, expectedAmbigInput);
// }
// void ambig(List<Message> msgs, int i, int[] expectedAmbigAlts, String expectedAmbigInput)
// throws Exception
// {
// List<Message> amsgs = getMessagesOfType(msgs, AmbiguityMessage.class);
// AmbiguityMessage a = (AmbiguityMessage)amsgs.get(i);
// if ( a==null ) assertNull(expectedAmbigAlts);
// else {
// assertEquals(a.conflictingAlts.toString(), Arrays.toString(expectedAmbigAlts));
// }
// assertEquals(expectedAmbigInput, a.input);
// }
// void unreachable(List<Message> msgs, int[] expectedUnreachableAlts)
// throws Exception
// {
// unreachable(msgs, 0, expectedUnreachableAlts);
// }
// void unreachable(List<Message> msgs, int i, int[] expectedUnreachableAlts)
// throws Exception
// {
// List<Message> amsgs = getMessagesOfType(msgs, UnreachableAltsMessage.class);
// UnreachableAltsMessage u = (UnreachableAltsMessage)amsgs.get(i);
// if ( u==null ) assertNull(expectedUnreachableAlts);
// else {
// assertEquals(u.conflictingAlts.toString(), Arrays.toString(expectedUnreachableAlts));
// }
// }
List<ANTLRMessage> getMessagesOfType(List<ANTLRMessage> msgs, Class<? extends ANTLRMessage> c) {
List<ANTLRMessage> filtered = new ArrayList<ANTLRMessage>();
for (ANTLRMessage m : msgs) {
if (m.getClass() == c) filtered.add(m);
}
return filtered;
}
public void checkRuleATN(Grammar g, String ruleName, String expecting) {
// DOTGenerator dot = new DOTGenerator(g);
// System.out.println(dot.getDOT(g.atn.ruleToStartState[g.getRule(ruleName).index]));
Rule r = g.getRule(ruleName);
ATNState startState = g.getATN().ruleToStartState[r.index];
ATNPrinter serializer = new ATNPrinter(g, startState);
String result = serializer.asString();
//System.out.print(result);
assertEquals(expecting, result);
}
public void testActions(String templates, String actionName, String action, String expected) throws org.antlr.runtime.RecognitionException {
int lp = templates.indexOf('(');
String name = templates.substring(0, lp);
STGroup group = new STGroupString(templates);
ST st = group.getInstanceOf(name);
st.add(actionName, action);
String grammar = st.render();
ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(grammar, equeue);
if (g.ast != null && !g.ast.hasErrors) {
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
ATNFactory factory = new ParserATNFactory(g);
if (g.isLexer()) factory = new LexerATNFactory((LexerGrammar) g);
g.atn = factory.createATN();
AnalysisPipeline anal = new AnalysisPipeline(g);
anal.process();
CodeGenerator gen = new CodeGenerator(g);
ST outputFileST = gen.generateParser(false);
String output = outputFileST.render();
//System.out.println(output);
String b = "#" + actionName + "#";
int start = output.indexOf(b);
String e = "#end-" + actionName + "#";
int end = output.indexOf(e);
String snippet = output.substring(start + b.length(), end);
assertEquals(expected, snippet);
}
if (equeue.size() > 0) {
// System.err.println(equeue.toString());
}
}
protected void checkGrammarSemanticsError(ErrorQueue equeue,
GrammarSemanticsMessage expectedMessage)
throws Exception {
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
ANTLRMessage m = equeue.errors.get(i);
if (m.getErrorType() == expectedMessage.getErrorType()) {
foundMsg = m;
}
}
assertNotNull("no error; " + expectedMessage.getErrorType() + " expected", foundMsg);
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs()));
if (equeue.size() != 1) {
System.err.println(equeue);
}
}
protected void checkGrammarSemanticsWarning(ErrorQueue equeue,
GrammarSemanticsMessage expectedMessage)
throws Exception {
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.warnings.size(); i++) {
ANTLRMessage m = equeue.warnings.get(i);
if (m.getErrorType() == expectedMessage.getErrorType()) {
foundMsg = m;
}
}
assertNotNull("no error; " + expectedMessage.getErrorType() + " expected", foundMsg);
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs()));
if (equeue.size() != 1) {
System.err.println(equeue);
}
}
protected void checkError(ErrorQueue equeue,
ANTLRMessage expectedMessage)
throws Exception {
//System.out.println("errors="+equeue);
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
ANTLRMessage m = equeue.errors.get(i);
if (m.getErrorType() == expectedMessage.getErrorType()) {
foundMsg = m;
}
}
assertTrue("no error; " + expectedMessage.getErrorType() + " expected", !equeue.errors.isEmpty());
assertTrue("too many errors; " + equeue.errors, equeue.errors.size() <= 1);
assertNotNull("couldn't find expected error: " + expectedMessage.getErrorType(), foundMsg);
/*
* assertTrue("error is not a GrammarSemanticsMessage", foundMsg
* instanceof GrammarSemanticsMessage);
*/
assertArrayEquals(expectedMessage.getArgs(), foundMsg.getArgs());
}
public static class FilteringTokenStream extends CommonTokenStream {
public FilteringTokenStream(TokenSource src) {
super(src);
}
Set<Integer> hide = new HashSet<Integer>();
@Override
protected boolean sync(int i) {
if (!super.sync(i)) {
return false;
}
Token t = get(i);
if (hide.contains(t.getType())) {
((WritableToken) t).setChannel(Token.HIDDEN_CHANNEL);
}
return true;
}
public void setTokenTypeChannel(int ttype, int channel) {
hide.add(ttype);
}
}
protected void writeTestFile(String parserName,
String lexerName,
String parserStartRuleName,
@ -971,7 +428,7 @@ public class BaseDartTest implements RuntimeTestSupport {
outputFileST.add("parserName", parserName);
outputFileST.add("lexerName", lexerName);
outputFileST.add("parserStartRuleName", parserStartRuleName);
writeFile(tmpdir, "Test.dart", outputFileST.render());
writeFile(getTempDirPath(), "Test.dart", outputFileST.render());
}
protected void writeLexerTestFile(String lexerName, boolean showDFA) {
@ -995,191 +452,7 @@ public class BaseDartTest implements RuntimeTestSupport {
);
outputFileST.add("lexerName", lexerName);
writeFile(tmpdir, "Test.dart", outputFileST.render());
writeFile(getTempDirPath(), "Test.dart", outputFileST.render());
}
protected void eraseFiles() {
if (tmpdir == null) {
return;
}
File tmpdirF = new File(tmpdir);
String[] files = tmpdirF.list();
for (int i = 0; files != null && i < files.length; i++) {
new File(tmpdir + "/" + files[i]).delete();
}
}
@Override
public void eraseTempDir() {
File tmpdirF = new File(tmpdir);
if (tmpdirF.exists()) {
eraseFiles();
tmpdirF.delete();
}
}
public String getFirstLineOfException() {
if (this.stderrDuringParse == null) {
return null;
}
String[] lines = this.stderrDuringParse.split("\n");
String prefix = "Exception in thread \"main\" ";
return lines[0].substring(prefix.length(), lines[0].length());
}
/**
* When looking at a result set that consists of a Map/HashTable
* we cannot rely on the output order, as the hashing algorithm or other aspects
* of the implementation may be different on differnt JDKs or platforms. Hence
* we take the Map, convert the keys to a List, sort them and Stringify the Map, which is a
* bit of a hack, but guarantees that we get the same order on all systems. We assume that
* the keys are strings.
*
* @param m The Map that contains keys we wish to return in sorted order
* @return A string that represents all the keys in sorted order.
*/
public <K, V> String sortMapToString(Map<K, V> m) {
// Pass in crap, and get nothing back
//
if (m == null) {
return null;
}
System.out.println("Map toString looks like: " + m.toString());
// Sort the keys in the Map
//
TreeMap<K, V> nset = new TreeMap<K, V>(m);
System.out.println("Tree map looks like: " + nset.toString());
return nset.toString();
}
public List<String> realElements(List<String> elements) {
return elements.subList(Token.MIN_USER_TOKEN_TYPE, elements.size());
}
public void assertNotNullOrEmpty(String message, String text) {
assertNotNull(message, text);
assertFalse(message, text.isEmpty());
}
public void assertNotNullOrEmpty(String text) {
assertNotNull(text);
assertFalse(text.isEmpty());
}
public static class IntTokenStream implements TokenStream {
public IntegerList types;
int p = 0;
public IntTokenStream(IntegerList types) {
this.types = types;
}
@Override
public void consume() {
p++;
}
@Override
public int LA(int i) {
return LT(i).getType();
}
@Override
public int mark() {
return index();
}
@Override
public int index() {
return p;
}
@Override
public void release(int marker) {
seek(marker);
}
@Override
public void seek(int index) {
p = index;
}
@Override
public int size() {
return types.size();
}
@Override
public String getSourceName() {
return UNKNOWN_SOURCE_NAME;
}
@Override
public Token LT(int i) {
CommonToken t;
int rawIndex = p + i - 1;
if (rawIndex >= types.size()) t = new CommonToken(Token.EOF);
else t = new CommonToken(types.get(rawIndex));
t.setTokenIndex(rawIndex);
return t;
}
@Override
public Token get(int i) {
return new CommonToken(types.get(i));
}
@Override
public TokenSource getTokenSource() {
return null;
}
@Override
public String getText() {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Interval interval) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(RuleContext ctx) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Token start, Token stop) {
throw new UnsupportedOperationException("can't give strings");
}
}
/**
* Sort a list
*/
public <T extends Comparable<? super T>> List<T> sort(List<T> data) {
List<T> dup = new ArrayList<T>();
dup.addAll(data);
Collections.sort(dup);
return dup;
}
/**
* Return map sorted by key
*/
public <K extends Comparable<? super K>, V> LinkedHashMap<K, V> sort(Map<K, V> data) {
LinkedHashMap<K, V> dup = new LinkedHashMap<K, V>();
List<K> keys = new ArrayList<K>();
keys.addAll(data.keySet());
Collections.sort(keys);
for (K k : keys) {
dup.put(k, data.get(k));
}
return dup;
}
}

29
runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LexerExecDescriptors.java
View File

@ -1057,4 +1057,33 @@ public class LexerExecDescriptors {
return new Pair<>(grammarName, grammar);
}
}
/**
* This is a regression test for antlr/antlr4#2709 "PHP target generates
* invalid output when $ is used as part of the literal in lexer rule"
* https://github.com/antlr/antlr4/issues/2709
*/
public static class EscapeTargetStringLiteral extends BaseLexerTestDescriptor {
/**
[@0,0:-1='<EOF>',<-1>,1:0]
*/
@CommentHasStringValue
public String output;
public String errors = null;
public String startRule = "";
public String grammarName = "L";
/**
lexer grammar L;
ACTION_WITH_DOLLAR: '$ACTION';
*/
@CommentHasStringValue
public String grammar;
@Override
public boolean ignore(String targetName) {
return !targetName.equals("PHP");
}
}
}

693
runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java
View File

@ -5,92 +5,36 @@
*/
package org.antlr.v4.test.runtime.go;
import org.antlr.v4.Tool;
import org.antlr.v4.automata.ATNFactory;
import org.antlr.v4.automata.ATNPrinter;
import org.antlr.v4.automata.LexerATNFactory;
import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.IntStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenSource;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.WritableToken;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNDeserializer;
import org.antlr.v4.runtime.atn.ATNSerializer;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.semantics.SemanticPipeline;
import org.antlr.v4.test.runtime.ErrorQueue;
import org.antlr.v4.test.runtime.RuntimeTestSupport;
import org.antlr.v4.test.runtime.StreamVacuum;
import org.antlr.v4.tool.ANTLRMessage;
import org.antlr.v4.tool.DOTGenerator;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.GrammarSemanticsMessage;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.test.runtime.*;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupString;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertFalse;
import static junit.framework.TestCase.assertNotNull;
import static junit.framework.TestCase.assertTrue;
import static junit.framework.TestCase.*;
import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
import static org.junit.Assert.assertArrayEquals;
public class BaseGoTest implements RuntimeTestSupport {
public File overall_tmpdir = null;
public File tmpdir = null; // this is where the parser package is stored, typically inside the tmpdir
public class BaseGoTest extends BaseRuntimeTestSupport implements RuntimeTestSupport {
private static File tmpGopath = null;
private static final String GO_RUNTIME_IMPORT_PATH = "github.com/antlr/antlr4/runtime/Go/antlr"; // TODO: Change this before merging with upstream
/**
* If error during parser execution, store stderr here; can't return stdout
* and stderr. This doesn't trap errors from running antlr.
*/
protected String stderrDuringParse;
private File parserTempDir; // "parser" with tempDir
/** Errors found while running antlr */
protected StringBuilder antlrToolErrors;
@Override
protected String getPropertyPrefix() {
return "antlr4-go";
}
/**
* Copies all files from go runtime to a temporary folder that is inside a valid GOPATH project structure.
*/
public static void groupSetUp() throws Exception {
tmpGopath = new File(System.getProperty("java.io.tmpdir"), "antlr-goruntime-tmpgopath-"
+ Long.toHexString(System.currentTimeMillis()));
tmpGopath = new File(System.getProperty("java.io.tmpdir"), "antlr-goruntime-tmpgopath-" + Long.toHexString(System.currentTimeMillis()));
ArrayList<String> pathsegments = new ArrayList<String>();
pathsegments.add("src");
@ -110,39 +54,12 @@ public class BaseGoTest implements RuntimeTestSupport {
}
for (File runtimeFile : runtimeFiles) {
File dest = new File(tmpPackageDir, runtimeFile.getName());
copyFile(runtimeFile, dest);
RuntimeTestUtils.copyFile(runtimeFile, dest);
}
cacheGoRuntime(tmpPackageDir);
}
@Override
public void testTearDown() throws Exception {
}
@Override
public String getTmpDir() {
return tmpdir.getPath();
}
@Override
public String getStdout() {
return null;
}
@Override
public String getParseErrors() {
return stderrDuringParse;
}
@Override
public String getANTLRToolErrors() {
if ( antlrToolErrors.length()==0 ) {
return null;
}
return antlrToolErrors.toString();
}
public static void groupTearDown() throws Exception {
eraseDirectory(tmpGopath);
}
@ -163,139 +80,22 @@ public class BaseGoTest implements RuntimeTestSupport {
}
}
private static void copyFile(File source, File dest) throws IOException {
InputStream is = new FileInputStream(source);
OutputStream os = new FileOutputStream(dest);
byte[] buf = new byte[4 << 10];
int l;
while ((l = is.read(buf)) > -1) {
os.write(buf, 0, l);
}
is.close();
os.close();
}
public void testSetUp() throws Exception {
// new output dir for each test
String prop = System.getProperty("antlr-go-test-dir");
if (prop != null && prop.length() > 0) {
overall_tmpdir = new File(prop);
}
else {
String threadName = Thread.currentThread().getName();
overall_tmpdir = new File(System.getProperty("java.io.tmpdir"),
getClass().getSimpleName()+"-"+threadName+"-"+System.currentTimeMillis());
}
if ( overall_tmpdir.exists())
this.eraseDirectory(overall_tmpdir);
tmpdir = new File(overall_tmpdir, "parser");
if ( tmpdir.exists()) {
this.eraseDirectory(tmpdir);
}
antlrToolErrors = new StringBuilder();
eraseParserTempDir();
super.testSetUp();
parserTempDir = new File(getTempTestDir(), "parser");
}
protected org.antlr.v4.Tool newTool(String[] args) {
return new Tool(args);
@Override
public File getTempParserDir() {
return parserTempDir;
}
protected Tool newTool() {
return new Tool(new String[]{"-o", tmpdir.getPath()});
}
protected ATN createATN(Grammar g, boolean useSerializer) {
if (g.atn == null) {
semanticProcess(g);
assertEquals(0, g.tool.getNumErrors());
ParserATNFactory f;
if (g.isLexer()) {
f = new LexerATNFactory((LexerGrammar) g);
}
else {
f = new ParserATNFactory(g);
}
g.atn = f.createATN();
assertEquals(0, g.tool.getNumErrors());
private void eraseParserTempDir() {
if(parserTempDir != null) {
eraseDirectory(parserTempDir);
parserTempDir = null;
}
ATN atn = g.atn;
if (useSerializer) {
char[] serialized = ATNSerializer.getSerializedAsChars(atn);
return new ATNDeserializer().deserialize(serialized);
}
return atn;
}
protected void semanticProcess(Grammar g) {
if (g.ast != null && !g.ast.hasErrors) {
System.out.println(g.ast.toStringTree());
Tool antlr = new Tool();
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
if (g.getImportedGrammars() != null) { // process imported grammars
// (if any)
for (Grammar imp : g.getImportedGrammars()) {
antlr.processNonCombinedGrammar(imp, false);
}
}
}
}
IntegerList getTypesFromString(Grammar g, String expecting) {
IntegerList expectingTokenTypes = new IntegerList();
if (expecting != null && !expecting.trim().isEmpty()) {
for (String tname : expecting.replace(" ", "").split(",")) {
int ttype = g.getTokenType(tname);
expectingTokenTypes.add(ttype);
}
}
return expectingTokenTypes;
}
public IntegerList getTokenTypesViaATN(String input,
LexerATNSimulator lexerATN) {
ANTLRInputStream in = new ANTLRInputStream(input);
IntegerList tokenTypes = new IntegerList();
int ttype;
do {
ttype = lexerATN.match(in, Lexer.DEFAULT_MODE);
tokenTypes.add(ttype);
} while (ttype != Token.EOF);
return tokenTypes;
}
public List<String> getTokenTypes(LexerGrammar lg, ATN atn, CharStream input) {
LexerATNSimulator interp = new LexerATNSimulator(atn,
new DFA[] { new DFA(
atn.modeToStartState.get(Lexer.DEFAULT_MODE)) }, null);
List<String> tokenTypes = new ArrayList<String>();
int ttype;
boolean hitEOF = false;
do {
if (hitEOF) {
tokenTypes.add("EOF");
break;
}
int t = input.LA(1);
ttype = interp.match(input, Lexer.DEFAULT_MODE);
if (ttype == Token.EOF) {
tokenTypes.add("EOF");
}
else {
tokenTypes.add(lg.typeToTokenList.get(ttype));
}
if (t == IntStream.EOF) {
hitEOF = true;
}
} while (ttype != Token.EOF);
return tokenTypes;
}
protected String execLexer(String grammarFileName, String grammarStr,
@ -309,25 +109,10 @@ public class BaseGoTest implements RuntimeTestSupport {
boolean success = rawGenerateAndBuildRecognizer(grammarFileName,
grammarStr, null, lexerName, "-no-listener");
assertTrue(success);
writeFile(overall_tmpdir.toString(), "input", input);
writeFile(getTempDirPath(), "input", input);
writeLexerTestFile(lexerName, showDFA);
String output = execModule("Test.go");
return output;
return execModule("Test.go");
}
//
// public String execParser(String grammarFileName, String grammarStr,
// String parserName, String lexerName, String listenerName,
// String visitorName, String startRuleName, String input,
// boolean debug)
// {
// boolean success = rawGenerateAndBuildRecognizer(grammarFileName,
// grammarStr, parserName, lexerName, "-visitor");
// assertTrue(success);
// writeFile(overall_tmpdir, "input", input);
// rawBuildRecognizerTestFile(parserName, lexerName, listenerName,
// visitorName, startRuleName, debug);
// return execRecognizer();
// }
@Override
public String execParser(String grammarFileName, String grammarStr,
@ -338,7 +123,7 @@ public class BaseGoTest implements RuntimeTestSupport {
boolean success = rawGenerateAndBuildRecognizer(grammarFileName,
grammarStr, parserName, lexerName, "-visitor");
assertTrue(success);
writeFile(overall_tmpdir.toString(), "input", input);
writeFile(getTempDirPath(), "input", input);
rawBuildRecognizerTestFile(parserName, lexerName, listenerName,
visitorName, startRuleName, showDiagnosticErrors);
return execRecognizer();
@ -356,7 +141,7 @@ public class BaseGoTest implements RuntimeTestSupport {
protected boolean rawGenerateAndBuildRecognizer(String grammarFileName,
String grammarStr, String parserName, String lexerName,
boolean defaultListener, String... extraOptions) {
ErrorQueue equeue = antlrOnString(getTmpDir(), "Go", grammarFileName, grammarStr,
ErrorQueue equeue = antlrOnString(getTempParserDirPath(), "Go", grammarFileName, grammarStr,
defaultListener, extraOptions);
if (!equeue.errors.isEmpty()) {
return false;
@ -367,7 +152,7 @@ public class BaseGoTest implements RuntimeTestSupport {
protected void rawBuildRecognizerTestFile(String parserName,
String lexerName, String listenerName, String visitorName,
String parserStartRuleName, boolean debug) {
this.stderrDuringParse = null;
setParseErrors(null);
if (parserName == null) {
writeLexerTestFile(lexerName, false);
}
@ -383,12 +168,12 @@ public class BaseGoTest implements RuntimeTestSupport {
public String execModule(String fileName) {
String goExecutable = locateGo();
String modulePath = new File(overall_tmpdir, fileName).getAbsolutePath();
String inputPath = new File(overall_tmpdir, "input").getAbsolutePath();
String modulePath = new File(getTempTestDir(), fileName).getAbsolutePath();
String inputPath = new File(getTempTestDir(), "input").getAbsolutePath();
try {
ProcessBuilder builder = new ProcessBuilder(goExecutable, "run", modulePath, inputPath);
builder.environment().put("GOPATH", tmpGopath.getPath());
builder.directory(overall_tmpdir);
builder.directory(getTempTestDir());
Process process = builder.start();
StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
@ -402,7 +187,7 @@ public class BaseGoTest implements RuntimeTestSupport {
output = null;
}
if (stderrVacuum.toString().length() > 0) {
this.stderrDuringParse = stderrVacuum.toString();
setParseErrors(stderrVacuum.toString());
}
return output;
}
@ -469,203 +254,6 @@ public class BaseGoTest implements RuntimeTestSupport {
return runtimeDir;
}
// void ambig(List<Message> msgs, int[] expectedAmbigAlts, String
// expectedAmbigInput)
// throws Exception
// {
// ambig(msgs, 0, expectedAmbigAlts, expectedAmbigInput);
// }
// void ambig(List<Message> msgs, int i, int[] expectedAmbigAlts, String
// expectedAmbigInput)
// throws Exception
// {
// List<Message> amsgs = getMessagesOfType(msgs, AmbiguityMessage.class);
// AmbiguityMessage a = (AmbiguityMessage)amsgs.get(i);
// if ( a==null ) assertNull(expectedAmbigAlts);
// else {
// assertEquals(a.conflictingAlts.toString(),
// Arrays.toString(expectedAmbigAlts));
// }
// assertEquals(expectedAmbigInput, a.input);
// }
// void unreachable(List<Message> msgs, int[] expectedUnreachableAlts)
// throws Exception
// {
// unreachable(msgs, 0, expectedUnreachableAlts);
// }
// void unreachable(List<Message> msgs, int i, int[]
// expectedUnreachableAlts)
// throws Exception
// {
// List<Message> amsgs = getMessagesOfType(msgs,
// UnreachableAltsMessage.class);
// UnreachableAltsMessage u = (UnreachableAltsMessage)amsgs.get(i);
// if ( u==null ) assertNull(expectedUnreachableAlts);
// else {
// assertEquals(u.conflictingAlts.toString(),
// Arrays.toString(expectedUnreachableAlts));
// }
// }
List<ANTLRMessage> getMessagesOfType(List<ANTLRMessage> msgs,
Class<? extends ANTLRMessage> c) {
List<ANTLRMessage> filtered = new ArrayList<ANTLRMessage>();
for (ANTLRMessage m : msgs) {
if (m.getClass() == c)
filtered.add(m);
}
return filtered;
}
void checkRuleATN(Grammar g, String ruleName, String expecting) {
ParserATNFactory f = new ParserATNFactory(g);
ATN atn = f.createATN();
DOTGenerator dot = new DOTGenerator(g);
System.out
.println(dot.getDOT(atn.ruleToStartState[g.getRule(ruleName).index]));
Rule r = g.getRule(ruleName);
ATNState startState = atn.ruleToStartState[r.index];
ATNPrinter serializer = new ATNPrinter(g, startState);
String result = serializer.asString();
// System.out.print(result);
assertEquals(expecting, result);
}
public void testActions(String templates, String actionName, String action,
String expected) throws org.antlr.runtime.RecognitionException {
int lp = templates.indexOf('(');
String name = templates.substring(0, lp);
STGroup group = new STGroupString(templates);
ST st = group.getInstanceOf(name);
st.add(actionName, action);
String grammar = st.render();
ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(grammar, equeue);
if (g.ast != null && !g.ast.hasErrors) {
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
ATNFactory factory = new ParserATNFactory(g);
if (g.isLexer())
factory = new LexerATNFactory((LexerGrammar) g);
g.atn = factory.createATN();
CodeGenerator gen = new CodeGenerator(g);
ST outputFileST = gen.generateParser();
String output = outputFileST.render();
// System.out.println(output);
String b = "#" + actionName + "#";
int start = output.indexOf(b);
String e = "#end-" + actionName + "#";
int end = output.indexOf(e);
String snippet = output.substring(start + b.length(), end);
assertEquals(expected, snippet);
}
if (equeue.size() > 0) {
System.err.println(equeue.toString());
}
}
protected void checkGrammarSemanticsError(ErrorQueue equeue,
GrammarSemanticsMessage expectedMessage) throws Exception {
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
ANTLRMessage m = equeue.errors.get(i);
if (m.getErrorType() == expectedMessage.getErrorType()) {
foundMsg = m;
}
}
assertNotNull("no error; " + expectedMessage.getErrorType()
+ " expected", foundMsg);
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
assertEquals(Arrays.toString(expectedMessage.getArgs()),
Arrays.toString(foundMsg.getArgs()));
if (equeue.size() != 1) {
System.err.println(equeue);
}
}
protected void checkGrammarSemanticsWarning(ErrorQueue equeue,
GrammarSemanticsMessage expectedMessage) throws Exception {
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.warnings.size(); i++) {
ANTLRMessage m = equeue.warnings.get(i);
if (m.getErrorType() == expectedMessage.getErrorType()) {
foundMsg = m;
}
}
assertNotNull("no error; " + expectedMessage.getErrorType()
+ " expected", foundMsg);
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
assertEquals(Arrays.toString(expectedMessage.getArgs()),
Arrays.toString(foundMsg.getArgs()));
if (equeue.size() != 1) {
System.err.println(equeue);
}
}
protected void checkError(ErrorQueue equeue, ANTLRMessage expectedMessage)
throws Exception {
// System.out.println("errors="+equeue);
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
ANTLRMessage m = equeue.errors.get(i);
if (m.getErrorType() == expectedMessage.getErrorType()) {
foundMsg = m;
}
}
assertTrue("no error; " + expectedMessage.getErrorType() + " expected",
!equeue.errors.isEmpty());
assertTrue("too many errors; " + equeue.errors,
equeue.errors.size() <= 1);
assertNotNull(
"couldn't find expected error: "
+ expectedMessage.getErrorType(), foundMsg);
/*
* assertTrue("error is not a GrammarSemanticsMessage", foundMsg
* instanceof GrammarSemanticsMessage);
*/
assertArrayEquals(expectedMessage.getArgs(), foundMsg.getArgs());
}
public static class FilteringTokenStream extends CommonTokenStream {
public FilteringTokenStream(TokenSource src) {
super(src);
}
Set<Integer> hide = new HashSet<Integer>();
@Override
protected boolean sync(int i) {
if (!super.sync(i)) {
return false;
}
Token t = get(i);
if (hide.contains(t.getType())) {
((WritableToken) t).setChannel(Token.HIDDEN_CHANNEL);
}
return true;
}
public void setTokenTypeChannel(int ttype, int channel) {
hide.add(ttype);
}
}
protected void mkdir(File dir) {
dir.mkdirs();
}
protected void writeParserTestFile(String parserName, String lexerName,
String listenerName, String visitorName,
String parserStartRuleName, boolean debug) {
@ -723,7 +311,7 @@ public class BaseGoTest implements RuntimeTestSupport {
outputFileST.add("listenerName", listenerName);
outputFileST.add("visitorName", visitorName);
outputFileST.add("parserStartRuleName", parserStartRuleName.substring(0, 1).toUpperCase() + parserStartRuleName.substring(1) );
writeFile(overall_tmpdir.toString(), "Test.go", outputFileST.render());
writeFile(getTempDirPath(), "Test.go", outputFileST.render());
}
@ -755,222 +343,7 @@ public class BaseGoTest implements RuntimeTestSupport {
+ "}\n"
+ "\n");
outputFileST.add("lexerName", lexerName);
writeFile(overall_tmpdir.toString(), "Test.go", outputFileST.render());
writeFile(getTempDirPath(), "Test.go", outputFileST.render());
}
public void writeRecognizer(String parserName, String lexerName,
String listenerName, String visitorName,
String parserStartRuleName, boolean debug) {
if (parserName == null) {
writeLexerTestFile(lexerName, debug);
}
else {
writeParserTestFile(parserName, lexerName, listenerName,
visitorName, parserStartRuleName, debug);
}
}
protected void eraseFilesEndingWith(final String filesEndingWith) {
File[] files = overall_tmpdir.listFiles(new FileFilter() {
@Override
public boolean accept(File pathname) {
return pathname.getName().endsWith(filesEndingWith);
}
});
for (File file : files) {
file.delete();
}
}
protected static void eraseDirectory(File dir) {
File[] files = dir.listFiles();
if (files != null) {
for (File file : files) {
if (file.isDirectory()) {
eraseDirectory(file);
}
else {
file.delete();
}
}
}
dir.delete();
}
public void eraseTempDir() {
boolean doErase = true;
String propName = "antlr-go-erase-test-dir";
String prop = System.getProperty(propName);
if (prop != null && prop.length() > 0)
doErase = Boolean.getBoolean(prop);
if (doErase) {
if ( overall_tmpdir.exists()) {
eraseDirectory(overall_tmpdir);
}
}
}
public String getFirstLineOfException() {
if (this.stderrDuringParse == null) {
return null;
}
String[] lines = this.stderrDuringParse.split("\n");
String prefix = "Exception in thread \"main\" ";
return lines[0].substring(prefix.length(), lines[0].length());
}
/**
* When looking at a result set that consists of a Map/HashTable we cannot
* rely on the output order, as the hashing algorithm or other aspects of
* the implementation may be different on differnt JDKs or platforms. Hence
* we take the Map, convert the keys to a List, sort them and Stringify the
* Map, which is a bit of a hack, but guarantees that we get the same order
* on all systems. We assume that the keys are strings.
*
* @param m
* The Map that contains keys we wish to return in sorted order
* @return A string that represents all the keys in sorted order.
*/
public <K, V> String sortMapToString(Map<K, V> m) {
// Pass in crap, and get nothing back
//
if (m == null) {
return null;
}
System.out.println("Map toString looks like: " + m.toString());
// Sort the keys in the Map
//
TreeMap<K, V> nset = new TreeMap<K, V>(m);
System.out.println("Tree map looks like: " + nset.toString());
return nset.toString();
}
public List<String> realElements(List<String> elements) {
return elements.subList(Token.MIN_USER_TOKEN_TYPE, elements.size());
}
public void assertNotNullOrEmpty(String message, String text) {
assertNotNull(message, text);
assertFalse(message, text.isEmpty());
}
public void assertNotNullOrEmpty(String text) {
assertNotNull(text);
assertFalse(text.isEmpty());
}
public static class IntTokenStream implements TokenStream {
IntegerList types;
int p = 0;
public IntTokenStream(IntegerList types) {
this.types = types;
}
@Override
public void consume() {
p++;
}
@Override
public int LA(int i) {
return LT(i).getType();
}
@Override
public int mark() {
return index();
}
@Override
public int index() {
return p;
}
@Override
public void release(int marker) {
seek(marker);
}
@Override
public void seek(int index) {
p = index;
}
@Override
public int size() {
return types.size();
}
@Override
public String getSourceName() {
return null;
}
@Override
public Token LT(int i) {
CommonToken t;
int rawIndex = p + i - 1;
if (rawIndex >= types.size())
t = new CommonToken(Token.EOF);
else
t = new CommonToken(types.get(rawIndex));
t.setTokenIndex(rawIndex);
return t;
}
@Override
public Token get(int i) {
return new org.antlr.v4.runtime.CommonToken(types.get(i));
}
@Override
public TokenSource getTokenSource() {
return null;
}
@Override
public String getText() {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Interval interval) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(RuleContext ctx) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Token start, Token stop) {
throw new UnsupportedOperationException("can't give strings");
}
}
/** Sort a list */
public <T extends Comparable<? super T>> List<T> sort(List<T> data) {
List<T> dup = new ArrayList<T>();
dup.addAll(data);
Collections.sort(dup);
return dup;
}
/** Return map sorted by key */
public <K extends Comparable<? super K>, V> LinkedHashMap<K, V> sort(
Map<K, V> data) {
LinkedHashMap<K, V> dup = new LinkedHashMap<K, V>();
List<K> keys = new ArrayList<K>();
keys.addAll(data.keySet());
Collections.sort(keys);
for (K k : keys) {
dup.put(k, data.get(k));
}
return dup;
}
}

647
runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java
View File

@ -5,7 +5,6 @@
*/
package org.antlr.v4.test.runtime.java;
import org.antlr.v4.Tool;
import org.antlr.v4.analysis.AnalysisPipeline;
import org.antlr.v4.automata.ATNFactory;
import org.antlr.v4.automata.ATNPrinter;
@ -14,32 +13,16 @@ import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.IntStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenSource;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.WritableToken;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNDeserializer;
import org.antlr.v4.runtime.atn.ATNSerializer;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.misc.Pair;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.semantics.SemanticPipeline;
import org.antlr.v4.test.runtime.BaseRuntimeTest;
import org.antlr.v4.test.runtime.ErrorQueue;
import org.antlr.v4.test.runtime.RuntimeTestSupport;
import org.antlr.v4.test.runtime.StreamVacuum;
import org.antlr.v4.test.runtime.*;
import org.antlr.v4.tool.ANTLRMessage;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.GrammarSemanticsMessage;
@ -68,13 +51,9 @@ import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertFalse;
@ -83,9 +62,7 @@ import static junit.framework.TestCase.assertTrue;
import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
import static org.junit.Assert.assertArrayEquals;
public class BaseJavaTest implements RuntimeTestSupport {
public static final String newline = System.getProperty("line.separator");
public static final String pathSep = System.getProperty("path.separator");
public class BaseJavaTest extends BaseRuntimeTestSupport implements RuntimeTestSupport {
/**
* When the {@code antlr.testinprocess} runtime property is set to
@ -103,289 +80,14 @@ public class BaseJavaTest implements RuntimeTestSupport {
*/
public static final boolean TEST_IN_SAME_PROCESS = Boolean.parseBoolean(System.getProperty("antlr.testinprocess"));
/**
* When the {@code antlr.preserve-test-dir} runtime property is set to
* {@code true}, the temporary directories created by the test run will not
* be removed at the end of the test run, even for tests that completed
* successfully.
* <p>
* <p>
* The default behavior (used in all other cases) is removing the temporary
* directories for all tests which completed successfully, and preserving
* the directories for tests which failed.</p>
*/
public static final boolean PRESERVE_TEST_DIR = true; //Boolean.parseBoolean(System.getProperty("antlr.preserve-test-dir"));
/**
* The base test directory is the directory where generated files get placed
* during unit test execution.
* <p>
* <p>
* The default value for this property is the {@code java.io.tmpdir} system
* property, and can be overridden by setting the
* {@code antlr.java-test-dir} property to a custom location. Note that the
* {@code antlr.java-test-dir} property directly affects the
* {@link #CREATE_PER_TEST_DIRECTORIES} value as well.</p>
*/
public static final String BASE_TEST_DIR;
/**
* When {@code true}, a temporary directory will be created for each test
* executed during the test run.
* <p>
* <p>
* This value is {@code true} when the {@code antlr.java-test-dir} system
* property is set, and otherwise {@code false}.</p>
*/
public static final boolean CREATE_PER_TEST_DIRECTORIES;
static {
String baseTestDir = System.getProperty("antlr.java-test-dir");
boolean perTestDirectories = false;
if ( baseTestDir==null || baseTestDir.isEmpty() ) {
baseTestDir = System.getProperty("java.io.tmpdir");
perTestDirectories = true;
}
if ( !new File(baseTestDir).isDirectory() ) {
throw new UnsupportedOperationException("The specified base test directory does not exist: "+baseTestDir);
}
BASE_TEST_DIR = baseTestDir;
CREATE_PER_TEST_DIRECTORIES = perTestDirectories;
}
/**
* Build up the full classpath we need, including the surefire path (if present)
*/
public static final String CLASSPATH = System.getProperty("java.class.path");
public String tmpdir = null;
/**
* If error during parser execution, store stderr here; can't return
* stdout and stderr. This doesn't trap errors from running antlr.
*/
protected String stderrDuringParse;
/**
* Errors found while running antlr
*/
protected StringBuilder antlrToolErrors;
@Override
public void testSetUp() throws Exception {
// STGroup.verbose = true;
// System.err.println("testSetUp "+Thread.currentThread().getName());
if ( CREATE_PER_TEST_DIRECTORIES ) {
// new output dir for each test
String threadName = Thread.currentThread().getName();
String testDirectory = getClass().getSimpleName()+"-"+threadName+"-"+System.nanoTime();
tmpdir = new File(BASE_TEST_DIR, testDirectory).getAbsolutePath();
}
else {
tmpdir = new File(BASE_TEST_DIR).getAbsolutePath();
if ( !PRESERVE_TEST_DIR && new File(tmpdir).exists() ) {
eraseFiles();
}
}
antlrToolErrors = new StringBuilder();
}
@Override
public void testTearDown() throws Exception {
}
@Override
public String getTmpDir() {
return tmpdir;
}
@Override
public String getStdout() {
return null;
}
@Override
public String getParseErrors() {
return stderrDuringParse;
}
@Override
public String getANTLRToolErrors() {
if ( antlrToolErrors.length()==0 ) {
return null;
}
return antlrToolErrors.toString();
}
protected org.antlr.v4.Tool newTool(String[] args) {
Tool tool = new Tool(args);
return tool;
}
protected ATN createATN(Grammar g, boolean useSerializer) {
if ( g.atn==null ) {
semanticProcess(g);
assertEquals(0, g.tool.getNumErrors());
ParserATNFactory f;
if ( g.isLexer() ) {
f = new LexerATNFactory((LexerGrammar) g);
}
else {
f = new ParserATNFactory(g);
}
g.atn = f.createATN();
assertEquals(0, g.tool.getNumErrors());
}
ATN atn = g.atn;
if ( useSerializer ) {
char[] serialized = ATNSerializer.getSerializedAsChars(atn);
return new ATNDeserializer().deserialize(serialized);
}
return atn;
}
protected void semanticProcess(Grammar g) {
if ( g.ast!=null && !g.ast.hasErrors ) {
// System.out.println(g.ast.toStringTree());
Tool antlr = new Tool();
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any)
for (Grammar imp : g.getImportedGrammars()) {
antlr.processNonCombinedGrammar(imp, false);
}
}
}
}
public DFA createDFA(Grammar g, DecisionState s) {
// PredictionDFAFactory conv = new PredictionDFAFactory(g, s);
// DFA dfa = conv.createDFA();
// conv.issueAmbiguityWarnings();
// System.out.print("DFA="+dfa);
// return dfa;
return null;
}
// public void minimizeDFA(DFA dfa) {
// DFAMinimizer dmin = new DFAMinimizer(dfa);
// dfa.minimized = dmin.minimize();
// }
IntegerList getTypesFromString(Grammar g, String expecting) {
IntegerList expectingTokenTypes = new IntegerList();
if ( expecting!=null && !expecting.trim().isEmpty() ) {
for (String tname : expecting.replace(" ", "").split(",")) {
int ttype = g.getTokenType(tname);
expectingTokenTypes.add(ttype);
}
}
return expectingTokenTypes;
}
public IntegerList getTokenTypesViaATN(String input, LexerATNSimulator lexerATN) {
ANTLRInputStream in = new ANTLRInputStream(input);
IntegerList tokenTypes = new IntegerList();
int ttype;
do {
ttype = lexerATN.match(in, Lexer.DEFAULT_MODE);
tokenTypes.add(ttype);
} while ( ttype!=Token.EOF );
return tokenTypes;
}
public List<String> getTokenTypes(LexerGrammar lg,
ATN atn,
CharStream input) {
LexerATNSimulator interp = new LexerATNSimulator(atn, new DFA[]{new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE))}, null);
List<String> tokenTypes = new ArrayList<String>();
int ttype;
boolean hitEOF = false;
do {
if ( hitEOF ) {
tokenTypes.add("EOF");
break;
}
int t = input.LA(1);
ttype = interp.match(input, Lexer.DEFAULT_MODE);
if ( ttype==Token.EOF ) {
tokenTypes.add("EOF");
}
else {
tokenTypes.add(lg.typeToTokenList.get(ttype));
}
if ( t==IntStream.EOF ) {
hitEOF = true;
}
} while ( ttype!=Token.EOF );
return tokenTypes;
}
List<ANTLRMessage> checkRuleDFA(String gtext, String ruleName, String expecting)
throws Exception {
ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(gtext, equeue);
ATN atn = createATN(g, false);
ATNState s = atn.ruleToStartState[g.getRule(ruleName).index];
if ( s==null ) {
System.err.println("no such rule: "+ruleName);
return null;
}
ATNState t = s.transition(0).target;
if ( !(t instanceof DecisionState) ) {
System.out.println(ruleName+" has no decision");
return null;
}
DecisionState blk = (DecisionState) t;
checkRuleDFA(g, blk, expecting);
return equeue.all;
}
List<ANTLRMessage> checkRuleDFA(String gtext, int decision, String expecting)
throws Exception {
ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(gtext, equeue);
ATN atn = createATN(g, false);
DecisionState blk = atn.decisionToState.get(decision);
checkRuleDFA(g, blk, expecting);
return equeue.all;
}
void checkRuleDFA(Grammar g, DecisionState blk, String expecting)
throws Exception {
DFA dfa = createDFA(g, blk);
String result = null;
if ( dfa!=null ) result = dfa.toString();
assertEquals(expecting, result);
}
List<ANTLRMessage> checkLexerDFA(String gtext, String expecting)
throws Exception {
return checkLexerDFA(gtext, LexerGrammar.DEFAULT_MODE_NAME, expecting);
}
List<ANTLRMessage> checkLexerDFA(String gtext, String modeName, String expecting)
throws Exception {
ErrorQueue equeue = new ErrorQueue();
LexerGrammar g = new LexerGrammar(gtext, equeue);
g.atn = createATN(g, false);
// LexerATNToDFAConverter conv = new LexerATNToDFAConverter(g);
// DFA dfa = conv.createDFA(modeName);
// g.setLookaheadDFA(0, dfa); // only one decision to worry about
//
// String result = null;
// if ( dfa!=null ) result = dfa.toString();
// assertEquals(expecting, result);
//
// return equeue.all;
return null;
protected String getPropertyPrefix() {
return "antrl4-java";
}
protected String load(String fileName, String encoding)
@ -420,7 +122,7 @@ public class BaseJavaTest implements RuntimeTestSupport {
protected boolean compile(String... fileNames) {
List<File> files = new ArrayList<File>();
for (String fileName : fileNames) {
File f = new File(tmpdir, fileName);
File f = new File(getTempTestDir(), fileName);
files.add(f);
}
@ -435,7 +137,7 @@ public class BaseJavaTest implements RuntimeTestSupport {
fileManager.getJavaFileObjectsFromFiles(files);
Iterable<String> compileOptions =
Arrays.asList("-g", "-source", "1.6", "-target", "1.6", "-implicit:class", "-Xlint:-options", "-d", tmpdir, "-cp", tmpdir+pathSep+CLASSPATH);
Arrays.asList("-g", "-source", "1.6", "-target", "1.6", "-implicit:class", "-Xlint:-options", "-d", getTempDirPath(), "-cp", getTempDirPath() + PATH_SEP + CLASSPATH);
JavaCompiler.CompilationTask task =
compiler.getTask(null, fileManager, null, compileOptions, null,
@ -469,11 +171,10 @@ public class BaseJavaTest implements RuntimeTestSupport {
null,
lexerName);
assertTrue(success);
writeFile(tmpdir, "input", input);
writeFile(getTempDirPath(), "input", input);
writeLexerTestFile(lexerName, showDFA);
compile("Test.java");
String output = execClass("Test");
return output;
return execClass("Test");
}
public ParseTree execParser(String startRuleName, String input,
@ -523,7 +224,7 @@ public class BaseJavaTest implements RuntimeTestSupport {
public Class<?> loadClassFromTempDir(String name) throws Exception {
ClassLoader loader =
new URLClassLoader(new URL[]{new File(tmpdir).toURI().toURL()},
new URLClassLoader(new URL[]{getTempTestDir().toURI().toURL()},
ClassLoader.getSystemClassLoader());
return loader.loadClass(name);
}
@ -571,7 +272,7 @@ public class BaseJavaTest implements RuntimeTestSupport {
lexerName,
"-visitor");
assertTrue(success);
writeFile(tmpdir, "input", input);
writeFile(getTempDirPath(), "input", input);
return rawExecRecognizer(parserName,
lexerName,
startRuleName,
@ -598,7 +299,7 @@ public class BaseJavaTest implements RuntimeTestSupport {
String... extraOptions)
{
ErrorQueue equeue =
BaseRuntimeTest.antlrOnString(getTmpDir(), "Java", grammarFileName, grammarStr, defaultListener, extraOptions);
BaseRuntimeTest.antlrOnString(getTempDirPath(), "Java", grammarFileName, grammarStr, defaultListener, extraOptions);
if (!equeue.errors.isEmpty()) {
return false;
}
@ -620,8 +321,7 @@ public class BaseJavaTest implements RuntimeTestSupport {
files.add(grammarName+"BaseVisitor.java");
}
}
boolean allIsWell = compile(files.toArray(new String[files.size()]));
return allIsWell;
return compile(files.toArray(new String[0]));
}
protected String rawExecRecognizer(String parserName,
@ -630,7 +330,7 @@ public class BaseJavaTest implements RuntimeTestSupport {
boolean debug,
boolean profile)
{
this.stderrDuringParse = null;
setParseErrors(null);
if ( parserName==null ) {
writeLexerTestFile(lexerName, false);
}
@ -653,7 +353,7 @@ public class BaseJavaTest implements RuntimeTestSupport {
public String execClass(String className) {
if (TEST_IN_SAME_PROCESS) {
try {
ClassLoader loader = new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() }, ClassLoader.getSystemClassLoader());
ClassLoader loader = new URLClassLoader(new URL[] { getTempTestDir().toURI().toURL() }, ClassLoader.getSystemClassLoader());
final Class<?> mainClass = (Class<?>)loader.loadClass(className);
final Method mainMethod = mainClass.getDeclaredMethod("main", String[].class);
PipedInputStream stdoutIn = new PipedInputStream();
@ -671,7 +371,7 @@ public class BaseJavaTest implements RuntimeTestSupport {
System.setErr(new PrintStream(stderrOut));
stdoutVacuum.start();
stderrVacuum.start();
mainMethod.invoke(null, (Object)new String[] { new File(tmpdir, "input").getAbsolutePath() });
mainMethod.invoke(null, (Object)new String[] { new File(getTempTestDir(), "input").getAbsolutePath() });
}
finally {
System.setErr(originalErr);
@ -690,7 +390,7 @@ public class BaseJavaTest implements RuntimeTestSupport {
output = null;
}
if ( stderrVacuum.toString().length()>0 ) {
this.stderrDuringParse = stderrVacuum.toString();
setParseErrors(stderrVacuum.toString());
}
return output;
}
@ -701,14 +401,14 @@ public class BaseJavaTest implements RuntimeTestSupport {
try {
String[] args = new String[] {
"java", "-classpath", tmpdir+pathSep+CLASSPATH,
"java", "-classpath", getTempDirPath() + PATH_SEP + CLASSPATH,
"-Dfile.encoding=UTF-8",
className, new File(tmpdir, "input").getAbsolutePath()
className, new File(getTempTestDir(), "input").getAbsolutePath()
};
// String cmdLine = Utils.join(args, " ");
// System.err.println("execParser: "+cmdLine);
Process process =
Runtime.getRuntime().exec(args, null, new File(tmpdir));
Runtime.getRuntime().exec(args, null, getTempTestDir());
StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
stdoutVacuum.start();
@ -721,7 +421,7 @@ public class BaseJavaTest implements RuntimeTestSupport {
output = null;
}
if ( stderrVacuum.toString().length()>0 ) {
this.stderrDuringParse = stderrVacuum.toString();
setParseErrors(stderrVacuum.toString());
}
return output;
}
@ -732,49 +432,6 @@ public class BaseJavaTest implements RuntimeTestSupport {
return null;
}
// void ambig(List<Message> msgs, int[] expectedAmbigAlts, String expectedAmbigInput)
// throws Exception
// {
// ambig(msgs, 0, expectedAmbigAlts, expectedAmbigInput);
// }
// void ambig(List<Message> msgs, int i, int[] expectedAmbigAlts, String expectedAmbigInput)
// throws Exception
// {
// List<Message> amsgs = getMessagesOfType(msgs, AmbiguityMessage.class);
// AmbiguityMessage a = (AmbiguityMessage)amsgs.get(i);
// if ( a==null ) assertNull(expectedAmbigAlts);
// else {
// assertEquals(a.conflictingAlts.toString(), Arrays.toString(expectedAmbigAlts));
// }
// assertEquals(expectedAmbigInput, a.input);
// }
// void unreachable(List<Message> msgs, int[] expectedUnreachableAlts)
// throws Exception
// {
// unreachable(msgs, 0, expectedUnreachableAlts);
// }
// void unreachable(List<Message> msgs, int i, int[] expectedUnreachableAlts)
// throws Exception
// {
// List<Message> amsgs = getMessagesOfType(msgs, UnreachableAltsMessage.class);
// UnreachableAltsMessage u = (UnreachableAltsMessage)amsgs.get(i);
// if ( u==null ) assertNull(expectedUnreachableAlts);
// else {
// assertEquals(u.conflictingAlts.toString(), Arrays.toString(expectedUnreachableAlts));
// }
// }
List<ANTLRMessage> getMessagesOfType(List<ANTLRMessage> msgs, Class<? extends ANTLRMessage> c) {
List<ANTLRMessage> filtered = new ArrayList<ANTLRMessage>();
for (ANTLRMessage m : msgs) {
if ( m.getClass() == c ) filtered.add(m);
}
return filtered;
}
public void checkRuleATN(Grammar g, String ruleName, String expecting) {
// DOTGenerator dot = new DOTGenerator(g);
// System.out.println(dot.getDOT(g.atn.ruleToStartState[g.getRule(ruleName).index]));
@ -824,25 +481,7 @@ public class BaseJavaTest implements RuntimeTestSupport {
}
}
protected void checkGrammarSemanticsError(ErrorQueue equeue,
GrammarSemanticsMessage expectedMessage)
throws Exception
{
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
ANTLRMessage m = equeue.errors.get(i);
if (m.getErrorType()==expectedMessage.getErrorType() ) {
foundMsg = m;
}
}
assertNotNull("no error; "+expectedMessage.getErrorType()+" expected", foundMsg);
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs()));
if ( equeue.size()!=1 ) {
System.err.println(equeue);
}
}
protected void checkGrammarSemanticsWarning(ErrorQueue equeue,
GrammarSemanticsMessage expectedMessage)
@ -864,49 +503,6 @@ public class BaseJavaTest implements RuntimeTestSupport {
}
}
protected void checkError(ErrorQueue equeue,
ANTLRMessage expectedMessage)
throws Exception
{
//System.out.println("errors="+equeue);
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
ANTLRMessage m = equeue.errors.get(i);
if (m.getErrorType()==expectedMessage.getErrorType() ) {
foundMsg = m;
}
}
assertTrue("no error; "+expectedMessage.getErrorType()+" expected", !equeue.errors.isEmpty());
assertTrue("too many errors; "+equeue.errors, equeue.errors.size()<=1);
assertNotNull("couldn't find expected error: "+expectedMessage.getErrorType(), foundMsg);
/*
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
*/
assertArrayEquals(expectedMessage.getArgs(), foundMsg.getArgs());
}
public static class FilteringTokenStream extends CommonTokenStream {
public FilteringTokenStream(TokenSource src) { super(src); }
Set<Integer> hide = new HashSet<Integer>();
@Override
protected boolean sync(int i) {
if (!super.sync(i)) {
return false;
}
Token t = get(i);
if ( hide.contains(t.getType()) ) {
((WritableToken)t).setChannel(Token.HIDDEN_CHANNEL);
}
return true;
}
public void setTokenTypeChannel(int ttype, int channel) {
hide.add(ttype);
}
}
protected void writeTestFile(String parserName,
String lexerName,
String parserStartRuleName,
@ -969,7 +565,7 @@ public class BaseJavaTest implements RuntimeTestSupport {
outputFileST.add("parserName", parserName);
outputFileST.add("lexerName", lexerName);
outputFileST.add("parserStartRuleName", parserStartRuleName);
writeFile(tmpdir, "Test.java", outputFileST.render());
writeFile(getTempDirPath(), "Test.java", outputFileST.render());
}
protected void writeLexerTestFile(String lexerName, boolean showDFA) {
@ -990,210 +586,13 @@ public class BaseJavaTest implements RuntimeTestSupport {
);
outputFileST.add("lexerName", lexerName);
writeFile(tmpdir, "Test.java", outputFileST.render());
writeFile(getTempDirPath(), "Test.java", outputFileST.render());
}
public void writeRecognizerAndCompile(String parserName, String lexerName,
String parserStartRuleName,
boolean debug,
boolean profile) {
if ( parserName==null ) {
writeLexerTestFile(lexerName, debug);
}
else {
writeTestFile(parserName,
lexerName,
parserStartRuleName,
debug,
profile);
}
compile("Test.java");
}
protected void eraseFiles(final String filesEndingWith) {
File tmpdirF = new File(tmpdir);
String[] files = tmpdirF.list();
for(int i = 0; files!=null && i < files.length; i++) {
if ( files[i].endsWith(filesEndingWith) ) {
new File(tmpdir+"/"+files[i]).delete();
}
}
}
protected void eraseFiles() {
if (tmpdir == null) {
return;
}
File tmpdirF = new File(tmpdir);
String[] files = tmpdirF.list();
for(int i = 0; files!=null && i < files.length; i++) {
new File(tmpdir+"/"+files[i]).delete();
}
}
public void eraseTempDir() {
File tmpdirF = new File(tmpdir);
if ( tmpdirF.exists() ) {
eraseFiles();
tmpdirF.delete();
}
}
public String getFirstLineOfException() {
if ( this.stderrDuringParse ==null ) {
return null;
}
String[] lines = this.stderrDuringParse.split("\n");
String prefix="Exception in thread \"main\" ";
return lines[0].substring(prefix.length(),lines[0].length());
}
/**
* When looking at a result set that consists of a Map/HashTable
* we cannot rely on the output order, as the hashing algorithm or other aspects
* of the implementation may be different on differnt JDKs or platforms. Hence
* we take the Map, convert the keys to a List, sort them and Stringify the Map, which is a
* bit of a hack, but guarantees that we get the same order on all systems. We assume that
* the keys are strings.
*
* @param m The Map that contains keys we wish to return in sorted order
* @return A string that represents all the keys in sorted order.
*/
public <K, V> String sortMapToString(Map<K, V> m) {
// Pass in crap, and get nothing back
//
if (m == null) {
return null;
}
System.out.println("Map toString looks like: " + m.toString());
// Sort the keys in the Map
//
TreeMap<K, V> nset = new TreeMap<K, V>(m);
System.out.println("Tree map looks like: " + nset.toString());
return nset.toString();
}
public List<String> realElements(List<String> elements) {
return elements.subList(Token.MIN_USER_TOKEN_TYPE, elements.size());
}
public void assertNotNullOrEmpty(String message, String text) {
assertNotNull(message, text);
assertFalse(message, text.isEmpty());
}
public void assertNotNullOrEmpty(String text) {
assertNotNull(text);
assertFalse(text.isEmpty());
}
public static class IntTokenStream implements TokenStream {
public IntegerList types;
int p=0;
public IntTokenStream(IntegerList types) { this.types = types; }
@Override
public void consume() { p++; }
@Override
public int LA(int i) { return LT(i).getType(); }
@Override
public int mark() {
return index();
}
@Override
public int index() { return p; }
@Override
public void release(int marker) {
seek(marker);
}
@Override
public void seek(int index) {
p = index;
}
@Override
public int size() {
return types.size();
}
@Override
public String getSourceName() {
return UNKNOWN_SOURCE_NAME;
}
@Override
public Token LT(int i) {
CommonToken t;
int rawIndex = p + i - 1;
if ( rawIndex>=types.size() ) t = new CommonToken(Token.EOF);
else t = new CommonToken(types.get(rawIndex));
t.setTokenIndex(rawIndex);
return t;
}
@Override
public Token get(int i) {
return new org.antlr.v4.runtime.CommonToken(types.get(i));
}
@Override
public TokenSource getTokenSource() {
return null;
}
@Override
public String getText() {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Interval interval) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(RuleContext ctx) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Token start, Token stop) {
throw new UnsupportedOperationException("can't give strings");
}
}
/** Sort a list */
public <T extends Comparable<? super T>> List<T> sort(List<T> data) {
List<T> dup = new ArrayList<T>();
dup.addAll(data);
Collections.sort(dup);
return dup;
}
/** Return map sorted by key */
public <K extends Comparable<? super K>,V> LinkedHashMap<K,V> sort(Map<K,V> data) {
LinkedHashMap<K,V> dup = new LinkedHashMap<K, V>();
List<K> keys = new ArrayList<K>();
keys.addAll(data.keySet());
Collections.sort(keys);
for (K k : keys) {
dup.put(k, data.get(k));
}
return dup;
}
}

788
runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java
View File

@ -5,229 +5,24 @@
*/
package org.antlr.v4.test.runtime.javascript;
import org.antlr.v4.Tool;
import org.antlr.v4.automata.ATNFactory;
import org.antlr.v4.automata.ATNPrinter;
import org.antlr.v4.automata.LexerATNFactory;
import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.IntStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenSource;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.WritableToken;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNDeserializer;
import org.antlr.v4.runtime.atn.ATNSerializer;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.semantics.SemanticPipeline;
import org.antlr.v4.test.runtime.ErrorQueue;
import org.antlr.v4.test.runtime.RuntimeTestSupport;
import org.antlr.v4.test.runtime.StreamVacuum;
import org.antlr.v4.tool.ANTLRMessage;
import org.antlr.v4.tool.DOTGenerator;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.GrammarSemanticsMessage;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
import org.antlr.v4.runtime.misc.Utils;
import org.antlr.v4.test.runtime.*;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupString;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.*;
import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.*;
public class BaseNodeTest implements RuntimeTestSupport {
// -J-Dorg.antlr.v4.test.BaseTest.level=FINE
// private static final Logger LOGGER =
// Logger.getLogger(BaseTest.class.getName());
public static final String newline = System.getProperty("line.separator");
public static final String pathSep = System.getProperty("path.separator");
public String tmpdir = null;
/**
* If error during parser execution, store stderr here; can't return stdout
* and stderr. This doesn't trap errors from running antlr.
*/
protected String stderrDuringParse;
/** Errors found while running antlr */
protected StringBuilder antlrToolErrors;
public class BaseNodeTest extends BaseRuntimeTestSupport implements RuntimeTestSupport {
@Override
public void testSetUp() throws Exception {
// new output dir for each test
String prop = System.getProperty("antlr-javascript-test-dir");
if (prop != null && prop.length() > 0) {
tmpdir = prop;
}
else {
tmpdir = new File(System.getProperty("java.io.tmpdir"), getClass()
.getSimpleName()+"-"+Thread.currentThread().getName()+"-"+System.currentTimeMillis())
.getAbsolutePath();
}
File dir = new File(tmpdir);
if (dir.exists())
this.eraseFiles(dir);
antlrToolErrors = new StringBuilder();
}
@Override
public void testTearDown() throws Exception {
}
@Override
public String getTmpDir() {
return tmpdir;
}
@Override
public String getStdout() {
return null;
}
@Override
public String getParseErrors() {
return stderrDuringParse;
}
@Override
public String getANTLRToolErrors() {
if ( antlrToolErrors.length()==0 ) {
return null;
}
return antlrToolErrors.toString();
}
protected org.antlr.v4.Tool newTool(String[] args) {
Tool tool = new Tool(args);
return tool;
}
protected Tool newTool() {
org.antlr.v4.Tool tool = new Tool(new String[] { "-o", tmpdir });
return tool;
}
protected ATN createATN(Grammar g, boolean useSerializer) {
if (g.atn == null) {
semanticProcess(g);
assertEquals(0, g.tool.getNumErrors());
ParserATNFactory f;
if (g.isLexer()) {
f = new LexerATNFactory((LexerGrammar) g);
}
else {
f = new ParserATNFactory(g);
}
g.atn = f.createATN();
assertEquals(0, g.tool.getNumErrors());
}
ATN atn = g.atn;
if (useSerializer) {
char[] serialized = ATNSerializer.getSerializedAsChars(atn);
return new ATNDeserializer().deserialize(serialized);
}
return atn;
}
protected void semanticProcess(Grammar g) {
if (g.ast != null && !g.ast.hasErrors) {
System.out.println(g.ast.toStringTree());
Tool antlr = new Tool();
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
if (g.getImportedGrammars() != null) { // process imported grammars
// (if any)
for (Grammar imp : g.getImportedGrammars()) {
antlr.processNonCombinedGrammar(imp, false);
}
}
}
}
IntegerList getTypesFromString(Grammar g, String expecting) {
IntegerList expectingTokenTypes = new IntegerList();
if (expecting != null && !expecting.trim().isEmpty()) {
for (String tname : expecting.replace(" ", "").split(",")) {
int ttype = g.getTokenType(tname);
expectingTokenTypes.add(ttype);
}
}
return expectingTokenTypes;
}
public IntegerList getTokenTypesViaATN(String input,
LexerATNSimulator lexerATN) {
ANTLRInputStream in = new ANTLRInputStream(input);
IntegerList tokenTypes = new IntegerList();
int ttype;
do {
ttype = lexerATN.match(in, Lexer.DEFAULT_MODE);
tokenTypes.add(ttype);
} while (ttype != Token.EOF);
return tokenTypes;
}
public List<String> getTokenTypes(LexerGrammar lg, ATN atn, CharStream input) {
LexerATNSimulator interp = new LexerATNSimulator(atn,
new DFA[] { new DFA(
atn.modeToStartState.get(Lexer.DEFAULT_MODE)) }, null);
List<String> tokenTypes = new ArrayList<String>();
int ttype;
boolean hitEOF = false;
do {
if (hitEOF) {
tokenTypes.add("EOF");
break;
}
int t = input.LA(1);
ttype = interp.match(input, Lexer.DEFAULT_MODE);
if (ttype == Token.EOF) {
tokenTypes.add("EOF");
}
else {
tokenTypes.add(lg.typeToTokenList.get(ttype));
}
if (t == IntStream.EOF) {
hitEOF = true;
}
} while (ttype != Token.EOF);
return tokenTypes;
protected String getPropertyPrefix() {
return "antlr4-javascript";
}
protected String execLexer(String grammarFileName, String grammarStr,
@ -241,10 +36,11 @@ public class BaseNodeTest implements RuntimeTestSupport {
boolean success = rawGenerateAndBuildRecognizer(grammarFileName,
grammarStr, null, lexerName, "-no-listener");
assertTrue(success);
writeFile(tmpdir, "input", input);
writeFile(getTempDirPath(), "input", input);
writeLexerTestFile(lexerName, showDFA);
writeFile(getTempDirPath(), "package.json", "{\"type\": \"module\"}");
String output = execModule("Test.js");
if ( output.length()==0 ) {
if ( output!=null && output.length()==0 ) {
output = null;
}
return output;
@ -259,9 +55,10 @@ public class BaseNodeTest implements RuntimeTestSupport {
boolean success = rawGenerateAndBuildRecognizer(grammarFileName,
grammarStr, parserName, lexerName, "-visitor");
assertTrue(success);
writeFile(tmpdir, "input", input);
writeFile(getTempDirPath(), "input", input);
rawBuildRecognizerTestFile(parserName, lexerName, listenerName,
visitorName, startRuleName, showDiagnosticErrors);
writeFile(getTempDirPath(), "package.json", "{\"type\": \"module\"}");
return execRecognizer();
}
@ -277,7 +74,7 @@ public class BaseNodeTest implements RuntimeTestSupport {
protected boolean rawGenerateAndBuildRecognizer(String grammarFileName,
String grammarStr, String parserName, String lexerName,
boolean defaultListener, String... extraOptions) {
ErrorQueue equeue = antlrOnString(getTmpDir(), "JavaScript", grammarFileName, grammarStr,
ErrorQueue equeue = antlrOnString(getTempDirPath(), "JavaScript", grammarFileName, grammarStr,
defaultListener, extraOptions);
if (!equeue.errors.isEmpty()) {
return false;
@ -308,7 +105,7 @@ public class BaseNodeTest implements RuntimeTestSupport {
protected void rawBuildRecognizerTestFile(String parserName,
String lexerName, String listenerName, String visitorName,
String parserStartRuleName, boolean debug) {
this.stderrDuringParse = null;
setParseErrors(null);
if (parserName == null) {
writeLexerTestFile(lexerName, false);
}
@ -323,18 +120,22 @@ public class BaseNodeTest implements RuntimeTestSupport {
}
public String execModule(String fileName) {
String nodejsPath = locateNodeJS();
String runtimePath = locateRuntime();
String modulePath = new File(new File(tmpdir), fileName)
.getAbsolutePath();
String inputPath = new File(new File(tmpdir), "input")
.getAbsolutePath();
try {
String npmPath = locateNpm();
if(!TestContext.isCI()) {
installRuntime(npmPath);
registerRuntime(npmPath);
}
String modulePath = new File(getTempTestDir(), fileName)
.getAbsolutePath();
linkRuntime(npmPath);
String nodejsPath = locateNodeJS();
String inputPath = new File(getTempTestDir(), "input")
.getAbsolutePath();
ProcessBuilder builder = new ProcessBuilder(nodejsPath, modulePath,
inputPath);
builder.environment().put("NODE_PATH",
runtimePath + File.pathSeparator + tmpdir);
builder.directory(new File(tmpdir));
builder.environment().put("NODE_PATH", getTempDirPath());
builder.directory(getTempTestDir());
Process process = builder.start();
StreamVacuum stdoutVacuum = new StreamVacuum(
process.getInputStream());
@ -342,7 +143,10 @@ public class BaseNodeTest implements RuntimeTestSupport {
process.getErrorStream());
stdoutVacuum.start();
stderrVacuum.start();
// TODO switch to jdk 8
process.waitFor();
// if(!process.waitFor(1L, TimeUnit.MINUTES))
// process.destroyForcibly();
stdoutVacuum.join();
stderrVacuum.join();
String output = stdoutVacuum.toString();
@ -350,25 +154,69 @@ public class BaseNodeTest implements RuntimeTestSupport {
output = null;
}
if (stderrVacuum.toString().length() > 0) {
this.stderrDuringParse = stderrVacuum.toString();
setParseErrors(stderrVacuum.toString());
}
return output;
}
catch (Exception e) {
} catch (Exception e) {
System.err.println("can't exec recognizer");
e.printStackTrace(System.err);
System.err.println();
return null;
}
return null;
}
private String locateTool(String tool) {
String[] roots = { "/usr/bin/", "/usr/local/bin/" };
for (String root : roots) {
if (new File(root + tool).exists()) {
return root + tool;
}
private void installRuntime(String npmPath) throws IOException, InterruptedException {
String runtimePath = locateRuntime();
ProcessBuilder builder = new ProcessBuilder(npmPath, "install");
builder.directory(new File(runtimePath));
builder.redirectError(new File(getTempTestDir(), "error.txt"));
builder.redirectOutput(new File(getTempTestDir(), "output.txt"));
Process process = builder.start();
// TODO switch to jdk 8
process.waitFor();
// if(!process.waitFor(30L, TimeUnit.SECONDS))
// process.destroyForcibly();
int error = process.exitValue();
if(error!=0)
throw new IOException("'npm install' failed");
}
private void registerRuntime(String npmPath) throws IOException, InterruptedException {
String runtimePath = locateRuntime();
ProcessBuilder builder = new ProcessBuilder(npmPath, "link");
builder.directory(new File(runtimePath));
builder.redirectError(new File(getTempTestDir(), "error.txt"));
builder.redirectOutput(new File(getTempTestDir(), "output.txt"));
Process process = builder.start();
// TODO switch to jdk 8
process.waitFor();
// if(!process.waitFor(30L, TimeUnit.SECONDS))
// process.destroyForcibly();
int error = process.exitValue();
if(error!=0)
throw new IOException("'npm link' failed");
}
private void linkRuntime(String npmPath) throws IOException, InterruptedException {
List<String> args = new ArrayList<>();
if(TestContext.isCircleCI())
args.add("sudo");
args.addAll(Arrays.asList(npmPath, "link", "antlr4"));
ProcessBuilder builder = new ProcessBuilder(args.toArray(new String[0]));
builder.directory(getTempTestDir());
File errorFile = new File(getTempTestDir(), "error.txt");
builder.redirectError(errorFile);
builder.redirectOutput(new File(getTempTestDir(), "output.txt"));
Process process = builder.start();
// TODO switch to jdk 8
process.waitFor();
// if(!process.waitFor(30L, TimeUnit.SECONDS))
// process.destroyForcibly();
int error = process.exitValue();
if(error!=0) {
char[] errors = Utils.readFile(errorFile.getAbsolutePath());
throw new IOException("'npm link antlr4' failed: " + new String(errors));
}
return null;
}
private boolean canExecute(String tool) {
@ -378,22 +226,34 @@ public class BaseNodeTest implements RuntimeTestSupport {
Process process = builder.start();
StreamVacuum vacuum = new StreamVacuum(process.getInputStream());
vacuum.start();
// TODO switch to jdk 8
process.waitFor();
// if(!process.waitFor(30L, TimeUnit.SECONDS))
// process.destroyForcibly();
vacuum.join();
return process.exitValue() == 0;
} catch (Exception e) {
return false;
}
catch (Exception e) {
;
}
private String locateNpm() {
// typically /usr/local/bin/npm
String prop = System.getProperty("antlr-javascript-npm");
if ( prop!=null && prop.length()!=0 ) {
if(prop.contains(" "))
prop = "\"" + prop + "\"";
return prop;
}
return false;
return "npm"; // everywhere
}
private String locateNodeJS() {
// typically /usr/local/bin/node
String propName = "antlr-javascript-nodejs";
String prop = System.getProperty(propName);
String prop = System.getProperty("antlr-javascript-nodejs");
if ( prop!=null && prop.length()!=0 ) {
if(prop.contains(" "))
prop = "\"" + prop + "\"";
return prop;
}
if (canExecute("nodejs")) {
@ -404,7 +264,7 @@ public class BaseNodeTest implements RuntimeTestSupport {
private String locateRuntime() {
final ClassLoader loader = Thread.currentThread().getContextClassLoader();
final URL runtimeSrc = loader.getResource("JavaScript/src");
final URL runtimeSrc = loader.getResource("JavaScript");
if ( runtimeSrc==null ) {
throw new RuntimeException("Cannot find JavaScript runtime");
}
@ -414,217 +274,15 @@ public class BaseNodeTest implements RuntimeTestSupport {
return runtimeSrc.getPath();
}
private boolean isWindows() {
return System.getProperty("os.name").toLowerCase().contains("windows");
}
// void ambig(List<Message> msgs, int[] expectedAmbigAlts, String
// expectedAmbigInput)
// throws Exception
// {
// ambig(msgs, 0, expectedAmbigAlts, expectedAmbigInput);
// }
// void ambig(List<Message> msgs, int i, int[] expectedAmbigAlts, String
// expectedAmbigInput)
// throws Exception
// {
// List<Message> amsgs = getMessagesOfType(msgs, AmbiguityMessage.class);
// AmbiguityMessage a = (AmbiguityMessage)amsgs.get(i);
// if ( a==null ) assertNull(expectedAmbigAlts);
// else {
// assertEquals(a.conflictingAlts.toString(),
// Arrays.toString(expectedAmbigAlts));
// }
// assertEquals(expectedAmbigInput, a.input);
// }
// void unreachable(List<Message> msgs, int[] expectedUnreachableAlts)
// throws Exception
// {
// unreachable(msgs, 0, expectedUnreachableAlts);
// }
// void unreachable(List<Message> msgs, int i, int[]
// expectedUnreachableAlts)
// throws Exception
// {
// List<Message> amsgs = getMessagesOfType(msgs,
// UnreachableAltsMessage.class);
// UnreachableAltsMessage u = (UnreachableAltsMessage)amsgs.get(i);
// if ( u==null ) assertNull(expectedUnreachableAlts);
// else {
// assertEquals(u.conflictingAlts.toString(),
// Arrays.toString(expectedUnreachableAlts));
// }
// }
List<ANTLRMessage> getMessagesOfType(List<ANTLRMessage> msgs,
Class<? extends ANTLRMessage> c) {
List<ANTLRMessage> filtered = new ArrayList<ANTLRMessage>();
for (ANTLRMessage m : msgs) {
if (m.getClass() == c)
filtered.add(m);
}
return filtered;
}
void checkRuleATN(Grammar g, String ruleName, String expecting) {
ParserATNFactory f = new ParserATNFactory(g);
ATN atn = f.createATN();
DOTGenerator dot = new DOTGenerator(g);
System.out
.println(dot.getDOT(atn.ruleToStartState[g.getRule(ruleName).index]));
Rule r = g.getRule(ruleName);
ATNState startState = atn.ruleToStartState[r.index];
ATNPrinter serializer = new ATNPrinter(g, startState);
String result = serializer.asString();
// System.out.print(result);
assertEquals(expecting, result);
}
public void testActions(String templates, String actionName, String action,
String expected) throws org.antlr.runtime.RecognitionException {
int lp = templates.indexOf('(');
String name = templates.substring(0, lp);
STGroup group = new STGroupString(templates);
ST st = group.getInstanceOf(name);
st.add(actionName, action);
String grammar = st.render();
ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(grammar, equeue);
if (g.ast != null && !g.ast.hasErrors) {
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
ATNFactory factory = new ParserATNFactory(g);
if (g.isLexer())
factory = new LexerATNFactory((LexerGrammar) g);
g.atn = factory.createATN();
CodeGenerator gen = new CodeGenerator(g);
ST outputFileST = gen.generateParser();
String output = outputFileST.render();
// System.out.println(output);
String b = "#" + actionName + "#";
int start = output.indexOf(b);
String e = "#end-" + actionName + "#";
int end = output.indexOf(e);
String snippet = output.substring(start + b.length(), end);
assertEquals(expected, snippet);
}
if (equeue.size() > 0) {
System.err.println(equeue.toString());
}
}
protected void checkGrammarSemanticsError(ErrorQueue equeue,
GrammarSemanticsMessage expectedMessage) throws Exception {
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
ANTLRMessage m = equeue.errors.get(i);
if (m.getErrorType() == expectedMessage.getErrorType()) {
foundMsg = m;
}
}
assertNotNull("no error; " + expectedMessage.getErrorType()
+ " expected", foundMsg);
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
assertEquals(Arrays.toString(expectedMessage.getArgs()),
Arrays.toString(foundMsg.getArgs()));
if (equeue.size() != 1) {
System.err.println(equeue);
}
}
protected void checkGrammarSemanticsWarning(ErrorQueue equeue,
GrammarSemanticsMessage expectedMessage) throws Exception {
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.warnings.size(); i++) {
ANTLRMessage m = equeue.warnings.get(i);
if (m.getErrorType() == expectedMessage.getErrorType()) {
foundMsg = m;
}
}
assertNotNull("no error; " + expectedMessage.getErrorType()
+ " expected", foundMsg);
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
assertEquals(Arrays.toString(expectedMessage.getArgs()),
Arrays.toString(foundMsg.getArgs()));
if (equeue.size() != 1) {
System.err.println(equeue);
}
}
protected void checkError(ErrorQueue equeue, ANTLRMessage expectedMessage)
throws Exception {
// System.out.println("errors="+equeue);
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
ANTLRMessage m = equeue.errors.get(i);
if (m.getErrorType() == expectedMessage.getErrorType()) {
foundMsg = m;
}
}
assertTrue("no error; " + expectedMessage.getErrorType() + " expected",
!equeue.errors.isEmpty());
assertTrue("too many errors; " + equeue.errors,
equeue.errors.size() <= 1);
assertNotNull(
"couldn't find expected error: "
+ expectedMessage.getErrorType(), foundMsg);
/*
* assertTrue("error is not a GrammarSemanticsMessage", foundMsg
* instanceof GrammarSemanticsMessage);
*/
assertArrayEquals(expectedMessage.getArgs(), foundMsg.getArgs());
}
public static class FilteringTokenStream extends CommonTokenStream {
public FilteringTokenStream(TokenSource src) {
super(src);
}
Set<Integer> hide = new HashSet<Integer>();
@Override
protected boolean sync(int i) {
if (!super.sync(i)) {
return false;
}
Token t = get(i);
if (hide.contains(t.getType())) {
((WritableToken) t).setChannel(Token.HIDDEN_CHANNEL);
}
return true;
}
public void setTokenTypeChannel(int ttype, int channel) {
hide.add(ttype);
}
}
protected void mkdir(String dir) {
File f = new File(dir);
f.mkdirs();
}
protected void writeParserTestFile(String parserName, String lexerName,
String listenerName, String visitorName,
String parserStartRuleName, boolean debug) {
ST outputFileST = new ST(
"var antlr4 = require('antlr4');\n"
+ "var <lexerName> = require('./<lexerName>');\n"
+ "var <parserName> = require('./<parserName>');\n"
+ "var <listenerName> = require('./<listenerName>').<listenerName>;\n"
+ "var <visitorName> = require('./<visitorName>').<visitorName>;\n"
"import antlr4 from 'antlr4';\n"
+ "import <lexerName> from './<lexerName>.js';\n"
+ "import <parserName> from './<parserName>.js';\n"
+ "import <listenerName> from './<listenerName>.js';\n"
+ "import <visitorName> from './<visitorName>.js';\n"
+ "\n"
+ "class TreeShapeListener extends antlr4.tree.ParseTreeListener {\n" +
" enterEveryRule(ctx) {\n" +
@ -640,11 +298,11 @@ public class BaseNodeTest implements RuntimeTestSupport {
+ "\n"
+ "function main(argv) {\n"
+ " var input = new antlr4.FileStream(argv[2], true);\n"
+ " var lexer = new <lexerName>.<lexerName>(input);\n"
+ " var lexer = new <lexerName>(input);\n"
+ " var stream = new antlr4.CommonTokenStream(lexer);\n"
+ "<createParser>"
+ " parser.buildParseTrees = true;\n"
+ " printer = function() {\n"
+ " const printer = function() {\n"
+ " this.println = function(s) { console.log(s); }\n"
+ " this.print = function(s) { process.stdout.write(s); }\n"
+ " return this;\n"
@ -654,10 +312,10 @@ public class BaseNodeTest implements RuntimeTestSupport {
+ " antlr4.tree.ParseTreeWalker.DEFAULT.walk(new TreeShapeListener(), tree);\n"
+ "}\n" + "\n" + "main(process.argv);\n" + "\n");
ST createParserST = new ST(
" var parser = new <parserName>.<parserName>(stream);\n");
" var parser = new <parserName>(stream);\n");
if (debug) {
createParserST = new ST(
" var parser = new <parserName>.<parserName>(stream);\n"
" var parser = new <parserName>(stream);\n"
+ " parser.addErrorListener(new antlr4.error.DiagnosticErrorListener());\n");
}
outputFileST.add("createParser", createParserST);
@ -666,17 +324,17 @@ public class BaseNodeTest implements RuntimeTestSupport {
outputFileST.add("listenerName", listenerName);
outputFileST.add("visitorName", visitorName);
outputFileST.add("parserStartRuleName", parserStartRuleName);
writeFile(tmpdir, "Test.js", outputFileST.render());
writeFile(getTempDirPath(), "Test.js", outputFileST.render());
}
protected void writeLexerTestFile(String lexerName, boolean showDFA) {
ST outputFileST = new ST(
"var antlr4 = require('antlr4');\n"
+ "var <lexerName> = require('./<lexerName>');\n"
"import antlr4 from 'antlr4';\n"
+ "import <lexerName> from './<lexerName>.js';\n"
+ "\n"
+ "function main(argv) {\n"
+ " var input = new antlr4.FileStream(argv[2], true);\n"
+ " var lexer = new <lexerName>.<lexerName>(input);\n"
+ " var lexer = new <lexerName>(input);\n"
+ " var stream = new antlr4.CommonTokenStream(lexer);\n"
+ " stream.fill();\n"
+ " for(var i=0; i\\<stream.tokens.length; i++) {\n"
@ -686,215 +344,7 @@ public class BaseNodeTest implements RuntimeTestSupport {
: "") + "}\n" + "\n" + "main(process.argv);\n"
+ "\n");
outputFileST.add("lexerName", lexerName);
writeFile(tmpdir, "Test.js", outputFileST.render());
writeFile(getTempDirPath(), "Test.js", outputFileST.render());
}
public void writeRecognizer(String parserName, String lexerName,
String listenerName, String visitorName,
String parserStartRuleName, boolean debug) {
if (parserName == null) {
writeLexerTestFile(lexerName, debug);
}
else {
writeParserTestFile(parserName, lexerName, listenerName,
visitorName, parserStartRuleName, debug);
}
}
protected void eraseFiles(final String filesEndingWith) {
File tmpdirF = new File(tmpdir);
String[] files = tmpdirF.list();
for (int i = 0; files != null && i < files.length; i++) {
if (files[i].endsWith(filesEndingWith)) {
new File(tmpdir + "/" + files[i]).delete();
}
}
}
protected void eraseFiles(File dir) {
String[] files = dir.list();
for (int i = 0; files != null && i < files.length; i++) {
new File(dir, files[i]).delete();
}
}
@Override
public void eraseTempDir() {
boolean doErase = true;
String propName = "antlr-javascript-erase-test-dir";
String prop = System.getProperty(propName);
if (prop != null && prop.length() > 0)
doErase = Boolean.getBoolean(prop);
if (doErase) {
File tmpdirF = new File(tmpdir);
if (tmpdirF.exists()) {
eraseFiles(tmpdirF);
tmpdirF.delete();
}
}
}
public String getFirstLineOfException() {
if (this.stderrDuringParse == null) {
return null;
}
String[] lines = this.stderrDuringParse.split("\n");
String prefix = "Exception in thread \"main\" ";
return lines[0].substring(prefix.length(), lines[0].length());
}
/**
* When looking at a result set that consists of a Map/HashTable we cannot
* rely on the output order, as the hashing algorithm or other aspects of
* the implementation may be different on differnt JDKs or platforms. Hence
* we take the Map, convert the keys to a List, sort them and Stringify the
* Map, which is a bit of a hack, but guarantees that we get the same order
* on all systems. We assume that the keys are strings.
*
* @param m
* The Map that contains keys we wish to return in sorted order
* @return A string that represents all the keys in sorted order.
*/
public <K, V> String sortMapToString(Map<K, V> m) {
// Pass in crap, and get nothing back
//
if (m == null) {
return null;
}
System.out.println("Map toString looks like: " + m.toString());
// Sort the keys in the Map
//
TreeMap<K, V> nset = new TreeMap<K, V>(m);
System.out.println("Tree map looks like: " + nset.toString());
return nset.toString();
}
public List<String> realElements(List<String> elements) {
return elements.subList(Token.MIN_USER_TOKEN_TYPE, elements.size());
}
public void assertNotNullOrEmpty(String message, String text) {
assertNotNull(message, text);
assertFalse(message, text.isEmpty());
}
public void assertNotNullOrEmpty(String text) {
assertNotNull(text);
assertFalse(text.isEmpty());
}
public static class IntTokenStream implements TokenStream {
IntegerList types;
int p = 0;
public IntTokenStream(IntegerList types) {
this.types = types;
}
@Override
public void consume() {
p++;
}
@Override
public int LA(int i) {
return LT(i).getType();
}
@Override
public int mark() {
return index();
}
@Override
public int index() {
return p;
}
@Override
public void release(int marker) {
seek(marker);
}
@Override
public void seek(int index) {
p = index;
}
@Override
public int size() {
return types.size();
}
@Override
public String getSourceName() {
return null;
}
@Override
public Token LT(int i) {
CommonToken t;
int rawIndex = p + i - 1;
if (rawIndex >= types.size())
t = new CommonToken(Token.EOF);
else
t = new CommonToken(types.get(rawIndex));
t.setTokenIndex(rawIndex);
return t;
}
@Override
public Token get(int i) {
return new org.antlr.v4.runtime.CommonToken(types.get(i));
}
@Override
public TokenSource getTokenSource() {
return null;
}
@Override
public String getText() {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Interval interval) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(RuleContext ctx) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Token start, Token stop) {
throw new UnsupportedOperationException("can't give strings");
}
}
/** Sort a list */
public <T extends Comparable<? super T>> List<T> sort(List<T> data) {
List<T> dup = new ArrayList<T>();
dup.addAll(data);
Collections.sort(dup);
return dup;
}
/** Return map sorted by key */
public <K extends Comparable<? super K>, V> LinkedHashMap<K, V> sort(
Map<K, V> data) {
LinkedHashMap<K, V> dup = new LinkedHashMap<K, V>();
List<K> keys = new ArrayList<K>();
keys.addAll(data.keySet());
Collections.sort(keys);
for (K k : keys) {
dup.put(k, data.get(k));
}
return dup;
}
}

207
runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java
View File

@ -11,24 +11,11 @@ import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.antlr.v4.Tool;
import org.antlr.v4.automata.LexerATNFactory;
import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNDeserializer;
import org.antlr.v4.runtime.atn.ATNSerializer;
import org.antlr.v4.semantics.SemanticPipeline;
import org.antlr.v4.test.runtime.ErrorQueue;
import org.antlr.v4.test.runtime.RuntimeTestSupport;
import org.antlr.v4.test.runtime.StreamVacuum;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.test.runtime.*;
import org.stringtemplate.v4.ST;
import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
@ -36,124 +23,12 @@ import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class BasePHPTest implements RuntimeTestSupport {
public static final String newline = System.getProperty("line.separator");
public class BasePHPTest extends BaseRuntimeTestSupport implements RuntimeTestSupport {
public String tmpdir = null;
/**
* If error during parser execution, store stderr here; can't return
* stdout and stderr. This doesn't trap errors from running antlr.
*/
protected String stderrDuringParse;
/**
* Errors found while running antlr
*/
protected StringBuilder antlrToolErrors;
private String getPropertyPrefix() {
public String getPropertyPrefix() {
return "antlr-php";
}
@Override
public void testSetUp() throws Exception {
// new output dir for each test
String propName = getPropertyPrefix() + "-test-dir";
String prop = System.getProperty(propName);
if (prop != null && prop.length() > 0) {
tmpdir = prop;
} else {
String classSimpleName = getClass().getSimpleName();
String threadName = Thread.currentThread().getName();
String childPath = String.format("%s-%s-%s", classSimpleName, threadName, System.currentTimeMillis());
tmpdir = new File(System.getProperty("java.io.tmpdir"), childPath).getAbsolutePath();
}
antlrToolErrors = new StringBuilder();
}
@Override
public void testTearDown() throws Exception {
}
@Override
public String getTmpDir() {
return tmpdir;
}
@Override
public String getStdout() {
return null;
}
@Override
public String getParseErrors() {
return stderrDuringParse;
}
@Override
public String getANTLRToolErrors() {
if (antlrToolErrors.length() == 0) {
return null;
}
return antlrToolErrors.toString();
}
protected ATN createATN(Grammar g, boolean useSerializer) {
if (g.atn == null) {
semanticProcess(g);
assertEquals(0, g.tool.getNumErrors());
ParserATNFactory f;
if (g.isLexer()) {
f = new LexerATNFactory((LexerGrammar) g);
} else {
f = new ParserATNFactory(g);
}
g.atn = f.createATN();
assertEquals(0, g.tool.getNumErrors());
}
ATN atn = g.atn;
if (useSerializer) {
char[] serialized = ATNSerializer.getSerializedAsChars(atn);
return new ATNDeserializer().deserialize(serialized);
}
return atn;
}
protected void semanticProcess(Grammar g) {
if (g.ast != null && !g.ast.hasErrors) {
Tool antlr = new Tool();
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
if (g.getImportedGrammars() != null) {
for (Grammar imp: g.getImportedGrammars()) {
antlr.processNonCombinedGrammar(imp, false);
}
}
}
}
protected String execLexer(
String grammarFileName,
String grammarStr,
String lexerName,
String input
) {
return execLexer(grammarFileName, grammarStr, lexerName, input, false);
}
@Override
public String execLexer(
String grammarFileName,
@ -170,11 +45,9 @@ public class BasePHPTest implements RuntimeTestSupport {
"-no-listener"
);
assertTrue(success);
writeFile(tmpdir, "input", input);
writeFile(getTempDirPath(), "input", input);
writeLexerTestFile(lexerName, showDFA);
String output = execModule("Test.php");
return output;
return execModule("Test.php");
}
public String execParser(
@ -224,7 +97,7 @@ public class BasePHPTest implements RuntimeTestSupport {
assertTrue(success);
writeFile(tmpdir, "input", input);
writeFile(getTempDirPath(), "input", input);
rawBuildRecognizerTestFile(
parserName,
@ -270,7 +143,7 @@ public class BasePHPTest implements RuntimeTestSupport {
boolean defaultListener,
String... extraOptions
) {
ErrorQueue equeue = antlrOnString(getTmpDir(), "PHP", grammarFileName, grammarStr, defaultListener, extraOptions);
ErrorQueue equeue = antlrOnString(getTempDirPath(), "PHP", grammarFileName, grammarStr, defaultListener, extraOptions);
if (!equeue.errors.isEmpty()) {
return false;
@ -307,7 +180,7 @@ public class BasePHPTest implements RuntimeTestSupport {
boolean debug,
boolean trace
) {
this.stderrDuringParse = null;
setParseErrors(null);
if (parserName == null) {
writeLexerTestFile(lexerName, false);
} else {
@ -331,15 +204,14 @@ public class BasePHPTest implements RuntimeTestSupport {
String phpPath = locatePhp();
String runtimePath = locateRuntime();
File tmpdirFile = new File(tmpdir);
String modulePath = new File(tmpdirFile, fileName).getAbsolutePath();
String inputPath = new File(tmpdirFile, "input").getAbsolutePath();
Path outputPath = tmpdirFile.toPath().resolve("output").toAbsolutePath();
String modulePath = new File(getTempTestDir(), fileName).getAbsolutePath();
String inputPath = new File(getTempTestDir(), "input").getAbsolutePath();
Path outputPath = getTempTestDir().toPath().resolve("output").toAbsolutePath();
try {
ProcessBuilder builder = new ProcessBuilder(phpPath, modulePath, inputPath, outputPath.toString());
builder.environment().put("RUNTIME", runtimePath);
builder.directory(tmpdirFile);
builder.directory(getTempTestDir());
Process process = builder.start();
StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
@ -355,7 +227,7 @@ public class BasePHPTest implements RuntimeTestSupport {
}
if (stderrVacuum.toString().length() > 0) {
this.stderrDuringParse = stderrVacuum.toString();
setParseErrors(stderrVacuum.toString());
}
return output;
@ -464,7 +336,7 @@ public class BasePHPTest implements RuntimeTestSupport {
outputFileST.add("lexerName", lexerName);
writeFile(tmpdir, "Test.php", outputFileST.render());
writeFile(getTempDirPath(), "Test.php", outputFileST.render());
}
protected void writeParserTestFile(
@ -546,54 +418,7 @@ public class BasePHPTest implements RuntimeTestSupport {
outputFileST.add("visitorName", visitorName);
outputFileST.add("parserStartRuleName", parserStartRuleName);
writeFile(tmpdir, "Test.php", outputFileST.render());
writeFile(getTempDirPath(), "Test.php", outputFileST.render());
}
protected void eraseFiles(File dir) {
String[] files = dir.list();
for (int i = 0; files != null && i < files.length; i++) {
new File(dir, files[i]).delete();
}
}
@Override
public void eraseTempDir() {
boolean doErase = true;
String propName = getPropertyPrefix() + "-erase-test-dir";
String prop = System.getProperty(propName);
if (prop != null && prop.length() > 0) {
doErase = Boolean.getBoolean(prop);
}
if (doErase) {
File tmpdirF = new File(tmpdir);
if (tmpdirF.exists()) {
eraseFiles(tmpdirF);
tmpdirF.delete();
}
}
}
/**
* Sort a list
*/
public <T extends Comparable<? super T>> List<T> sort(List<T> data) {
List<T> dup = new ArrayList<T>();
dup.addAll(data);
Collections.sort(dup);
return dup;
}
/**
* Return map sorted by key
*/
public <K extends Comparable<? super K>, V> LinkedHashMap<K, V> sort(Map<K, V> data) {
LinkedHashMap<K, V> dup = new LinkedHashMap<K, V>();
List<K> keys = new ArrayList<K>();
keys.addAll(data.keySet());
Collections.sort(keys);
for (K k: keys) {
dup.put(k, data.get(k));
}
return dup;
}
}

762
runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java
View File

@ -5,339 +5,38 @@
*/
package org.antlr.v4.test.runtime.python;
import org.antlr.v4.Tool;
import org.antlr.v4.automata.ATNFactory;
import org.antlr.v4.automata.ATNPrinter;
import org.antlr.v4.automata.LexerATNFactory;
import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.IntStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenSource;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.WritableToken;
import org.antlr.v4.runtime.atn.ATN;
import org.antlr.v4.runtime.atn.ATNDeserializer;
import org.antlr.v4.runtime.atn.ATNSerializer;
import org.antlr.v4.runtime.atn.ATNState;
import org.antlr.v4.runtime.atn.DecisionState;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.IntegerList;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.semantics.SemanticPipeline;
import org.antlr.v4.test.runtime.ErrorQueue;
import org.antlr.v4.test.runtime.RuntimeTestSupport;
import org.antlr.v4.test.runtime.StreamVacuum;
import org.antlr.v4.test.runtime.TestOutputReading;
import org.antlr.v4.tool.ANTLRMessage;
import org.antlr.v4.tool.DOTGenerator;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.GrammarSemanticsMessage;
import org.antlr.v4.tool.LexerGrammar;
import org.antlr.v4.tool.Rule;
import org.junit.rules.TestRule;
import org.junit.rules.TestWatcher;
import org.antlr.v4.test.runtime.*;
import org.junit.runner.Description;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupString;
import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URL;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.*;
public abstract class BasePythonTest implements RuntimeTestSupport {
// -J-Dorg.antlr.v4.test.BaseTest.level=FINE
// private static final Logger LOGGER = Logger.getLogger(BaseTest.class.getName());
public static final String newline = System.getProperty("line.separator");
public static final String pathSep = System.getProperty("path.separator");
public abstract class BasePythonTest extends BaseRuntimeTestSupport implements RuntimeTestSupport {
public String tmpdir = null;
@Override
protected void testSucceeded(Description description) {
eraseTempPyCache();
eraseTempDir();
}
/** If error during parser execution, store stderr here; can't return
* stdout and stderr. This doesn't trap errors from running antlr.
*/
protected String stderrDuringParse;
/** Errors found while running antlr */
protected StringBuilder antlrToolErrors;
@org.junit.Rule
public final TestRule testWatcher = new TestWatcher() {
@Override
protected void succeeded(Description description) {
// remove tmpdir if no error.
eraseTempPyCache();
eraseTempDir();
}
};
private String getPropertyPrefix() {
@Override
protected String getPropertyPrefix() {
return "antlr-" + getLanguage().toLowerCase();
}
@Override
public void testSetUp() throws Exception {
// new output dir for each test
String propName = getPropertyPrefix() + "-test-dir";
String prop = System.getProperty(propName);
if(prop!=null && prop.length()>0) {
tmpdir = prop;
}
else {
tmpdir = new File(System.getProperty("java.io.tmpdir"), getClass().getSimpleName()+
"-"+Thread.currentThread().getName()+"-"+System.currentTimeMillis()).getAbsolutePath();
}
antlrToolErrors = new StringBuilder();
}
@Override
public void testTearDown() throws Exception {
}
@Override
public String getTmpDir() {
return tmpdir;
}
@Override
public String getStdout() {
return null;
}
@Override
public String getParseErrors() {
return stderrDuringParse;
}
@Override
public String getANTLRToolErrors() {
if ( antlrToolErrors.length()==0 ) {
return null;
}
return antlrToolErrors.toString();
}
protected org.antlr.v4.Tool newTool(String[] args) {
Tool tool = new Tool(args);
return tool;
}
protected Tool newTool() {
org.antlr.v4.Tool tool = new Tool(new String[] {"-o", tmpdir});
return tool;
}
protected ATN createATN(Grammar g, boolean useSerializer) {
if ( g.atn==null ) {
semanticProcess(g);
assertEquals(0, g.tool.getNumErrors());
ParserATNFactory f;
if ( g.isLexer() ) {
f = new LexerATNFactory((LexerGrammar)g);
}
else {
f = new ParserATNFactory(g);
}
g.atn = f.createATN();
assertEquals(0, g.tool.getNumErrors());
}
ATN atn = g.atn;
if (useSerializer) {
char[] serialized = ATNSerializer.getSerializedAsChars(atn);
return new ATNDeserializer().deserialize(serialized);
}
return atn;
}
protected void semanticProcess(Grammar g) {
if ( g.ast!=null && !g.ast.hasErrors ) {
System.out.println(g.ast.toStringTree());
Tool antlr = new Tool();
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
if ( g.getImportedGrammars()!=null ) { // process imported grammars (if any)
for (Grammar imp : g.getImportedGrammars()) {
antlr.processNonCombinedGrammar(imp, false);
}
}
}
}
public DFA createDFA(Grammar g, DecisionState s) {
// PredictionDFAFactory conv = new PredictionDFAFactory(g, s);
// DFA dfa = conv.createDFA();
// conv.issueAmbiguityWarnings();
// System.out.print("DFA="+dfa);
// return dfa;
return null;
}
// public void minimizeDFA(DFA dfa) {
// DFAMinimizer dmin = new DFAMinimizer(dfa);
// dfa.minimized = dmin.minimize();
// }
IntegerList getTypesFromString(Grammar g, String expecting) {
IntegerList expectingTokenTypes = new IntegerList();
if ( expecting!=null && !expecting.trim().isEmpty() ) {
for (String tname : expecting.replace(" ", "").split(",")) {
int ttype = g.getTokenType(tname);
expectingTokenTypes.add(ttype);
}
}
return expectingTokenTypes;
}
public IntegerList getTokenTypesViaATN(String input, LexerATNSimulator lexerATN) {
ANTLRInputStream in = new ANTLRInputStream(input);
IntegerList tokenTypes = new IntegerList();
int ttype;
do {
ttype = lexerATN.match(in, Lexer.DEFAULT_MODE);
tokenTypes.add(ttype);
} while ( ttype!= Token.EOF );
return tokenTypes;
}
public List<String> getTokenTypes(LexerGrammar lg,
ATN atn,
CharStream input)
{
LexerATNSimulator interp = new LexerATNSimulator(atn,new DFA[] { new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE)) },null);
List<String> tokenTypes = new ArrayList<String>();
int ttype;
boolean hitEOF = false;
do {
if ( hitEOF ) {
tokenTypes.add("EOF");
break;
}
int t = input.LA(1);
ttype = interp.match(input, Lexer.DEFAULT_MODE);
if ( ttype == Token.EOF ) {
tokenTypes.add("EOF");
}
else {
tokenTypes.add(lg.typeToTokenList.get(ttype));
}
if ( t== IntStream.EOF ) {
hitEOF = true;
}
} while ( ttype!=Token.EOF );
return tokenTypes;
}
List<ANTLRMessage> checkRuleDFA(String gtext, String ruleName, String expecting)
throws Exception
{
ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(gtext, equeue);
ATN atn = createATN(g, false);
ATNState s = atn.ruleToStartState[g.getRule(ruleName).index];
if ( s==null ) {
System.err.println("no such rule: "+ruleName);
return null;
}
ATNState t = s.transition(0).target;
if ( !(t instanceof DecisionState) ) {
System.out.println(ruleName+" has no decision");
return null;
}
DecisionState blk = (DecisionState)t;
checkRuleDFA(g, blk, expecting);
return equeue.all;
}
List<ANTLRMessage> checkRuleDFA(String gtext, int decision, String expecting)
throws Exception
{
ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(gtext, equeue);
ATN atn = createATN(g, false);
DecisionState blk = atn.decisionToState.get(decision);
checkRuleDFA(g, blk, expecting);
return equeue.all;
}
void checkRuleDFA(Grammar g, DecisionState blk, String expecting)
throws Exception
{
DFA dfa = createDFA(g, blk);
String result = null;
if ( dfa!=null ) result = dfa.toString();
assertEquals(expecting, result);
}
List<ANTLRMessage> checkLexerDFA(String gtext, String expecting)
throws Exception
{
return checkLexerDFA(gtext, LexerGrammar.DEFAULT_MODE_NAME, expecting);
}
List<ANTLRMessage> checkLexerDFA(String gtext, String modeName, String expecting)
throws Exception
{
ErrorQueue equeue = new ErrorQueue();
LexerGrammar g = new LexerGrammar(gtext, equeue);
g.atn = createATN(g, false);
// LexerATNToDFAConverter conv = new LexerATNToDFAConverter(g);
// DFA dfa = conv.createDFA(modeName);
// g.setLookaheadDFA(0, dfa); // only one decision to worry about
//
// String result = null;
// if ( dfa!=null ) result = dfa.toString();
// assertEquals(expecting, result);
//
// return equeue.all;
return null;
}
protected abstract String getLanguage();
protected String execLexer(String grammarFileName,
String grammarStr,
String lexerName,
String input)
{
return execLexer(grammarFileName, grammarStr, lexerName, input, false);
}
@Override
public String execLexer(String grammarFileName,
String grammarStr,
@ -350,31 +49,12 @@ public abstract class BasePythonTest implements RuntimeTestSupport {
null,
lexerName,"-no-listener");
assertTrue(success);
writeFile(tmpdir, "input", input);
writeFile(getTempDirPath(), "input", input);
writeLexerTestFile(lexerName, showDFA);
String output = execModule("Test.py");
return output;
}
public ParseTree execStartRule(String startRuleName, Parser parser)
throws IllegalAccessException, InvocationTargetException,
NoSuchMethodException
{
Method startRule = null;
Object[] args = null;
try {
startRule = parser.getClass().getMethod(startRuleName);
}
catch (NoSuchMethodException nsme) {
// try with int _p arg for recursive func
startRule = parser.getClass().getMethod(startRuleName, int.class);
args = new Integer[] {0};
}
ParseTree result = (ParseTree)startRule.invoke(parser, args);
// System.out.println("parse tree = "+result.toStringTree(parser));
return result;
return execModule("Test.py");
}
@Override
public String execParser(String grammarFileName,
String grammarStr,
String parserName,
@ -405,7 +85,7 @@ public abstract class BasePythonTest implements RuntimeTestSupport {
lexerName,
"-visitor");
assertTrue(success);
writeFile(tmpdir, "input", input);
writeFile(getTempDirPath(), "input", input);
rawBuildRecognizerTestFile(parserName,
lexerName,
listenerName,
@ -434,8 +114,7 @@ public abstract class BasePythonTest implements RuntimeTestSupport {
boolean defaultListener,
String... extraOptions)
{
ErrorQueue equeue =
antlrOnString(getTmpDir(), getLanguage(), grammarFileName, grammarStr, defaultListener, extraOptions);
ErrorQueue equeue = antlrOnString(getTempDirPath(), getLanguage(), grammarFileName, grammarStr, defaultListener, extraOptions);
if (!equeue.errors.isEmpty()) {
return false;
}
@ -465,7 +144,7 @@ public abstract class BasePythonTest implements RuntimeTestSupport {
boolean debug,
boolean trace)
{
this.stderrDuringParse = null;
setParseErrors(null);
if ( parserName==null ) {
writeLexerTestFile(lexerName, false);
}
@ -486,7 +165,7 @@ public abstract class BasePythonTest implements RuntimeTestSupport {
public String execModule(String fileName) {
String pythonPath = locatePython();
String runtimePath = locateRuntime();
File tmpdirFile = new File(tmpdir);
File tmpdirFile = new File(getTempDirPath());
String modulePath = new File(tmpdirFile, fileName).getAbsolutePath();
String inputPath = new File(tmpdirFile, "input").getAbsolutePath();
Path outputPath = tmpdirFile.toPath().resolve("output").toAbsolutePath();
@ -502,7 +181,7 @@ public abstract class BasePythonTest implements RuntimeTestSupport {
stderrVacuum.join();
String output = TestOutputReading.read(outputPath);
if ( stderrVacuum.toString().length()>0 ) {
this.stderrDuringParse = stderrVacuum.toString();
setParseErrors(stderrVacuum.toString());
}
return output;
}
@ -513,31 +192,34 @@ public abstract class BasePythonTest implements RuntimeTestSupport {
return null;
}
private String locateTool(String tool) {
private String locateTool(List<String> tools) {
String[] roots = {
"/opt/local/bin", "/usr/bin/", "/usr/local/bin/",
"/Users/"+System.getProperty("user.name")+"/anaconda3/bin/"
"/Users/"+System.getProperty("user.name")+"/anaconda3/bin/",
"/Users/"+System.getProperty("user.name")+"/opt/anaconda3/bin/"
};
for(String root : roots) {
if(new File(root + tool).exists()) {
return root+tool;
for (String tool : tools) {
if ( new File(root+tool).exists() ) {
return root+tool;
}
}
}
throw new RuntimeException("Could not locate " + tool);
throw new RuntimeException("Could not locate " + tools);
}
protected String locatePython() {
String propName = getPropertyPrefix() + "-python";
String prop = System.getProperty(propName);
if(prop==null || prop.length()==0)
prop = locateTool(getPythonExecutable());
prop = locateTool(getPythonExecutables());
File file = new File(prop);
if(!file.exists())
throw new RuntimeException("Missing system property:" + propName);
return file.getAbsolutePath();
}
protected abstract String getPythonExecutable();
protected abstract List<String> getPythonExecutables();
protected String locateRuntime() { return locateRuntime(getLanguage()); }
@ -553,190 +235,6 @@ public abstract class BasePythonTest implements RuntimeTestSupport {
return runtimeSrc.getPath();
}
private boolean isWindows() {
return System.getProperty("os.name").toLowerCase().contains("windows");
}
// void ambig(List<Message> msgs, int[] expectedAmbigAlts, String expectedAmbigInput)
// throws Exception
// {
// ambig(msgs, 0, expectedAmbigAlts, expectedAmbigInput);
// }
// void ambig(List<Message> msgs, int i, int[] expectedAmbigAlts, String expectedAmbigInput)
// throws Exception
// {
// List<Message> amsgs = getMessagesOfType(msgs, AmbiguityMessage.class);
// AmbiguityMessage a = (AmbiguityMessage)amsgs.get(i);
// if ( a==null ) assertNull(expectedAmbigAlts);
// else {
// assertEquals(a.conflictingAlts.toString(), Arrays.toString(expectedAmbigAlts));
// }
// assertEquals(expectedAmbigInput, a.input);
// }
// void unreachable(List<Message> msgs, int[] expectedUnreachableAlts)
// throws Exception
// {
// unreachable(msgs, 0, expectedUnreachableAlts);
// }
// void unreachable(List<Message> msgs, int i, int[] expectedUnreachableAlts)
// throws Exception
// {
// List<Message> amsgs = getMessagesOfType(msgs, UnreachableAltsMessage.class);
// UnreachableAltsMessage u = (UnreachableAltsMessage)amsgs.get(i);
// if ( u==null ) assertNull(expectedUnreachableAlts);
// else {
// assertEquals(u.conflictingAlts.toString(), Arrays.toString(expectedUnreachableAlts));
// }
// }
List<ANTLRMessage> getMessagesOfType(List<ANTLRMessage> msgs, Class<? extends ANTLRMessage> c) {
List<ANTLRMessage> filtered = new ArrayList<ANTLRMessage>();
for (ANTLRMessage m : msgs) {
if ( m.getClass() == c ) filtered.add(m);
}
return filtered;
}
void checkRuleATN(Grammar g, String ruleName, String expecting) {
ParserATNFactory f = new ParserATNFactory(g);
ATN atn = f.createATN();
DOTGenerator dot = new DOTGenerator(g);
System.out.println(dot.getDOT(atn.ruleToStartState[g.getRule(ruleName).index]));
Rule r = g.getRule(ruleName);
ATNState startState = atn.ruleToStartState[r.index];
ATNPrinter serializer = new ATNPrinter(g, startState);
String result = serializer.asString();
//System.out.print(result);
assertEquals(expecting, result);
}
public void testActions(String templates, String actionName, String action, String expected) throws org.antlr.runtime.RecognitionException {
int lp = templates.indexOf('(');
String name = templates.substring(0, lp);
STGroup group = new STGroupString(templates);
ST st = group.getInstanceOf(name);
st.add(actionName, action);
String grammar = st.render();
ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(grammar, equeue);
if ( g.ast!=null && !g.ast.hasErrors ) {
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
ATNFactory factory = new ParserATNFactory(g);
if ( g.isLexer() ) factory = new LexerATNFactory((LexerGrammar)g);
g.atn = factory.createATN();
CodeGenerator gen = new CodeGenerator(g);
ST outputFileST = gen.generateParser();
String output = outputFileST.render();
//System.out.println(output);
String b = "#" + actionName + "#";
int start = output.indexOf(b);
String e = "#end-" + actionName + "#";
int end = output.indexOf(e);
String snippet = output.substring(start+b.length(),end);
assertEquals(expected, snippet);
}
if ( equeue.size()>0 ) {
System.err.println(equeue.toString());
}
}
protected void checkGrammarSemanticsError(ErrorQueue equeue,
GrammarSemanticsMessage expectedMessage)
throws Exception
{
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
ANTLRMessage m = equeue.errors.get(i);
if (m.getErrorType()==expectedMessage.getErrorType() ) {
foundMsg = m;
}
}
assertNotNull("no error; "+expectedMessage.getErrorType()+" expected", foundMsg);
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs()));
if ( equeue.size()!=1 ) {
System.err.println(equeue);
}
}
protected void checkGrammarSemanticsWarning(ErrorQueue equeue,
GrammarSemanticsMessage expectedMessage)
throws Exception
{
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.warnings.size(); i++) {
ANTLRMessage m = equeue.warnings.get(i);
if (m.getErrorType()==expectedMessage.getErrorType() ) {
foundMsg = m;
}
}
assertNotNull("no error; "+expectedMessage.getErrorType()+" expected", foundMsg);
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
assertEquals(Arrays.toString(expectedMessage.getArgs()), Arrays.toString(foundMsg.getArgs()));
if ( equeue.size()!=1 ) {
System.err.println(equeue);
}
}
protected void checkError(ErrorQueue equeue,
ANTLRMessage expectedMessage)
throws Exception
{
//System.out.println("errors="+equeue);
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
ANTLRMessage m = equeue.errors.get(i);
if (m.getErrorType()==expectedMessage.getErrorType() ) {
foundMsg = m;
}
}
assertTrue("no error; "+expectedMessage.getErrorType()+" expected", !equeue.errors.isEmpty());
assertTrue("too many errors; "+equeue.errors, equeue.errors.size()<=1);
assertNotNull("couldn't find expected error: "+expectedMessage.getErrorType(), foundMsg);
/*
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
*/
assertArrayEquals(expectedMessage.getArgs(), foundMsg.getArgs());
}
public static class FilteringTokenStream extends CommonTokenStream {
public FilteringTokenStream(TokenSource src) { super(src); }
Set<Integer> hide = new HashSet<Integer>();
@Override
protected boolean sync(int i) {
if (!super.sync(i)) {
return false;
}
Token t = get(i);
if ( hide.contains(t.getType()) ) {
((WritableToken)t).setChannel(Token.HIDDEN_CHANNEL);
}
return true;
}
public void setTokenTypeChannel(int ttype, int channel) {
hide.add(ttype);
}
}
protected void mkdir(String dir) {
File f = new File(dir);
f.mkdirs();
}
protected abstract void writeParserTestFile(String parserName,
String lexerName,
String listenerName,
@ -749,213 +247,13 @@ public abstract class BasePythonTest implements RuntimeTestSupport {
protected abstract void writeLexerTestFile(String lexerName, boolean showDFA);
public void writeRecognizer(String parserName, String lexerName,
String listenerName, String visitorName,
String parserStartRuleName, boolean debug, boolean trace) {
if ( parserName==null ) {
writeLexerTestFile(lexerName, debug);
}
else {
writeParserTestFile(parserName,
lexerName,
listenerName,
visitorName,
parserStartRuleName,
debug,
trace);
}
}
protected void eraseFiles(final String filesEndingWith) {
File tmpdirF = new File(tmpdir);
String[] files = tmpdirF.list();
for(int i = 0; files!=null && i < files.length; i++) {
if ( files[i].endsWith(filesEndingWith) ) {
new File(tmpdir+"/"+files[i]).delete();
}
}
}
protected void eraseFiles(File dir) {
String[] files = dir.list();
for(int i = 0; files!=null && i < files.length; i++) {
new File(dir,files[i]).delete();
}
}
@Override
public void eraseTempDir() {
boolean doErase = true;
String propName = getPropertyPrefix() + "-erase-test-dir";
String prop = System.getProperty(propName);
if(prop!=null && prop.length()>0)
doErase = Boolean.getBoolean(prop);
if(doErase) {
File tmpdirF = new File(tmpdir);
if ( tmpdirF.exists() ) {
eraseFiles(tmpdirF);
tmpdirF.delete();
}
}
}
protected void eraseTempPyCache() {
File tmpdirF = new File(tmpdir+"/__pycache__");
File tmpdirF = new File(getTempTestDir() + "/__pycache__");
if ( tmpdirF.exists() ) {
eraseFiles(tmpdirF);
eraseFilesInDir(tmpdirF);
tmpdirF.delete();
}
}
public String getFirstLineOfException() {
if ( this.stderrDuringParse ==null ) {
return null;
}
String[] lines = this.stderrDuringParse.split("\n");
String prefix="Exception in thread \"main\" ";
return lines[0].substring(prefix.length(),lines[0].length());
}
/**
* When looking at a result set that consists of a Map/HashTable
* we cannot rely on the output order, as the hashing algorithm or other aspects
* of the implementation may be different on differnt JDKs or platforms. Hence
* we take the Map, convert the keys to a List, sort them and Stringify the Map, which is a
* bit of a hack, but guarantees that we get the same order on all systems. We assume that
* the keys are strings.
*
* @param m The Map that contains keys we wish to return in sorted order
* @return A string that represents all the keys in sorted order.
*/
public <K, V> String sortMapToString(Map<K, V> m) {
// Pass in crap, and get nothing back
//
if (m == null) {
return null;
}
System.out.println("Map toString looks like: " + m.toString());
// Sort the keys in the Map
//
TreeMap<K, V> nset = new TreeMap<K, V>(m);
System.out.println("Tree map looks like: " + nset.toString());
return nset.toString();
}
public List<String> realElements(List<String> elements) {
return elements.subList(Token.MIN_USER_TOKEN_TYPE, elements.size());
}
public void assertNotNullOrEmpty(String message, String text) {
assertNotNull(message, text);
assertFalse(message, text.isEmpty());
}
public void assertNotNullOrEmpty(String text) {
assertNotNull(text);
assertFalse(text.isEmpty());
}
public static class IntTokenStream implements TokenStream {
IntegerList types;
int p=0;
public IntTokenStream(IntegerList types) { this.types = types; }
@Override
public void consume() { p++; }
@Override
public int LA(int i) { return LT(i).getType(); }
@Override
public int mark() {
return index();
}
@Override
public int index() { return p; }
@Override
public void release(int marker) {
seek(marker);
}
@Override
public void seek(int index) {
p = index;
}
@Override
public int size() {
return types.size();
}
@Override
public String getSourceName() {
return null;
}
@Override
public Token LT(int i) {
CommonToken t;
int rawIndex = p + i - 1;
if ( rawIndex>=types.size() ) t = new CommonToken(Token.EOF);
else t = new CommonToken(types.get(rawIndex));
t.setTokenIndex(rawIndex);
return t;
}
@Override
public Token get(int i) {
return new org.antlr.v4.runtime.CommonToken(types.get(i));
}
@Override
public TokenSource getTokenSource() {
return null;
}
@Override
public String getText() {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Interval interval) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(RuleContext ctx) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Token start, Token stop) {
throw new UnsupportedOperationException("can't give strings");
}
}
/** Sort a list */
public <T extends Comparable<? super T>> List<T> sort(List<T> data) {
List<T> dup = new ArrayList<T>();
dup.addAll(data);
Collections.sort(dup);
return dup;
}
/** Return map sorted by key */
public <K extends Comparable<? super K>,V> LinkedHashMap<K,V> sort(Map<K,V> data) {
LinkedHashMap<K,V> dup = new LinkedHashMap<K, V>();
List<K> keys = new ArrayList<K>();
keys.addAll(data.keySet());
Collections.sort(keys);
for (K k : keys) {
dup.put(k, data.get(k));
}
return dup;
}
}

Some files were not shown because too many files have changed in this diff Show More