forked from jasder/antlr
Merge remote-tracking branch 'refs/remotes/antlr/master'
This commit is contained in:
Floor
commit
96cee174ce
|
|
@ -1,8 +1,8 @@
|
|||
<!--
|
||||
Before submitting an issue to ANTLR, please check off these boxes:
|
||||
|
||||
- [ ] I am not submitting a question on how to use ANTLR; instead, go to [antlr4-discussion google group](https://groups.google.com/forum/#!forum/antlr-discussion) or ask at [stackoverflow](http://stackoverflow.com/questions/tagged/antlr4)
|
||||
- [ ] I have done a search of the existing issues to make sure I'm not sending in a duplicate
|
||||
|
||||
[cut]
|
||||
Please include information about the expected behavior, actual behavior, and the smallest grammar or code that reproduces the behavior. If appropriate, please indicate the code generation targets such as Java, C#, ... Pointers into offending code regions are also very welcome.
|
||||
[/cut]
|
||||
-->
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
[cut]
|
||||
<!--
|
||||
Thank you for proposing a contribution to the ANTLR project. In order to accept changes from the outside world, all contributors must "sign" the [contributors.txt](https://github.com/antlr/antlr4/blob/master/contributors.txt) contributors certificate of origin. It's an unfortunate reality of today's fuzzy and bizarre world of open-source ownership.
|
||||
|
||||
Make sure you are already in the contributors.txt file or add a commit to this pull request with the appropriate change. Thanks!
|
||||
[/cut]
|
||||
-->
|
||||
|
|
@ -31,6 +31,9 @@ __pycache__/
|
|||
|
||||
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||
*.userprefs
|
||||
*.user
|
||||
.vs/
|
||||
project.lock.json
|
||||
|
||||
# Build results
|
||||
[Dd]ebug/
|
||||
|
|
|
|||
10
.travis.yml
10
.travis.yml
|
|
@ -45,12 +45,22 @@ matrix:
|
|||
- os: linux
|
||||
jdk: oraclejdk7
|
||||
env: TARGET=csharp
|
||||
- os: linux
|
||||
jdk: oraclejdk7
|
||||
dist: trusty
|
||||
env: TARGET=dotnet
|
||||
- os: linux
|
||||
jdk: oraclejdk7
|
||||
env: TARGET=python2
|
||||
- os: linux
|
||||
jdk: oraclejdk7
|
||||
env: TARGET=python3
|
||||
addons:
|
||||
apt:
|
||||
sources:
|
||||
- deadsnakes # source required so it finds the package definition below
|
||||
packages:
|
||||
- python3.5
|
||||
- os: linux
|
||||
jdk: oraclejdk7
|
||||
env: TARGET=javascript
|
||||
|
|
|
|||
|
|
@ -3,6 +3,4 @@
|
|||
set -euo pipefail
|
||||
|
||||
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
|
||||
sudo add-apt-repository ppa:fkrull/deadsnakes -y
|
||||
sudo add-apt-repository ppa:rwky/nodejs -y
|
||||
sudo apt-get update -qq
|
||||
|
|
|
|||
|
|
@ -3,8 +3,6 @@
|
|||
set -euo pipefail
|
||||
|
||||
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
|
||||
sudo add-apt-repository ppa:fkrull/deadsnakes -y
|
||||
sudo add-apt-repository ppa:rwky/nodejs -y
|
||||
sudo apt-get update -qq
|
||||
echo "deb http://download.mono-project.com/repo/debian wheezy/snapshots/3.12.1 main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list
|
||||
sudo apt-get install -qq mono-complete
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# install dotnet
|
||||
sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ trusty main" > /etc/apt/sources.list.d/dotnetdev.list'
|
||||
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 417A0893
|
||||
sudo apt-get update
|
||||
|
||||
sudo apt-get install dotnet-dev-1.0.0-preview2.1-003177
|
||||
|
||||
# install mvn
|
||||
wget http://apache.mirrors.lucidnetworks.net/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz && \
|
||||
wget https://www.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz.md5 && \
|
||||
echo "$(cat apache-maven-3.3.9-bin.tar.gz.md5) apache-maven-3.3.9-bin.tar.gz" > apache-maven-3.3.9-bin.tar.gz.md5 && \
|
||||
md5sum -c *.md5
|
||||
|
||||
sudo rm -rf /usr/local/maven/ && sudo mkdir -p /usr/local/maven && \
|
||||
sudo tar xzvf apache-maven-3.3.9-bin.tar.gz -C /usr/local/maven --strip-components=1
|
||||
|
||||
mvn -v
|
||||
|
||||
|
|
@ -3,8 +3,6 @@
|
|||
set -euo pipefail
|
||||
|
||||
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
|
||||
sudo add-apt-repository ppa:fkrull/deadsnakes -y
|
||||
sudo add-apt-repository ppa:rwky/nodejs -y
|
||||
sudo apt-get update -qq
|
||||
eval "$(sudo gimme 1.7.3)"
|
||||
( go version ; go env ) || true
|
||||
|
|
|
|||
|
|
@ -3,6 +3,4 @@
|
|||
set -euo pipefail
|
||||
|
||||
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
|
||||
sudo add-apt-repository ppa:fkrull/deadsnakes -y
|
||||
sudo add-apt-repository ppa:rwky/nodejs -y
|
||||
sudo apt-get update -qq
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
set -euo pipefail
|
||||
|
||||
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
|
||||
sudo add-apt-repository ppa:fkrull/deadsnakes -y
|
||||
sudo add-apt-repository ppa:rwky/nodejs -y
|
||||
sudo apt-get update -qq
|
||||
curl -sL https://deb.nodesource.com/setup_0.12 | sudo -E bash -
|
||||
sudo apt-get install -qq nodejs
|
||||
node --version
|
||||
|
|
|
|||
|
|
@ -3,7 +3,5 @@
|
|||
set -euo pipefail
|
||||
|
||||
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
|
||||
sudo add-apt-repository ppa:fkrull/deadsnakes -y
|
||||
sudo add-apt-repository ppa:rwky/nodejs -y
|
||||
sudo apt-get update -qq
|
||||
python --version
|
||||
|
|
|
|||
|
|
@ -2,9 +2,4 @@
|
|||
|
||||
set -euo pipefail
|
||||
|
||||
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
|
||||
sudo add-apt-repository ppa:fkrull/deadsnakes -y
|
||||
sudo add-apt-repository ppa:rwky/nodejs -y
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -qq python3.5
|
||||
python3 --version
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
#!/bin/bash
|
||||
|
||||
mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=csharp.* -Dantlr-csharp-netstandard=true test
|
||||
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
# Contributing to ANTLR 4
|
||||
|
||||
1. [Fork](https://help.github.com/articles/fork-a-repo) the [antlr/antlr4 repo](https://github.com/antlr/antlr4)
|
||||
2. Install and configure [EditorConfig](http://editorconfig.org/) so your text editor or IDE uses the ANTLR 4 coding style
|
||||
3. [Build ANTLR 4](doc/building-antlr.md)
|
||||
4. [Run the ANTLR project unit tests](doc/antlr-project-testing.md)
|
||||
5. Create a [pull request](https://help.github.com/articles/using-pull-requests/) including your change
|
||||
|
||||
|
||||
**Note:** You must sign the `contributors.txt` certificate of origin with your pull request if you've not done so before.
|
||||
28
LICENSE.txt
28
LICENSE.txt
|
|
@ -1,5 +1,5 @@
|
|||
[The "BSD 3-clause license"]
|
||||
Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions
|
||||
|
|
@ -24,3 +24,29 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
||||
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
=====
|
||||
|
||||
MIT License for codepointat.js from https://git.io/codepointat
|
||||
MIT License for fromcodepoint.js from https://git.io/vDW1m
|
||||
|
||||
Copyright Mathias Bynens <https://mathiasbynens.be/>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to
|
||||
permit persons to whom the Software is furnished to do so, subject to
|
||||
the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be
|
||||
included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ ANTLR project lead and supreme dictator for life
|
|||
* David Sisson (C++ initial target and test)
|
||||
* [Janyou](https://github.com/janyou) (Swift target)
|
||||
* [Ewan Mellor](https://github.com/ewanmellor), [Hanzhou Shi](https://github.com/hanjoes) (Swift target merging)
|
||||
* [Ben Hamilton](https://github.com/bhamiltoncx) (Full Unicode support in serialized ATN and all languages' runtimes for code points > U+FFFF)
|
||||
|
||||
## Useful information
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
~ Use of this file is governed by the BSD 3-clause license that
|
||||
~ can be found in the LICENSE.txt file in the project root.
|
||||
-->
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<!--
|
||||
~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
~ Use of this file is governed by the BSD 3-clause license that
|
||||
~ can be found in the LICENSE.txt file in the project root.
|
||||
-->
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
<parent>
|
||||
<groupId>org.antlr</groupId>
|
||||
<artifactId>antlr4-master</artifactId>
|
||||
<version>4.6.1-SNAPSHOT</version>
|
||||
<version>4.7.1-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>antlr4-maven-plugin</artifactId>
|
||||
<packaging>maven-plugin</packaging>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!--
|
||||
~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
~ Use of this file is governed by the BSD 3-clause license that
|
||||
~ can be found in the LICENSE.txt file in the project root.
|
||||
-->
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<!--
|
||||
~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
~ Use of this file is governed by the BSD 3-clause license that
|
||||
~ can be found in the LICENSE.txt file in the project root.
|
||||
-->
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<!--
|
||||
~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
~ Use of this file is governed by the BSD 3-clause license that
|
||||
~ can be found in the LICENSE.txt file in the project root.
|
||||
-->
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<!--
|
||||
~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
~ Use of this file is governed by the BSD 3-clause license that
|
||||
~ can be found in the LICENSE.txt file in the project root.
|
||||
-->
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<!--
|
||||
~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
~ Use of this file is governed by the BSD 3-clause license that
|
||||
~ can be found in the LICENSE.txt file in the project root.
|
||||
-->
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<!--
|
||||
~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
~ Use of this file is governed by the BSD 3-clause license that
|
||||
~ can be found in the LICENSE.txt file in the project root.
|
||||
-->
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
ANTLR Project Contributors Certification of Origin and Rights
|
||||
ANTLR Project Contributors Certification of Origin and Rights
|
||||
|
||||
All contributors to ANTLR v4 must formally agree to abide by this
|
||||
certificate of origin by signing on the bottom with their github
|
||||
|
|
@ -129,3 +129,17 @@ YYYY/MM/DD, github id, Full name, email
|
|||
2016/12/11, Gaulouis, Gaulouis, gaulouis.com@gmail.com
|
||||
2016/12/22, akosthekiss, Akos Kiss, akiss@inf.u-szeged.hu
|
||||
2016/12/24, adrpo, Adrian Pop, adrian.pop@liu.se
|
||||
2017/01/11, robertbrignull, Robert Brignull, robertbrignull@gmail.com
|
||||
2017/01/13, marcelo-rocha, Marcelo Rocha, mcrocha@gmail.com
|
||||
2017/01/23, bhamiltoncx, Ben Hamilton, bhamiltoncx+antlr@gmail.com
|
||||
2017/01/18, mshockwave, Bekket McClane, yihshyng223@gmail.com
|
||||
2017/02/10, lionelplessis, Lionel Plessis, lionelplessis@users.noreply.github.com
|
||||
2017/02/14, lecode-official, David Neumann, david.neumann@lecode.de
|
||||
2017/02/14, xied75, Dong Xie, xied75@gmail.com
|
||||
2017/02/20, Thomasb81, Thomas Burg, thomasb81@gmail.com
|
||||
2017/02/26, jvasileff, John Vasileff, john@vasileff.com
|
||||
2017/03/08, harry-tallbelt, Igor Vysokopoyasny, harry.tallbelt@gmail.com
|
||||
2017/03/09, teverett, Tom Everett, tom@khubla.com
|
||||
2017/03/03, chund, Christian Hund, christian.hund@gmail.com
|
||||
2017/03/15, robertvanderhulst, Robert van der Hulst, robert@xsharp.eu
|
||||
2017/03/28, cmd-johnson, Jonas Auer, jonas.auer.94@gmail.com
|
||||
|
|
|
|||
|
|
@ -239,7 +239,7 @@ AnnotatingErrorListener.prototype.syntaxError = function(recognizer, offendingSy
|
|||
With this, all that remains to be done is plug the listener in when we parse the code. Here is how I do it:
|
||||
```js
|
||||
var validate = function(input) {
|
||||
var stream = new antlr4.InputStream(input);
|
||||
var stream = CharStreams.fromString(input);
|
||||
var lexer = new mylanguage.MyLexer(stream);
|
||||
var tokens = new antlr4.CommonTokenStream(lexer);
|
||||
var parser = new mylanguage.MyParser(tokens);
|
||||
|
|
@ -252,4 +252,4 @@ var validate = function(input) {
|
|||
};
|
||||
```
|
||||
You know what? That's it! You now have an ACE editor that does syntax validation using ANTLR! I hope you find this useful, and simple enough to get started.
|
||||
WNow wait, hey! How do you debug this? Well, as usual, using Chrome, since no other browser is able to debug worker code. What a shame...
|
||||
Now wait, hey! How do you debug this? Well, as usual, using Chrome, since no other browser is able to debug worker code. What a shame...
|
||||
|
|
|
|||
|
|
@ -256,6 +256,14 @@ where `<TARGET>` is replaced with Java, Cpp, CSharp, Python2, ... in the various
|
|||
|
||||
In order to turn off a test for a particular target, we need to use the `ignore` method. Given a target name, a descriptor object can decide whether to ignore the test. This is not always convenient but it is fully general and works well for the one case we have now where we have to ignore `Visitor` tests in all targets except JavaScript.
|
||||
|
||||
### Target API/library testing
|
||||
|
||||
Some parts of the runtime API need to be tested with code written specifically in the target language. For example, you can see all of the Java runtime API tests here:
|
||||
|
||||
[https://github.com/antlr/antlr4/tree/master/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api](https://github.com/antlr/antlr4/tree/master/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api)
|
||||
|
||||
Notice that it is under an `api` dir. The directory above is where all of the `Test*` files go.
|
||||
|
||||
### Cross-language actions embedded within grammars
|
||||
|
||||
To get:
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ Resolving deltas: 100% (31898/31898), done.
|
|||
Checking connectivity... done.
|
||||
$ cd antlr4
|
||||
$ export MAVEN_OPTS="-Xmx1G" # don't forget this on linux
|
||||
$ mvn clean # must be separate, not part of install/compile
|
||||
$ mvn -DskipTests install
|
||||
...
|
||||
[INFO] ------------------------------------------------------------------------
|
||||
|
|
@ -61,7 +62,7 @@ $ mvn -DskipTests install
|
|||
[INFO] ------------------------------------------------------------------------
|
||||
```
|
||||
|
||||
We do `install` not `compile` as tool tests and such refer to modules that must be pulled from the maven install local cache.
|
||||
**NOTE:** We do `install` not `compile` as tool tests and such refer to modules that must be pulled from the maven install local cache.
|
||||
|
||||
# Installing libs to mvn cache locally
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# C++
|
||||
|
||||
The C++ target supports all platforms that can either run MS Visual Studio 2013 (or newer), XCode 7 (or newer) or CMake (C++11 required). All build tools can either create static or dynamic libraries, both as 64bit or 32bit arch. Additionally, XCode can create an iOS library.
|
||||
The C++ target supports all platforms that can either run MS Visual Studio 2013 (or newer), XCode 7 (or newer) or CMake (C++11 required). All build tools can either create static or dynamic libraries, both as 64bit or 32bit arch. Additionally, XCode can create an iOS library. Also see [Antlr4 for C++ with CMake: A practical example](http://blorente.me//Antlr-,-C++-and-CMake-Wait-what.html).
|
||||
|
||||
## How to create a C++ lexer or parser?
|
||||
This is pretty much the same as creating a Java lexer or parser, except you need to specify the language target, for example:
|
||||
|
|
@ -106,16 +106,12 @@ For gcc and clang it is possible to use the `-fvisibility=hidden` setting to hid
|
|||
Since C++ has no built-in memory management we need to take extra care. For that we rely mostly on smart pointers, which however might cause time penalties or memory side effects (like cyclic references) if not used with care. Currently however the memory household looks very stable. Generally, when you see a raw pointer in code consider this as being managed elsewehere. You should never try to manage such a pointer (delete, assign to smart pointer etc.).
|
||||
|
||||
### Unicode Support
|
||||
Encoding is mostly an input issue, i.e. when the lexer converts text input into lexer tokens. The parser is completely encoding unaware. However, lexer input in the grammar is defined by character ranges with either a single member (e.g. 'a' or [a] or [abc]), an explicit range (e.g. 'a'..'z' or [a-z]), the full Unicode range (for a wildcard) and the full Unicode range minus a sub range (for negated ranges, e.g. ~[a]). The explicit ranges (including single member ranges) are encoded in the serialized ATN by 16bit numbers, hence cannot reach beyond 0xFFFF (the Unicode BMP), while the implicit ranges can include any value (and hence support the full Unicode set, up to 0x10FFFF).
|
||||
Encoding is mostly an input issue, i.e. when the lexer converts text input into lexer tokens. The parser is completely encoding unaware.
|
||||
|
||||
> An interesting side note here is that the Java target fully supports Unicode as well, despite the inherent limitations from the serialized ATN. That's possible because the Java String class represents characters beyond the BMP as surrogate pairs (two 16bit values) and even reads them as 2 separate input characters. To make this work a character range for an identifier in a grammar must include the surrogate pairs area (for a Java parser).
|
||||
|
||||
The C++ target however always expects UTF-8 input (either in a string or via a wide stream) which is then converted to UTF-32 (a char32_t array) and fed to the lexer. ANTLR, when parsing your grammar, limits character ranges explicitly to the BMP currently. So, in order to allow specifying the full Unicode set the C++ target uses a little trick: whenever an explicit character range includes the (unused) codepoint 0xFFFF in a grammar it is silently extended to the full Unicode range. It's clear that this is an all-or-nothing solution. You cannot define a subset of Unicode codepoints > 0xFFFF that way. This can only be solved if ANTLR supports larger character intervals.
|
||||
|
||||
The differences in handling characters beyond the BMP leads to a difference between Java and C++ lexers: the character offsets may not concur. This is because Java reads two 16bit values per Unicode char (if that falls into the surrogate area) while a C++ parser only reads one 32bit value. That usually doesn't have practical consequences, but might confuse people when comparing token positions.
|
||||
The C++ target always expects UTF-8 input (either in a string or stream) which is then converted to UTF-32 (a char32_t array) and fed to the lexer.
|
||||
|
||||
### Named Actions
|
||||
In order to help customizing the generated files there are a number of additional socalled **named actions**. These actions are tight to specific areas in the generated code and allow to add custom (target specific) code. All targets support these actions
|
||||
In order to help customizing the generated files there are a number of additional socalled **named actions**. These actions are tight to specific areas in the generated code and allow to add custom (target specific) code. All targets support these actions
|
||||
|
||||
* @parser::header
|
||||
* @parser::members
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ using Antlr4.Runtime;
|
|||
|
||||
public void MyParseMethod() {
|
||||
String input = "your text to parse here";
|
||||
AntlrInputStream stream = new InputStream(input);
|
||||
ICharStream stream = CharStreams.fromString(input);
|
||||
ITokenSource lexer = new MyGrammarLexer(stream);
|
||||
ITokenStream tokens = new CommonTokenStream(lexer);
|
||||
MyGrammarParser parser = new MyGrammarParser(tokens);
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ Make sure to use two-stage parsing. See example in [bug report](https://github.c
|
|||
|
||||
```Java
|
||||
|
||||
CharStream input = new ANTLRFileStream(args[0]);
|
||||
CharStream input = CharStreams.fromPath(Paths.get(args[0]));
|
||||
ExprLexer lexer = new ExprLexer(input);
|
||||
CommonTokenStream tokens = new CommonTokenStream(lexer);
|
||||
ExprParser parser = new ExprParser(tokens);
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ Links in the documentation refer to various sections of the book but have been r
|
|||
<a href=""><img src=images/tpantlr2.png width=120></a>
|
||||
<a href=""><img src=images/tpdsl.png width=120></a>
|
||||
<a href="https://www.youtube.com/watch?v=OAoA3E-cyug"><img src=images/teronbook.png width=250></a>
|
||||
|
||||
|
||||
This documentation is a reference and summarizes grammar syntax and the key semantics of ANTLR grammars. The source code for all examples in the book, not just this chapter, are free at the publisher's website. The following video is a general tour of ANTLR 4 and includes a description of how to use parse tree listeners to process Java files easily:
|
||||
|
||||
<a href="https://vimeo.com/59285751"><img src=images/tertalk.png width=200></a>
|
||||
|
|
@ -51,8 +51,10 @@ This documentation is a reference and summarizes grammar syntax and the key sema
|
|||
|
||||
* [Runtime Libraries and Code Generation Targets](targets.md)
|
||||
|
||||
* [Unicode U+FFFF, U+10FFFF character streams](unicode.md)
|
||||
|
||||
* [Parsing binary streams](parsing-binary-files.md)
|
||||
|
||||
|
||||
* [Parser and lexer interpreters](interpreters.md)
|
||||
|
||||
* [Resources](resources.md)
|
||||
|
|
@ -61,6 +63,8 @@ This documentation is a reference and summarizes grammar syntax and the key sema
|
|||
|
||||
* [Building ANTLR itself](building-antlr.md)
|
||||
|
||||
* [Contributing to ANTLR](/CONTRIBUTING.md)
|
||||
|
||||
* [Cutting an ANTLR Release](releasing-antlr.md)
|
||||
|
||||
* [ANTLR project unit tests](antlr-project-testing.md)
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ public static ParseTree parse(String fileName,
|
|||
throws IOException
|
||||
{
|
||||
final Grammar g = Grammar.load(combinedGrammarFileName);
|
||||
LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRFileStream(fileName));
|
||||
LexerInterpreter lexEngine = g.createLexerInterpreter(CharStreams.fromPath(Paths.get(fileName)));
|
||||
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
|
||||
ParserInterpreter parser = g.createParserInterpreter(tokens);
|
||||
ParseTree t = parser.parse(g.getRule(startRule).index);
|
||||
|
|
@ -58,7 +58,7 @@ public static ParseTree parse(String fileNameToParse,
|
|||
{
|
||||
final LexerGrammar lg = (LexerGrammar) Grammar.load(lexerGrammarFileName);
|
||||
final Grammar pg = Grammar.load(parserGrammarFileName, lg);
|
||||
ANTLRFileStream input = new ANTLRFileStream(fileNameToParse);
|
||||
CharStream input = CharStreams.fromPath(Paths.get(fileNameToParse));
|
||||
LexerInterpreter lexEngine = lg.createLexerInterpreter(input);
|
||||
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
|
||||
ParserInterpreter parser = pg.createParserInterpreter(tokens);
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ Now a fully functioning script might look like the following:
|
|||
|
||||
```javascript
|
||||
var input = "your text to parse here"
|
||||
var chars = new antlr4.InputStream(input);
|
||||
var chars = CharStreams.fromString(input);
|
||||
var lexer = new MyGrammarLexer.MyGrammarLexer(chars);
|
||||
var tokens = new antlr4.CommonTokenStream(lexer);
|
||||
var parser = new MyGrammarParser.MyGrammarParser(tokens);
|
||||
|
|
|
|||
|
|
@ -58,13 +58,37 @@ Match that character or sequence of characters. E.g., ’while’ or ’=’.</t
|
|||
|
||||
<tr>
|
||||
<td>[char set]</td><td>
|
||||
Match one of the characters specified in the character set. Interpret x-y as set of characters between range x and y, inclusively. The following escaped characters are interpreted as single special characters: \n, \r, \b, \t, and \f. To get ], \, or - you must escape them with \. You can also use Unicode character specifications: \uXXXX. Here are a few examples:
|
||||
<p>Match one of the characters specified in the character set. Interpret <tt>x-y</tt> as the set of characters between range <tt>x</tt> and <tt>y</tt>, inclusively. The following escaped characters are interpreted as single special characters: <tt>\n</tt>, <tt>\r</tt>, <tt>\b</tt>, <tt>\t</tt>, <tt>\f</tt>, <tt>\uXXXX</tt>, and <tt>\u{XXXXXX}</tt>. To get <tt>]</tt>, <tt>\</tt>, or <tt>-</tt> you must escape them with <tt>\</tt>.</p>
|
||||
|
||||
<p>You can also include all characters matching Unicode properties (general category, boolean, or enumerated including scripts and blocks) with <tt>\p{PropertyName}</tt> or <tt>\p{EnumProperty=Value}</tt>. (You can invert the test with <tt>\P{PropertyName}</tt> or <tt>\P{EnumProperty=Value}</tt>).</p>
|
||||
|
||||
<p>For a list of valid Unicode property names, see <a href="http://unicode.org/reports/tr44/#Properties">Unicode Standard Annex #44</a>. (ANTLR also supports <a href="http://unicode.org/reports/tr44/#General_Category_Values">short and long Unicode general category names and values</a> like <tt>\p{Lu}</tt>, <tt>\p{Z}</tt>, <tt>\p{Symbol}</tt>, <tt>\p{Blk=Latin_1_Sup}</tt>, and <tt>\p{Block=Latin_1_Supplement}</tt>.)</p>
|
||||
|
||||
<p>As a shortcut for <tt>\p{Block=Latin_1_Supplement}</tt>, you can refer to blocks using <a href="http://www.unicode.org/Public/UCD/latest/ucd/Blocks.txt">Unicode block names</a> prefixed with <tt>In</tt> and with spaces changed to <tt>_</tt>. For example: <tt>\p{InLatin_1_Supplement}</tt>, <tt>\p{InYijing_Hexagram_Symbols}</tt>, and <tt>\p{InAncient_Greek_Numbers}</tt>.</p>
|
||||
|
||||
<p>A few extra properties are supported:</p>
|
||||
<ul>
|
||||
<li><tt>\p{Extended_Pictographic}</tt> (see <a href="http://unicode.org/reports/tr35/">UTS #35</a>)</li>
|
||||
<li><tt>\p{EmojiPresentation=EmojiDefault}</tt> (code points which have colorful emoji-style presentation by default but which can also be displayed text-style)</li>
|
||||
<li><tt>\p{EmojiPresentation=TextDefault}</tt> (code points which have black-and-white text-style presentation by default but which can also be displayed emoji-style)</li>
|
||||
<li><tt>\p{EmojiPresentation=Text}</tt> (code points which have only black-and-white text-style and lack a colorful emoji-style presentation)</li>
|
||||
</ul>
|
||||
|
||||
<p>Property names are <b>case-insensitive</b>, and <tt>_</tt> and <tt>-</tt> are treated identically</p>
|
||||
|
||||
<p>Here are a few examples:</p>
|
||||
|
||||
<pre>
|
||||
WS : [ \n\u000D] -> skip ; // same as [ \n\r]
|
||||
|
||||
|
||||
UNICODE_WS : [\p{White_Space}] -> skip; // match all Unicode whitespace
|
||||
|
||||
ID : [a-zA-Z] [a-zA-Z0-9]* ; // match usual identifier spec
|
||||
|
||||
|
||||
UNICODE_ID : [\p{Alpha}\p{General_Category=Other_Letter}] [\p{Alnum}\p{General_Category=Other_Letter}]* ; // match full Unicode alphabetic ids
|
||||
|
||||
EMOJI : [\u{1F4A9}\u{1F926}] ; // note Unicode code points > U+FFFF
|
||||
|
||||
DASHBRACK : [\-\]]+ ; // match - or ] one or more times
|
||||
</pre>
|
||||
</td>
|
||||
|
|
|
|||
|
|
@ -81,14 +81,18 @@ These more or less correspond to `isJavaIdentifierPart` and `isJavaIdentifierSta
|
|||
|
||||
ANTLR does not distinguish between character and string literals as most languages do. All literal strings one or more characters in length are enclosed in single quotes such as `’;’`, `’if’`, `’>=’`, and `’\’'` (refers to the one-character string containing the single quote character). Literals never contain regular expressions.
|
||||
|
||||
Literals can contain Unicode escape sequences of the form `\uXXXX`, where XXXX is the hexadecimal Unicode character value. For example, `’\u00E8’` is the French letter with a grave accent: `’è’`. ANTLR also understands the usual special escape sequences: `’\n’` (newline), `’\r’` (carriage return), `’\t’` (tab), `’\b’` (backspace), and `’\f’` (form feed). You can use Unicode characters directly within literals or use the Unicode escape sequences:
|
||||
Literals can contain Unicode escape sequences of the form `’\uXXXX’` (for Unicode code points up to `’U+FFFF’`) or `’\u{XXXXXX}’` (for all Unicode code points), where `’XXXX’` is the hexadecimal Unicode code point value.
|
||||
|
||||
For example, `’\u00E8’` is the French letter with a grave accent: `’è’`, and `’\u{1F4A9}’` is the famous emoji: `’💩’`.
|
||||
|
||||
ANTLR also understands the usual special escape sequences: `’\n’` (newline), `’\r’` (carriage return), `’\t’` (tab), `’\b’` (backspace), and `’\f’` (form feed). You can use Unicode code points directly within literals or use the Unicode escape sequences:
|
||||
|
||||
```
|
||||
grammar Foreign;
|
||||
a : '外' ;
|
||||
```
|
||||
|
||||
The recognizers that ANTLR generates assume a character vocabulary containing all Unicode characters. The input file encoding assumed by the runtime library depends on the target language. For the Java target, the runtime library assumes files are in UTF-8. Using the constructors, you can specify a different encoding. See, for example, ANTLR’s `ANTLRFileStream`.
|
||||
The recognizers that ANTLR generates assume a character vocabulary containing all Unicode characters. The input file encoding assumed by the runtime library depends on the target language. For the Java target, the runtime library assumes files are in UTF-8. Using the factory methods in `CharStreams`, you can specify a different encoding.
|
||||
|
||||
## Actions
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ grammar IP;
|
|||
|
||||
file : ip+ (MARKER ip)* ;
|
||||
|
||||
ip : BYTE '.' BYTE '.' BYTE '.' BYTE ;
|
||||
ip : BYTE BYTE BYTE BYTE ;
|
||||
|
||||
MARKER : '\u00CA' '\u00FE' ;
|
||||
BYTE : '\u0000'..'\u00FF' ;
|
||||
|
|
@ -42,7 +42,7 @@ public class WriteBinaryFile {
|
|||
};
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
Files.write(new File("resources/ips").toPath(), bytes);
|
||||
Files.write(new File("/tmp/ips").toPath(), bytes);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
|
@ -50,14 +50,15 @@ public class WriteBinaryFile {
|
|||
Now we need to create a stream of bytes satisfactory to ANTLR, which is as simple as:
|
||||
|
||||
```java
|
||||
ANTLRFileStream bytesAsChar = new ANTLRFileStream("resources/ips", "ISO-8859-1");
|
||||
CharStream bytesAsChar = CharStreams.fromFileName("/tmp/ips", StandardCharsets.ISO_8859_1);
|
||||
```
|
||||
|
||||
The `ISO-8859-1` encoding is just the 8-bit char encoding for LATIN-1, which effectively tells the stream to treat each byte as a character. That's what we want. Then we have the usual test rig:
|
||||
|
||||
|
||||
```java
|
||||
ANTLRFileStream bytesAsChar = new ANTLRFileStream("resources/ips", "ISO-8859-1");
|
||||
//ANTLRFileStream bytesAsChar = new ANTLRFileStream("/tmp/ips", "ISO-8859-1"); DEPRECATED in 4.7
|
||||
CharStream bytesAsChar = CharStreams.fromFileName("/tmp/ips", StandardCharsets.ISO_8859_1);
|
||||
IPLexer lexer = new IPLexer(bytesAsChar);
|
||||
CommonTokenStream tokens = new CommonTokenStream(lexer);
|
||||
IPParser parser = new IPParser(tokens);
|
||||
|
|
@ -93,6 +94,8 @@ We can't just print out the text because we are not reading in text. We need to
|
|||
|
||||
## Custom stream
|
||||
|
||||
(*ANTLRFileStream was deprecated in 4.7*)
|
||||
|
||||
If you want to play around with the stream, you can. Here's an example that alters how "text" is computed from the byte stream (which changes how tokens print out their text as well):
|
||||
|
||||
```java
|
||||
|
|
@ -124,7 +127,7 @@ class BinaryANTLRFileStream extends ANTLRFileStream {
|
|||
The new test code starts out like this:
|
||||
|
||||
```java
|
||||
ANTLRFileStream bytesAsChar = new BinaryANTLRFileStream("resources/ips");
|
||||
ANTLRFileStream bytesAsChar = new BinaryANTLRFileStream("/tmp/ips");
|
||||
IPLexer lexer = new IPLexer(bytesAsChar);
|
||||
...
|
||||
```
|
||||
|
|
@ -155,9 +158,9 @@ Error handling proceeds exactly like any other parser. For example, let's alter
|
|||
|
||||
```java
|
||||
public static final byte[] bytes = {
|
||||
(byte)172, '.', 0, '.', '.', 1, (byte)0xCA, (byte)0xFE, // OOOPS
|
||||
(byte)10, '.', 10, '.', 10, '.', 1, (byte)0xCA, (byte)0xFE,
|
||||
(byte)10, '.', 10, '.', 10, '.', 99
|
||||
(byte)172, 0, 1, (byte)0xCA, (byte)0xFE, // OOOPS
|
||||
(byte)10, 10, 10, 1, (byte)0xCA, (byte)0xFE,
|
||||
(byte)10, 10, 10, 99
|
||||
};
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -9,17 +9,17 @@ Create a pre-release or full release at github; [Example 4.5-rc-1](https://githu
|
|||
Wack any existing tag as mvn will create one and it fails if already there.
|
||||
|
||||
```
|
||||
$ git tag -d 4.6
|
||||
$ git push origin :refs/tags/4.6
|
||||
$ git push upstream :refs/tags/4.6
|
||||
$ git tag -d 4.7
|
||||
$ git push origin :refs/tags/4.7
|
||||
$ git push upstream :refs/tags/4.7
|
||||
```
|
||||
|
||||
### Create release candidate tag
|
||||
|
||||
```bash
|
||||
$ git tag -a 4.6-rc1 -m 'heading towards 4.6'
|
||||
$ git push origin 4.6-rc1
|
||||
$ git push upstream 4.6-rc1
|
||||
$ git tag -a 4.7-rc1 -m 'heading towards 4.7'
|
||||
$ git push origin 4.7-rc1
|
||||
$ git push upstream 4.7-rc1
|
||||
```
|
||||
|
||||
## Bump version
|
||||
|
|
@ -107,7 +107,7 @@ Here is the file template
|
|||
|
||||
## Maven deploy snapshot
|
||||
|
||||
The goal is to get a snapshot, such as `4.6-SNAPSHOT`, to the staging server: [antlr4 tool](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4) and [antlr4 java runtime](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-runtime).
|
||||
The goal is to get a snapshot, such as `4.7-SNAPSHOT`, to the staging server: [antlr4 tool](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4) and [antlr4 java runtime](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-runtime).
|
||||
|
||||
Do this:
|
||||
|
||||
|
|
@ -115,15 +115,15 @@ Do this:
|
|||
$ mvn deploy -DskipTests
|
||||
...
|
||||
[INFO] --- maven-deploy-plugin:2.7:deploy (default-deploy) @ antlr4-tool-testsuite ---
|
||||
Downloading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.6-SNAPSHOT/maven-metadata.xml
|
||||
Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.6-SNAPSHOT/antlr4-tool-testsuite-4.6-20161211.173752-1.jar
|
||||
Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.6-SNAPSHOT/antlr4-tool-testsuite-4.6-20161211.173752-1.jar (3 KB at 3.4 KB/sec)
|
||||
Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.6-SNAPSHOT/antlr4-tool-testsuite-4.6-20161211.173752-1.pom
|
||||
Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.6-SNAPSHOT/antlr4-tool-testsuite-4.6-20161211.173752-1.pom (3 KB at 6.5 KB/sec)
|
||||
Downloading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/maven-metadata.xml
|
||||
Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/antlr4-tool-testsuite-4.7-20161211.173752-1.jar
|
||||
Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/antlr4-tool-testsuite-4.7-20161211.173752-1.jar (3 KB at 3.4 KB/sec)
|
||||
Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/antlr4-tool-testsuite-4.7-20161211.173752-1.pom
|
||||
Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/antlr4-tool-testsuite-4.7-20161211.173752-1.pom (3 KB at 6.5 KB/sec)
|
||||
Downloading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml
|
||||
Downloaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml (371 B at 1.4 KB/sec)
|
||||
Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.6-SNAPSHOT/maven-metadata.xml
|
||||
Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.6-SNAPSHOT/maven-metadata.xml (774 B at 1.8 KB/sec)
|
||||
Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/maven-metadata.xml
|
||||
Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/maven-metadata.xml (774 B at 1.8 KB/sec)
|
||||
Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml
|
||||
Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml (388 B at 0.9 KB/sec)
|
||||
[INFO] ------------------------------------------------------------------------
|
||||
|
|
@ -164,18 +164,18 @@ It will start out by asking you the version number:
|
|||
|
||||
```
|
||||
...
|
||||
What is the release version for "ANTLR 4"? (org.antlr:antlr4-master) 4.6: : 4.6
|
||||
What is the release version for "ANTLR 4 Runtime"? (org.antlr:antlr4-runtime) 4.6: :
|
||||
What is the release version for "ANTLR 4 Tool"? (org.antlr:antlr4) 4.6: :
|
||||
What is the release version for "ANTLR 4 Maven plugin"? (org.antlr:antlr4-maven-plugin) 4.6: :
|
||||
What is the release version for "ANTLR 4 Runtime Test Generator"? (org.antlr:antlr4-runtime-testsuite) 4.6: :
|
||||
What is the release version for "ANTLR 4 Tool Tests"? (org.antlr:antlr4-tool-testsuite) 4.6: :
|
||||
What is SCM release tag or label for "ANTLR 4"? (org.antlr:antlr4-master) antlr4-master-4.6: : 4.6
|
||||
What is the new development version for "ANTLR 4"? (org.antlr:antlr4-master) 4.6.1-SNAPSHOT:
|
||||
What is the release version for "ANTLR 4"? (org.antlr:antlr4-master) 4.7: : 4.7
|
||||
What is the release version for "ANTLR 4 Runtime"? (org.antlr:antlr4-runtime) 4.7: :
|
||||
What is the release version for "ANTLR 4 Tool"? (org.antlr:antlr4) 4.7: :
|
||||
What is the release version for "ANTLR 4 Maven plugin"? (org.antlr:antlr4-maven-plugin) 4.7: :
|
||||
What is the release version for "ANTLR 4 Runtime Test Generator"? (org.antlr:antlr4-runtime-testsuite) 4.7: :
|
||||
What is the release version for "ANTLR 4 Tool Tests"? (org.antlr:antlr4-tool-testsuite) 4.7: :
|
||||
What is SCM release tag or label for "ANTLR 4"? (org.antlr:antlr4-master) antlr4-master-4.7: : 4.7
|
||||
What is the new development version for "ANTLR 4"? (org.antlr:antlr4-master) 4.7.1-SNAPSHOT:
|
||||
...
|
||||
```
|
||||
|
||||
Maven will go through your pom.xml files to update versions from 4.6-SNAPSHOT to 4.6 for release and then to 4.6.1-SNAPSHOT after release, which is done with:
|
||||
Maven will go through your pom.xml files to update versions from 4.7-SNAPSHOT to 4.7 for release and then to 4.7.1-SNAPSHOT after release, which is done with:
|
||||
|
||||
```bash
|
||||
mvn release:perform -Darguments="-DskipTests"
|
||||
|
|
@ -194,11 +194,11 @@ and on the left click "Staging Repositories". You click the staging repo and clo
|
|||
Copy the jars to antlr.org site and update download/index.html
|
||||
|
||||
```bash
|
||||
cp ~/.m2/repository/org/antlr/antlr4-runtime/4.6/antlr4-runtime-4.6.jar ~/antlr/sites/website-antlr4/download/antlr-runtime-4.6.jar
|
||||
cp ~/.m2/repository/org/antlr/antlr4/4.6/antlr4-4.6-complete.jar ~/antlr/sites/website-antlr4/download/antlr-4.6-complete.jar
|
||||
cp ~/.m2/repository/org/antlr/antlr4-runtime/4.7/antlr4-runtime-4.7.jar ~/antlr/sites/website-antlr4/download/antlr-runtime-4.7.jar
|
||||
cp ~/.m2/repository/org/antlr/antlr4/4.7/antlr4-4.7-complete.jar ~/antlr/sites/website-antlr4/download/antlr-4.7-complete.jar
|
||||
cd ~/antlr/sites/website-antlr4/download
|
||||
git add antlr-4.6-complete.jar
|
||||
git add antlr-runtime-4.6.jar
|
||||
git add antlr-4.7-complete.jar
|
||||
git add antlr-runtime-4.7.jar
|
||||
```
|
||||
|
||||
Update on site:
|
||||
|
|
@ -210,7 +210,7 @@ Update on site:
|
|||
* scripts/topnav.js
|
||||
|
||||
```
|
||||
git commit -a -m 'add 4.6 jars'
|
||||
git commit -a -m 'add 4.7 jars'
|
||||
git push origin gh-pages
|
||||
```
|
||||
|
||||
|
|
@ -220,16 +220,24 @@ git push origin gh-pages
|
|||
|
||||
```bash
|
||||
cd runtime/JavaScript/src
|
||||
zip -r /tmp/antlr-javascript-runtime-4.6.zip antlr4
|
||||
cp /tmp/antlr-javascript-runtime-4.6.zip ~/antlr/sites/website-antlr4/download
|
||||
zip -r /tmp/antlr-javascript-runtime-4.7.zip antlr4
|
||||
cp /tmp/antlr-javascript-runtime-4.7.zip ~/antlr/sites/website-antlr4/download
|
||||
# git add, commit, push
|
||||
```
|
||||
|
||||
**Push to npm**
|
||||
|
||||
```bash
|
||||
cd runtime/JavaScript/src
|
||||
npm login
|
||||
npm publish antlr4
|
||||
```
|
||||
|
||||
Move target to website
|
||||
|
||||
```bash
|
||||
pushd ~/antlr/sites/website-antlr4/download
|
||||
git add antlr-javascript-runtime-4.6.zip
|
||||
git add antlr-javascript-runtime-4.7.zip
|
||||
git commit -a -m 'update JS runtime'
|
||||
git push origin gh-pages
|
||||
popd
|
||||
|
|
@ -283,7 +291,7 @@ Type the following command:
|
|||
```bash
|
||||
$ nuget pack Package.nuspec
|
||||
Attempting to build package from 'Package.nuspec'.
|
||||
Successfully created package '/Users/parrt/antlr/code/antlr4/runtime/CSharp/runtime/CSharp/Antlr4.Runtime.Standard.4.6.0.nupkg'.
|
||||
Successfully created package '/Users/parrt/antlr/code/antlr4/runtime/CSharp/runtime/CSharp/Antlr4.Runtime.Standard.4.7.0.nupkg'.
|
||||
```
|
||||
|
||||
This should display: Successfully created package *<package-path>*
|
||||
|
|
@ -309,15 +317,15 @@ rm Antlr4.Runtime/obj/net20/Release/Antlr4.Runtime.dll
|
|||
# build
|
||||
xbuild /p:Configuration=Release Antlr4.Runtime/Antlr4.Runtime.mono.csproj
|
||||
# zip it up to get a version number on zip filename
|
||||
zip --junk-paths /tmp/antlr-csharp-runtime-4.6.zip Antlr4.Runtime/obj/net20/Release/Antlr4.Runtime.Standard.dll
|
||||
cp /tmp/antlr-csharp-runtime-4.6.zip ~/antlr/sites/website-antlr4/download
|
||||
zip --junk-paths /tmp/antlr-csharp-runtime-4.7.zip Antlr4.Runtime/obj/net20/Release/Antlr4.Runtime.Standard.dll
|
||||
cp /tmp/antlr-csharp-runtime-4.7.zip ~/antlr/sites/website-antlr4/download
|
||||
```
|
||||
|
||||
Move target to website
|
||||
|
||||
```bash
|
||||
pushd ~/antlr/sites/website-antlr4/download
|
||||
git add antlr-csharp-runtime-4.6.zip
|
||||
git add antlr-csharp-runtime-4.7.zip
|
||||
git commit -a -m 'update C# runtime'
|
||||
git push origin gh-pages
|
||||
popd
|
||||
|
|
@ -386,7 +394,7 @@ On a Mac (with XCode 7+ installed):
|
|||
```bash
|
||||
cd runtime/Cpp
|
||||
./deploy-macos.sh
|
||||
cp antlr4-cpp-runtime-macos.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.6-macos.zip
|
||||
cp antlr4-cpp-runtime-macos.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-macos.zip
|
||||
```
|
||||
|
||||
On any Mac or Linux machine:
|
||||
|
|
@ -394,7 +402,7 @@ On any Mac or Linux machine:
|
|||
```bash
|
||||
cd runtime/Cpp
|
||||
./deploy-source.sh
|
||||
cp antlr4-cpp-runtime-source.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.6-source.zip
|
||||
cp antlr4-cpp-runtime-source.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-source.zip
|
||||
```
|
||||
|
||||
On a Windows machine the build scripts checks if VS 2013 and/or VS 2015 are installed and builds binaries for each, if found. This script requires 7z to be installed (http://7-zip.org).
|
||||
|
|
@ -402,7 +410,7 @@ On a Windows machine the build scripts checks if VS 2013 and/or VS 2015 are inst
|
|||
```bash
|
||||
cd runtime/Cpp
|
||||
deploy-windows.cmd
|
||||
cp antlr4-cpp-runtime-vs2015.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.6-vs2015.zip
|
||||
cp antlr4-cpp-runtime-vs2015.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-vs2015.zip
|
||||
```
|
||||
|
||||
Move target to website (**_rename to a specific ANTLR version first if needed_**):
|
||||
|
|
@ -410,9 +418,9 @@ Move target to website (**_rename to a specific ANTLR version first if needed_**
|
|||
```bash
|
||||
pushd ~/antlr/sites/website-antlr4/download
|
||||
# vi index.html
|
||||
git add antlr4cpp-runtime-4.6-macos.zip
|
||||
git add antlr4cpp-runtime-4.6-windows.zip
|
||||
git add antlr4cpp-runtime-4.6-source.zip
|
||||
git add antlr4cpp-runtime-4.7-macos.zip
|
||||
git add antlr4cpp-runtime-4.7-windows.zip
|
||||
git add antlr4cpp-runtime-4.7-source.zip
|
||||
git commit -a -m 'update C++ runtime'
|
||||
git push origin gh-pages
|
||||
popd
|
||||
|
|
@ -434,9 +442,9 @@ cd ~/antlr/sites/website-antlr4/api
|
|||
git checkout gh-pages
|
||||
git pull origin gh-pages
|
||||
cd Java
|
||||
jar xvf ~/.m2/repository/org/antlr/antlr4-runtime/4.6/antlr4-runtime-4.6-javadoc.jar
|
||||
jar xvf ~/.m2/repository/org/antlr/antlr4-runtime/4.7/antlr4-runtime-4.7-javadoc.jar
|
||||
cd ../JavaTool
|
||||
jar xvf ~/.m2/repository/org/antlr/antlr4/4.6/antlr4-4.6-javadoc.jar
|
||||
jar xvf ~/.m2/repository/org/antlr/antlr4/4.7/antlr4-4.7-javadoc.jar
|
||||
git commit -a -m 'freshen api doc'
|
||||
git push origin gh-pages
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,41 +0,0 @@
|
|||
# Overview of the ANTLR Runtime Test Suite
|
||||
|
||||
An important part of ANTLR4 is its runtime test suite, which consist of 2 subparts:
|
||||
|
||||
* Tests for the tool itself
|
||||
* Tests for the ANTLR runtime
|
||||
|
||||
Usually the tests are executed while compiling and installing an ANTLR4 jar from source code. The command for that is simply:
|
||||
|
||||
```bash
|
||||
$ mvn install
|
||||
```
|
||||
|
||||
to be executed in the root of the ANTLR4 repository. More details about this can be found in [Building ANTLR](building-antlr.md).
|
||||
|
||||
However, you don't need to run the installation again and again just to run the tests. Instead use
|
||||
|
||||
```bash
|
||||
$ mvn test
|
||||
```
|
||||
|
||||
to only trigger testing. You can find all runtime tests in the [runtime-testsuite/resources/org/antlr/v4/test/runtime](../runtime-testsuite/resources/org/antlr/v4/test/runtime) subfolder (tool tests under [tool-testsuite/test/org/antlr/v4/test/tool](../tool-testsuite/test/org/antlr/v4/test/tool)). The tool tests are just a bunch of Java test cases that test the tool's internal behavior (e.g. for code generation). We focus on the runtime tests here.
|
||||
|
||||
The underlying process of running the tests is quite a complicated setup to cater especially for a flexible test specification that can run with different target runtimes. Everything runs in Java, except for the actual target runtime tests. Runtime tests run first, followed by the tool tests. If there was a test failure in the first step, the tool tests are not executed, however. These are the steps involved when running the runtime tests:
|
||||
|
||||
* Generate Java JUnit test cases from the test templates, once for each target (C++, C#, Python, Java, Javascript atm.).
|
||||
* These test cases generate grammar files when executed and run the target specific parser generation step, including compiling a binary, if necessary (e.g. for C++ and C#).
|
||||
* Finally run the compiled test module using the input specified in the test template. The output (usually a token or parse tree dump) is then compared against the expected output, specified in the test template as well. This also includes any error messages written to the console.
|
||||
|
||||
## Generating JUnit Tests
|
||||
|
||||
The test specification part makes heavy use of the StringTemplate engine to allow defining target language agnostic tests. For that all tests are described in template (`stg`) files. You can find them in the [templates](../runtime-testsuite/resources/org/antlr/v4/test/runtime/templates) subfolder of the runtime tests folder. Read more about the folder structure in the [antlr-project-testing.md](antlr-project-testing.md) file. As lined out there you have to run
|
||||
|
||||
```bash
|
||||
$ mvn -Pgen generate-test-sources
|
||||
```
|
||||
everytime you change any of the test templates or your target language specific template (which is used to translate certain text to your specific language). And a small hint: this command can be executed from the ANTLR source root as well. No need to dig into a subfolder.
|
||||
|
||||
## Running the Generated Tests
|
||||
|
||||
After generation you can run the tests as written above (`mvn install` or `mvn test`, both recompile ANTLR if necessary), which takes about 40 minutes for the full set (of which 30 mins are alone consumed by the C++ target tests). Which tests actually run is controlled by the [runtime tests pom.xml file](../runtime-testsuite/pom.xml). Look for the `maven-surefire-plugin` plugin entry and especially its includes. If you ever want to run tests only for a specific target, comment out all other `<include>` elements. For a specific test change the wildcard to that specific test name. This is especially helpful when debugging a test (e.g. when it fails) or when creating/changing tests. Additionally, some targets require to install additional dependencies you may not want to add to your box (e.g. mono, python 3.5) just to run e.g. the Java or C++ tests.
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
# Lexers and Unicode text
|
||||
|
||||
Prior to ANTLR 4.7, generated lexers in most targets only supported part of the Unicode standard (code points up to `U+FFFF`). As of ANTLR 4.7, the lexers in all language runtimes support the full range of Unicode code points up to `U+10FFFF`.
|
||||
|
||||
C++, Python, Go, and Swift APIs didn't need any API changes to support Unicode code points, so we decided to leave those class interfaces as-is.
|
||||
|
||||
Java, C#, and JavaScript runtimes required changes and, rather than break the previous interface, we deprecated them. (The *Java-target* deprecated `ANTLRInputStream` and `ANTLRFileStream` APIs only support Unicode code points up to `U+FFFF`.) Now, those targets must create `CharStream`s from input using `CharStreams.fromPath()`, `CharStreams.fromFileName()`, etc...
|
||||
|
||||
A big shout out to Ben Hamilton (github bhamiltoncx) for his superhuman
|
||||
efforts across all targets to get true support for U+10FFFF code points.
|
||||
|
||||
## Example
|
||||
|
||||
The Java, C#, and JavaScript runtimes use the new factory style stream creation interface. For example, here is some sample Java code that uses `CharStreams.fromPath()`:
|
||||
|
||||
```java
|
||||
public static void main(String[] args) {
|
||||
CharStream charStream = CharStreams.fromPath(Paths.get(args[0]));
|
||||
Lexer lexer = new UnicodeLexer(charStream);
|
||||
CommonTokenStream tokens = new CommonTokenStream(lexer);
|
||||
tokens.fill();
|
||||
for (Token token : tokens.getTokens()) {
|
||||
System.out.println("Got token: " + token.toString());
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
# Unicode Code Points in Lexer Grammars
|
||||
|
||||
To refer to Unicode [code points](https://en.wikipedia.org/wiki/Code_point)
|
||||
in lexer grammars, use the `\u` string escape plus up to 4 hex digits. For example, to create
|
||||
a lexer rule for a single Cyrillic character by creating a range from
|
||||
`U+0400` to `U+04FF`:
|
||||
|
||||
```ANTLR
|
||||
CYRILLIC : '\u0400'..'\u04FF' ; // or [\u0400-\u04FF] without quotes
|
||||
```
|
||||
|
||||
Unicode literals larger than U+FFFF must use the extended `\u{12345}` syntax. For example, to create a lexer rule for a selection of smiley faces
|
||||
from the [Emoticons Unicode block](http://www.unicode.org/charts/PDF/U1F600.pdf):
|
||||
|
||||
```ANTLR
|
||||
EMOTICONS : ('\u{1F600}' | '\u{1F602}' | '\u{1F615}') ; // or [\u{1F600}\u{1F602}\u{1F615}]
|
||||
```
|
||||
|
||||
Finally, lexer char sets can include Unicode properties. Each Unicode code point has at least one property that describes the type group to which it belongs (e.g. alpha, number, punctuation). Other properties can be the language script or special binary properties and Unicode code blocks. That means however, that a property specifies a group of code points, hence they are only allowed in lexer char sets.
|
||||
|
||||
```ANTLR
|
||||
EMOJI : [\p{Emoji}] ;
|
||||
JAPANESE : [\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}] ;
|
||||
NOT_CYRILLIC : [\P{Script=Cyrillic}] ;
|
||||
```
|
||||
|
||||
See [lexer-rules.md](lexer-rules.md#lexer-rule-elements) for more detail on Unicode
|
||||
escapes in lexer rules.
|
||||
|
||||
## Migration
|
||||
|
||||
|
||||
Code for **4.6** looked like this:
|
||||
|
||||
|
||||
```java
|
||||
CharStream input = new ANTLRFileStream("myinputfile");
|
||||
JavaLexer lexer = new JavaLexer(input);
|
||||
CommonTokenStream tokens = new CommonTokenStream(lexer);
|
||||
```
|
||||
|
||||
(It didn't use UTF-8 by default, despite the documentation saying so previously; it actually depended on the calling environments default.)
|
||||
|
||||
Code for **4.7** assumes UTF-8 by default and looks like this:
|
||||
|
||||
```java
|
||||
CharStream input = CharStreams.fromFileName("inputfile");
|
||||
JavaLexer lexer = new JavaLexer(input);
|
||||
CommonTokenStream tokens = new CommonTokenStream(lexer);
|
||||
```
|
||||
|
||||
Or, if you'd like to specify the file encoding:
|
||||
|
||||
```java
|
||||
CharStream input = CharStreams.fromFileName("inputfile", Charset.forName("windows-1252"));
|
||||
```
|
||||
|
||||
### Motivation
|
||||
|
||||
After a [lively discussion](https://github.com/antlr/antlr4/pull/1771), I (parrt) decided not to simply gut the 4.6 `ANTLRFileStream` and `ANTLRInputStream` to incorporate the new U+10FFFF functionality. I decided to *deprecate* the old interface and recommend use of the new interface to prevent confusion. My reasoning is summarized as:
|
||||
|
||||
* I didn't like the idea of breaking all 4.6 code. To get the previous streams to properly support > 16 bit Unicode would require a lot of changes to the method signatures.
|
||||
* Using `int` buffer element types would double the size of memory required to hold streams in memory, given that we buffer everything (and I didn't want to change that aspect of the streams).
|
||||
* The new factory-style interface supports creation of the smallest possible code point buffer element size according to the Unicode code points found in the input stream. This means using half as much memory
|
||||
as the old {@link ANTLRFileStream}, which assumed 16-bit characters, for ASCII text.
|
||||
* Through some [serious testing and performance tweaking](https://github.com/antlr/antlr4/pull/1781), the new streams perform as fast or faster than the 4.6 streams.
|
||||
|
||||
**WARNING**. *You should avoid using both the deprecated and the new streams* in the same application because you will see
|
||||
a nontrivial performance degradation. This speed hit is because the
|
||||
`Lexer`'s internal code goes from a monomorphic to megamorphic
|
||||
dynamic dispatch to get characters from the input stream. Java's
|
||||
on-the-fly compiler (JIT) is unable to perform the same optimizations
|
||||
so stick with either the old or the new streams, if performance is
|
||||
a primary concern. See the [extreme debugging and spelunking](https://github.com/antlr/antlr4/pull/1781) needed to identify this issue in our timing rig.
|
||||
|
||||
### Legacy grammar using surrogate code units
|
||||
|
||||
Legacy grammars that did their own UTF-16 surrogate code unit matching will need to continue to use `ANTLRInputStream` (Java target) until the parser-application code can upgrade to `CharStreams` interface. Then the surrogate code unit matching should be removed from the grammar in favor of letting the new streams do the decoding.
|
||||
|
||||
Prior to 4.7, application code could directly pass `Token.getStartIndex()` and `Token.getStopIndex()` to Java and C# String APIs (because both used UTF-16 code units as the fundamental unit of length). With the new streams, clients will have to convert from code point indices to UTF-16 code unit indices. Here is some (Java) code to show you the necessary logic:
|
||||
|
||||
```java
|
||||
public final class CodePointCounter {
|
||||
private final String input;
|
||||
public int inputIndex = 0;
|
||||
public int codePointIndex = 0;
|
||||
|
||||
public int advanceToIndex(int newCodePointIndex) {
|
||||
assert newCodePointIndex >= codePointIndex;
|
||||
while (codePointIndex < newCodePointOffset) {
|
||||
int codePoint = Character.codePointAt(input, inputIndex);
|
||||
inputIndex += Character.charCount(codePoint);
|
||||
codePointIndex++;
|
||||
}
|
||||
return inputIndex;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Character Buffering, Unbuffered streams
|
||||
|
||||
The ANTLR character streams still buffer all the input when you create
|
||||
the stream, as they have done for ~20 years.
|
||||
|
||||
If you need unbuffered
|
||||
access, please note that it becomes challenging to create
|
||||
parse trees. The parse tree has to point to tokens which will either
|
||||
point into a stale location in an unbuffered stream or you have to copy
|
||||
the characters out of the buffer into the token. That defeats the purpose
|
||||
of unbuffered input. See the [ANTLR 4 book](https://www.amazon.com/Definitive-ANTLR-4-Reference/dp/1934356999) "13.8 Unbuffered Character and Token Streams". Unbuffered streams are primarily
|
||||
useful for processing infinite streams *during the parse* and require that you manually buffer characters. Use `UnbufferedCharStream` and `UnbufferedTokenStream`.
|
||||
|
||||
```java
|
||||
CharStream input = new UnbufferedCharStream(is);
|
||||
CSVLexer lex = new CSVLexer(input); // copy text out of sliding buffer and store in tokens
|
||||
lex.setTokenFactory(new CommonTokenFactory(true));
|
||||
TokenStream tokens = new UnbufferedTokenStream<CommonToken>(lex);
|
||||
CSVParser parser = new CSVParser(tokens);
|
||||
parser.setBuildParseTree(false);
|
||||
parser.file();
|
||||
```
|
||||
|
||||
Your grammar that needs to have embedded actions that access the tokens as they are created, but before they disappear and are garbage collected. For example,
|
||||
|
||||
```
|
||||
data : a=INT {int x = Integer.parseInt($a.text);} ;
|
||||
```
|
||||
|
||||
From the code comments of `CommonTokenFactory`:
|
||||
|
||||
> That `true` in `new CommonTokenFactory(true)` indicates whether `CommonToken.setText` should be called after
|
||||
constructing tokens to explicitly set the text. This is useful for cases
|
||||
where the input stream might not be able to provide arbitrary substrings
|
||||
of text from the input after the lexer creates a token (e.g. the
|
||||
implementation of `CharStream.getText` in
|
||||
`UnbufferedCharStream` throws an
|
||||
`UnsupportedOperationException`). Explicitly setting the token text
|
||||
allows `Token.getText` to be called at any time regardless of the
|
||||
input stream implementation.
|
||||
|
||||
*Currently, only Java, C++, and C# have these unbuffered streams implemented*.
|
||||
264
pom.xml
264
pom.xml
|
|
@ -1,145 +1,151 @@
|
|||
<!--
|
||||
~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
~ Use of this file is governed by the BSD 3-clause license that
|
||||
~ can be found in the LICENSE.txt file in the project root.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
<artifactId>oss-parent</artifactId>
|
||||
<version>9</version>
|
||||
</parent>
|
||||
<groupId>org.antlr</groupId>
|
||||
<artifactId>antlr4-master</artifactId>
|
||||
<version>4.6.1-SNAPSHOT</version>
|
||||
<packaging>pom</packaging>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
<artifactId>oss-parent</artifactId>
|
||||
<version>9</version>
|
||||
</parent>
|
||||
<groupId>org.antlr</groupId>
|
||||
<artifactId>antlr4-master</artifactId>
|
||||
<version>4.7.1-SNAPSHOT</version>
|
||||
<packaging>pom</packaging>
|
||||
|
||||
<name>ANTLR 4</name>
|
||||
<description>ANTLR 4 Master Build POM</description>
|
||||
<url>http://www.antlr.org</url>
|
||||
<inceptionYear>1992</inceptionYear>
|
||||
<organization>
|
||||
<name>ANTLR</name>
|
||||
<url>http://www.antlr.org</url>
|
||||
</organization>
|
||||
<name>ANTLR 4</name>
|
||||
<description>ANTLR 4 Master Build POM</description>
|
||||
<url>http://www.antlr.org</url>
|
||||
<inceptionYear>1992</inceptionYear>
|
||||
<organization>
|
||||
<name>ANTLR</name>
|
||||
<url>http://www.antlr.org</url>
|
||||
</organization>
|
||||
|
||||
<licenses>
|
||||
<license>
|
||||
<name>The BSD License</name>
|
||||
<url>http://www.antlr.org/license.html</url>
|
||||
<distribution>repo</distribution>
|
||||
</license>
|
||||
</licenses>
|
||||
<licenses>
|
||||
<license>
|
||||
<name>The BSD License</name>
|
||||
<url>http://www.antlr.org/license.html</url>
|
||||
<distribution>repo</distribution>
|
||||
</license>
|
||||
</licenses>
|
||||
|
||||
<developers>
|
||||
<developer>
|
||||
<name>Terence Parr</name>
|
||||
<url>http://parrt.cs.usfca.edu</url>
|
||||
<roles>
|
||||
<role>Project lead - ANTLR</role>
|
||||
</roles>
|
||||
</developer>
|
||||
<developer>
|
||||
<name>Sam Harwell</name>
|
||||
<url>http://tunnelvisionlabs.com</url>
|
||||
<roles>
|
||||
<role>Developer</role>
|
||||
</roles>
|
||||
</developer>
|
||||
<developer>
|
||||
<name>Eric Vergnaud</name>
|
||||
<roles>
|
||||
<role>Developer - JavaScript, C#, Python 2, Python 3</role>
|
||||
</roles>
|
||||
</developer>
|
||||
<developer>
|
||||
<name>Peter Boyer</name>
|
||||
<roles>
|
||||
<role>Developer - Go</role>
|
||||
</roles>
|
||||
</developer>
|
||||
<developer>
|
||||
<name>Jim Idle</name>
|
||||
<email>jimi@idle.ws</email>
|
||||
<url>http://www.linkedin.com/in/jimidle</url>
|
||||
<roles>
|
||||
<role>Developer - Maven Plugin</role>
|
||||
</roles>
|
||||
</developer>
|
||||
<developer>
|
||||
<name>Mike Lischke</name>
|
||||
<roles>
|
||||
<role>Developer - C++ Target</role>
|
||||
</roles>
|
||||
</developer>
|
||||
</developers>
|
||||
<developers>
|
||||
<developer>
|
||||
<name>Terence Parr</name>
|
||||
<url>http://parrt.cs.usfca.edu</url>
|
||||
<roles>
|
||||
<role>Project lead - ANTLR</role>
|
||||
</roles>
|
||||
</developer>
|
||||
<developer>
|
||||
<name>Sam Harwell</name>
|
||||
<url>http://tunnelvisionlabs.com</url>
|
||||
<roles>
|
||||
<role>Developer</role>
|
||||
</roles>
|
||||
</developer>
|
||||
<developer>
|
||||
<name>Eric Vergnaud</name>
|
||||
<roles>
|
||||
<role>Developer - JavaScript, C#, Python 2, Python 3</role>
|
||||
</roles>
|
||||
</developer>
|
||||
<developer>
|
||||
<name>Peter Boyer</name>
|
||||
<roles>
|
||||
<role>Developer - Go</role>
|
||||
</roles>
|
||||
</developer>
|
||||
<developer>
|
||||
<name>Jim Idle</name>
|
||||
<email>jimi@idle.ws</email>
|
||||
<url>http://www.linkedin.com/in/jimidle</url>
|
||||
<roles>
|
||||
<role>Developer - Maven Plugin</role>
|
||||
</roles>
|
||||
</developer>
|
||||
<developer>
|
||||
<name>Mike Lischke</name>
|
||||
<roles>
|
||||
<role>Developer - C++ Target</role>
|
||||
</roles>
|
||||
</developer>
|
||||
<developer>
|
||||
<name>Tom Everett</name>
|
||||
<roles>
|
||||
<role>Developer</role>
|
||||
</roles>
|
||||
</developer>
|
||||
</developers>
|
||||
|
||||
<modules>
|
||||
<module>runtime/Java</module>
|
||||
<module>tool</module>
|
||||
<module>antlr4-maven-plugin</module>
|
||||
<module>tool-testsuite</module>
|
||||
<module>runtime-testsuite/annotations</module>
|
||||
<module>runtime-testsuite/processors</module>
|
||||
<module>runtime-testsuite</module>
|
||||
</modules>
|
||||
<modules>
|
||||
<module>runtime/Java</module>
|
||||
<module>tool</module>
|
||||
<module>antlr4-maven-plugin</module>
|
||||
<module>tool-testsuite</module>
|
||||
<module>runtime-testsuite/annotations</module>
|
||||
<module>runtime-testsuite/processors</module>
|
||||
<module>runtime-testsuite</module>
|
||||
</modules>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
<antlr.testinprocess>true</antlr.testinprocess>
|
||||
<maven.compiler.source>1.7</maven.compiler.source>
|
||||
<maven.compiler.target>1.7</maven.compiler.target>
|
||||
</properties>
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
<antlr.testinprocess>true</antlr.testinprocess>
|
||||
<maven.compiler.source>1.7</maven.compiler.source>
|
||||
<maven.compiler.target>1.7</maven.compiler.target>
|
||||
</properties>
|
||||
|
||||
<mailingLists>
|
||||
<mailingList>
|
||||
<name>antlr-discussion</name>
|
||||
<archive>https://groups.google.com/forum/?fromgroups#!forum/antlr-discussion</archive>
|
||||
</mailingList>
|
||||
</mailingLists>
|
||||
<mailingLists>
|
||||
<mailingList>
|
||||
<name>antlr-discussion</name>
|
||||
<archive>https://groups.google.com/forum/?fromgroups#!forum/antlr-discussion</archive>
|
||||
</mailingList>
|
||||
</mailingLists>
|
||||
|
||||
<issueManagement>
|
||||
<system>GitHub Issues</system>
|
||||
<url>https://github.com/antlr/antlr4/issues</url>
|
||||
</issueManagement>
|
||||
<issueManagement>
|
||||
<system>GitHub Issues</system>
|
||||
<url>https://github.com/antlr/antlr4/issues</url>
|
||||
</issueManagement>
|
||||
|
||||
<scm>
|
||||
<url>https://github.com/antlr/antlr4/tree/master</url>
|
||||
<connection>scm:git:git://github.com/antlr/antlr4.git</connection>
|
||||
<developerConnection>scm:git:git@github.com:antlr/antlr4.git</developerConnection>
|
||||
<tag>HEAD</tag>
|
||||
</scm>
|
||||
<scm>
|
||||
<url>https://github.com/antlr/antlr4/tree/master</url>
|
||||
<connection>scm:git:git://github.com/antlr/antlr4.git</connection>
|
||||
<developerConnection>scm:git:git@github.com:antlr/antlr4.git</developerConnection>
|
||||
<tag>HEAD</tag>
|
||||
</scm>
|
||||
|
||||
<build>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>resources</directory>
|
||||
</resource>
|
||||
</resources>
|
||||
<testResources>
|
||||
<testResource>
|
||||
<directory>test</directory>
|
||||
</testResource>
|
||||
</testResources>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.6.0</version>
|
||||
<configuration>
|
||||
<source>${maven.compiler.source}</source>
|
||||
<target>${maven.compiler.target}</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
</build>
|
||||
<build>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>resources</directory>
|
||||
</resource>
|
||||
</resources>
|
||||
<testResources>
|
||||
<testResource>
|
||||
<directory>test</directory>
|
||||
</testResource>
|
||||
</testResources>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.6.0</version>
|
||||
<configuration>
|
||||
<source>${maven.compiler.source}</source>
|
||||
<target>${maven.compiler.target}</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
</build>
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<!--
|
||||
~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
~ Use of this file is governed by the BSD 3-clause license that
|
||||
~ can be found in the LICENSE.txt file in the project root.
|
||||
-->
|
||||
|
|
@ -9,7 +9,7 @@
|
|||
<parent>
|
||||
<groupId>org.antlr</groupId>
|
||||
<artifactId>antlr4-master</artifactId>
|
||||
<version>4.6.1-SNAPSHOT</version>
|
||||
<version>4.7.1-SNAPSHOT</version>
|
||||
<relativePath>../../pom.xml</relativePath>
|
||||
</parent>
|
||||
<artifactId>antlr4-runtime-test-annotations</artifactId>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<!--
|
||||
~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
~ Use of this file is governed by the BSD 3-clause license that
|
||||
~ can be found in the LICENSE.txt file in the project root.
|
||||
-->
|
||||
|
|
@ -10,7 +10,7 @@
|
|||
<parent>
|
||||
<groupId>org.antlr</groupId>
|
||||
<artifactId>antlr4-master</artifactId>
|
||||
<version>4.6.1-SNAPSHOT</version>
|
||||
<version>4.7.1-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>antlr4-runtime-testsuite</artifactId>
|
||||
<name>ANTLR 4 Runtime Tests (2nd generation)</name>
|
||||
|
|
@ -77,6 +77,11 @@
|
|||
<version>1.0.4</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.openjdk.jol</groupId>
|
||||
<artifactId>jol-core</artifactId>
|
||||
<version>0.8</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
|
@ -95,6 +100,8 @@
|
|||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>2.19.1</version>
|
||||
<configuration>
|
||||
<!-- SUREFIRE-951: file.encoding cannot be set via systemPropertyVariables -->
|
||||
<argLine>-Dfile.encoding=UTF-8</argLine>
|
||||
<includes>
|
||||
<include>**/csharp/Test*.java</include>
|
||||
<include>**/java/Test*.java</include>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<!--
|
||||
~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
~ Use of this file is governed by the BSD 3-clause license that
|
||||
~ can be found in the LICENSE.txt file in the project root.
|
||||
-->
|
||||
|
|
@ -9,7 +9,7 @@
|
|||
<parent>
|
||||
<groupId>org.antlr</groupId>
|
||||
<artifactId>antlr4-master</artifactId>
|
||||
<version>4.6.1-SNAPSHOT</version>
|
||||
<version>4.7.1-SNAPSHOT</version>
|
||||
<relativePath>../../pom.xml</relativePath>
|
||||
</parent>
|
||||
<artifactId>antlr4-runtime-test-annotation-processors</artifactId>
|
||||
|
|
|
|||
|
|
@ -1 +1,7 @@
|
|||
#
|
||||
# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
# Use of this file is governed by the BSD 3-clause license that
|
||||
# can be found in the LICENSE.txt file in the project root.
|
||||
#
|
||||
|
||||
org.antlr.v4.test.runtime.CommentHasStringValueProcessor
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
writeln(s) ::= <<Console.WriteLine(<s>);>>
|
||||
write(s) ::= <<Console.Write(<s>);>>
|
||||
writeList(s) ::= <<Console.WriteLine(<s; separator="+">);>>
|
||||
writeln(s) ::= <<Output.WriteLine(<s>);>>
|
||||
write(s) ::= <<Output.Write(<s>);>>
|
||||
writeList(s) ::= <<Output.WriteLine(<s; separator="+">);>>
|
||||
|
||||
False() ::= "false"
|
||||
|
||||
|
|
@ -176,8 +176,14 @@ public class PositionAdjustingLexerATNSimulator : LexerATNSimulator {
|
|||
BasicListener(X) ::= <<
|
||||
@parser::members {
|
||||
public class LeafListener : TBaseListener {
|
||||
private readonly TextWriter Output;
|
||||
|
||||
public LeafListener(TextWriter output) {
|
||||
Output = output;
|
||||
}
|
||||
|
||||
public override void VisitTerminal(ITerminalNode node) {
|
||||
Console.WriteLine(node.Symbol.Text);
|
||||
Output.WriteLine(node.Symbol.Text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -185,7 +191,7 @@ public class LeafListener : TBaseListener {
|
|||
|
||||
WalkListener(s) ::= <<
|
||||
ParseTreeWalker walker = new ParseTreeWalker();
|
||||
walker.Walk(new LeafListener(), <s>);
|
||||
walker.Walk(new LeafListener(Output), <s>);
|
||||
>>
|
||||
|
||||
TreeNodeWithAltNumField(X) ::= <<
|
||||
|
|
@ -204,6 +210,12 @@ public class MyRuleNode : ParserRuleContext {
|
|||
TokenGetterListener(X) ::= <<
|
||||
@parser::members {
|
||||
public class LeafListener : TBaseListener {
|
||||
private readonly TextWriter Output;
|
||||
|
||||
public LeafListener(TextWriter output) {
|
||||
Output = output;
|
||||
}
|
||||
|
||||
public override void ExitA(TParser.AContext ctx) {
|
||||
if (ctx.ChildCount==2)
|
||||
{
|
||||
|
|
@ -214,11 +226,11 @@ public class LeafListener : TBaseListener {
|
|||
}
|
||||
sb.Length = sb.Length - 2;
|
||||
sb.Append ("]");
|
||||
Console.Write ("{0} {1} {2}", ctx.INT (0).Symbol.Text,
|
||||
Output.Write ("{0} {1} {2}", ctx.INT (0).Symbol.Text,
|
||||
ctx.INT (1).Symbol.Text, sb.ToString());
|
||||
}
|
||||
else
|
||||
Console.WriteLine(ctx.ID().Symbol);
|
||||
Output.WriteLine(ctx.ID().Symbol);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -227,12 +239,18 @@ public class LeafListener : TBaseListener {
|
|||
RuleGetterListener(X) ::= <<
|
||||
@parser::members {
|
||||
public class LeafListener : TBaseListener {
|
||||
private readonly TextWriter Output;
|
||||
|
||||
public LeafListener(TextWriter output) {
|
||||
Output = output;
|
||||
}
|
||||
|
||||
public override void ExitA(TParser.AContext ctx) {
|
||||
if (ctx.ChildCount==2) {
|
||||
Console.Write("{0} {1} {2}",ctx.b(0).Start.Text,
|
||||
Output.Write("{0} {1} {2}",ctx.b(0).Start.Text,
|
||||
ctx.b(1).Start.Text,ctx.b()[0].Start.Text);
|
||||
} else
|
||||
Console.WriteLine(ctx.b(0).Start.Text);
|
||||
Output.WriteLine(ctx.b(0).Start.Text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -242,12 +260,18 @@ public class LeafListener : TBaseListener {
|
|||
LRListener(X) ::= <<
|
||||
@parser::members {
|
||||
public class LeafListener : TBaseListener {
|
||||
private readonly TextWriter Output;
|
||||
|
||||
public LeafListener(TextWriter output) {
|
||||
Output = output;
|
||||
}
|
||||
|
||||
public override void ExitE(TParser.EContext ctx) {
|
||||
if (ctx.ChildCount==3) {
|
||||
Console.Write("{0} {1} {2}\n",ctx.e(0).Start.Text,
|
||||
Output.Write("{0} {1} {2}\n",ctx.e(0).Start.Text,
|
||||
ctx.e(1).Start.Text, ctx.e()[0].Start.Text);
|
||||
} else
|
||||
Console.WriteLine(ctx.INT().Symbol.Text);
|
||||
Output.WriteLine(ctx.INT().Symbol.Text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -256,11 +280,17 @@ public class LeafListener : TBaseListener {
|
|||
LRWithLabelsListener(X) ::= <<
|
||||
@parser::members {
|
||||
public class LeafListener : TBaseListener {
|
||||
private readonly TextWriter Output;
|
||||
|
||||
public LeafListener(TextWriter output) {
|
||||
Output = output;
|
||||
}
|
||||
|
||||
public override void ExitCall(TParser.CallContext ctx) {
|
||||
Console.Write("{0} {1}",ctx.e().Start.Text,ctx.eList());
|
||||
Output.Write("{0} {1}",ctx.e().Start.Text,ctx.eList());
|
||||
}
|
||||
public override void ExitInt(TParser.IntContext ctx) {
|
||||
Console.WriteLine(ctx.INT().Symbol.Text);
|
||||
Output.WriteLine(ctx.INT().Symbol.Text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -274,12 +304,12 @@ void foo() {
|
|||
}
|
||||
>>
|
||||
|
||||
Declare_foo() ::= <<public void foo() {Console.WriteLine("foo");}>>
|
||||
Declare_foo() ::= <<public void foo() {Output.WriteLine("foo");}>>
|
||||
|
||||
Invoke_foo() ::= "this.foo();"
|
||||
|
||||
Declare_pred() ::= <<bool pred(bool v) {
|
||||
Console.WriteLine("eval="+v.ToString().ToLower());
|
||||
Output.WriteLine("eval="+v.ToString().ToLower());
|
||||
return v;
|
||||
}
|
||||
>>
|
||||
|
|
|
|||
|
|
@ -325,5 +325,5 @@ func pred(v bool) bool {
|
|||
|
||||
Invoke_pred(v) ::= <<pred(<v>)>>
|
||||
ContextRuleFunction(ctx, rule) ::= "<ctx>.<rule>"
|
||||
StringType() ::= "String"
|
||||
StringType() ::= "string"
|
||||
ContextMember(ctx, subctx, member) ::= "<ctx>.<subctx>.<member; format={cap}>"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
writeln(s) ::= <<print(<s>)>>
|
||||
write(s) ::= <<print(<s>,end='')>>
|
||||
writeList(s) ::= <<print(<s: {v | str(<v>)}; separator="+">)>>
|
||||
writeln(s) ::= <<print(<s>, file=self._output)>>
|
||||
write(s) ::= <<print(<s>,end='', file=self._output)>>
|
||||
writeList(s) ::= <<print(<s: {v | str(<v>)}; separator="+">, file=self._output)>>
|
||||
|
||||
False() ::= "False"
|
||||
|
||||
|
|
@ -152,14 +152,16 @@ else:
|
|||
from <X>Listener import <X>Listener
|
||||
|
||||
class LeafListener(TListener):
|
||||
def __init__(self, output):
|
||||
self._output = output
|
||||
def visitTerminal(self, node):
|
||||
print(node.symbol.text)
|
||||
print(node.symbol.text, file=self._output)
|
||||
}
|
||||
>>
|
||||
|
||||
WalkListener(s) ::= <<
|
||||
walker = ParseTreeWalker()
|
||||
walker.walk(TParser.LeafListener(), <s>)
|
||||
walker.walk(TParser.LeafListener(self._output), <s>)
|
||||
>>
|
||||
|
||||
TreeNodeWithAltNumField(X) ::= <<
|
||||
|
|
@ -183,11 +185,13 @@ else:
|
|||
from <X>Listener import <X>Listener
|
||||
|
||||
class LeafListener(TListener):
|
||||
def __init__(self, output):
|
||||
self._output = output
|
||||
def exitA(self, ctx):
|
||||
if ctx.getChildCount()==2:
|
||||
print(ctx.INT(0).symbol.text + ' ' + ctx.INT(1).symbol.text + ' ' + str_list(ctx.INT()))
|
||||
print(ctx.INT(0).symbol.text + ' ' + ctx.INT(1).symbol.text + ' ' + str_list(ctx.INT()), file=self._output)
|
||||
else:
|
||||
print(str(ctx.ID().symbol))
|
||||
print(str(ctx.ID().symbol), file=self._output)
|
||||
}
|
||||
>>
|
||||
|
||||
|
|
@ -199,11 +203,13 @@ else:
|
|||
from <X>Listener import <X>Listener
|
||||
|
||||
class LeafListener(TListener):
|
||||
def __init__(self, output):
|
||||
self._output = output
|
||||
def exitA(self, ctx):
|
||||
if ctx.getChildCount()==2:
|
||||
print(ctx.b(0).start.text + ' ' + ctx.b(1).start.text + ' ' + ctx.b()[0].start.text)
|
||||
print(ctx.b(0).start.text + ' ' + ctx.b(1).start.text + ' ' + ctx.b()[0].start.text, file=self._output)
|
||||
else:
|
||||
print(ctx.b(0).start.text)
|
||||
print(ctx.b(0).start.text, file=self._output)
|
||||
}
|
||||
>>
|
||||
|
||||
|
|
@ -216,11 +222,13 @@ else:
|
|||
from <X>Listener import <X>Listener
|
||||
|
||||
class LeafListener(TListener):
|
||||
def __init__(self, output):
|
||||
self._output = output
|
||||
def exitE(self, ctx):
|
||||
if ctx.getChildCount()==3:
|
||||
print(ctx.e(0).start.text + ' ' + ctx.e(1).start.text + ' ' + ctx.e()[0].start.text)
|
||||
print(ctx.e(0).start.text + ' ' + ctx.e(1).start.text + ' ' + ctx.e()[0].start.text, file=self._output)
|
||||
else:
|
||||
print(ctx.INT().symbol.text)
|
||||
print(ctx.INT().symbol.text, file=self._output)
|
||||
}
|
||||
>>
|
||||
|
||||
|
|
@ -232,10 +240,12 @@ else:
|
|||
from <X>Listener import <X>Listener
|
||||
|
||||
class LeafListener(TListener):
|
||||
def __init__(self, output):
|
||||
self._output = output
|
||||
def exitCall(self, ctx):
|
||||
print(ctx.e().start.text + ' ' + str(ctx.eList()))
|
||||
print(ctx.e().start.text + ' ' + str(ctx.eList()), file=self._output)
|
||||
def exitInt(self, ctx):
|
||||
print(ctx.INT().symbol.text)
|
||||
print(ctx.INT().symbol.text, file=self._output)
|
||||
}
|
||||
>>
|
||||
|
||||
|
|
@ -247,13 +257,13 @@ def foo():
|
|||
>>
|
||||
|
||||
Declare_foo() ::= <<def foo(self):
|
||||
print('foo')
|
||||
print('foo', file=self._output)
|
||||
>>
|
||||
|
||||
Invoke_foo() ::= "self.foo()"
|
||||
|
||||
Declare_pred() ::= <<def pred(self, v):
|
||||
print('eval=' + str(v).lower())
|
||||
print('eval=' + str(v).lower(), file=self._output)
|
||||
return v
|
||||
|
||||
>>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
writeln(s) ::= <<print(<s>)>>
|
||||
write(s) ::= <<print(<s>,end='')>>
|
||||
writeList(s) ::= <<print(<s: {v | str(<v>)}; separator="+">)>>
|
||||
writeln(s) ::= <<print(<s>, file=self._output)>>
|
||||
write(s) ::= <<print(<s>,end='',file=self._output)>>
|
||||
writeList(s) ::= <<print(<s: {v | str(<v>)}; separator="+">, file=self._output)>>
|
||||
|
||||
False() ::= "False"
|
||||
|
||||
|
|
@ -152,8 +152,10 @@ def isIdentifierChar(c):
|
|||
BasicListener(X) ::= <<
|
||||
@parser::members {
|
||||
class LeafListener(MockListener):
|
||||
def __init__(self, output):
|
||||
self._output = output
|
||||
def visitTerminal(self, node):
|
||||
print(node.symbol.text)
|
||||
print(node.symbol.text, file=self._output)
|
||||
}
|
||||
>>
|
||||
|
||||
|
|
@ -164,7 +166,7 @@ else:
|
|||
from TListener import TListener
|
||||
TParser.LeafListener.__bases__ = (TListener,)
|
||||
walker = ParseTreeWalker()
|
||||
walker.walk(TParser.LeafListener(), <s>)
|
||||
walker.walk(TParser.LeafListener(self._output), <s>)
|
||||
>>
|
||||
|
||||
TreeNodeWithAltNumField(X) ::= <<
|
||||
|
|
@ -183,22 +185,26 @@ class MyRuleNode(ParserRuleContext):
|
|||
TokenGetterListener(X) ::= <<
|
||||
@parser::members {
|
||||
class LeafListener(MockListener):
|
||||
def __init__(self, output):
|
||||
self._output = output
|
||||
def exitA(self, ctx):
|
||||
if ctx.getChildCount()==2:
|
||||
print(ctx.INT(0).symbol.text + ' ' + ctx.INT(1).symbol.text + ' ' + str_list(ctx.INT()))
|
||||
print(ctx.INT(0).symbol.text + ' ' + ctx.INT(1).symbol.text + ' ' + str_list(ctx.INT()), file=self._output)
|
||||
else:
|
||||
print(str(ctx.ID().symbol))
|
||||
print(str(ctx.ID().symbol), file=self._output)
|
||||
}
|
||||
>>
|
||||
|
||||
RuleGetterListener(X) ::= <<
|
||||
@parser::members {
|
||||
class LeafListener(MockListener):
|
||||
def __init__(self, output):
|
||||
self._output = output
|
||||
def exitA(self, ctx):
|
||||
if ctx.getChildCount()==2:
|
||||
print(ctx.b(0).start.text + ' ' + ctx.b(1).start.text + ' ' + ctx.b()[0].start.text)
|
||||
print(ctx.b(0).start.text + ' ' + ctx.b(1).start.text + ' ' + ctx.b()[0].start.text, file=self._output)
|
||||
else:
|
||||
print(ctx.b(0).start.text)
|
||||
print(ctx.b(0).start.text, file=self._output)
|
||||
}
|
||||
>>
|
||||
|
||||
|
|
@ -206,21 +212,25 @@ class LeafListener(MockListener):
|
|||
LRListener(X) ::= <<
|
||||
@parser::members {
|
||||
class LeafListener(MockListener):
|
||||
def __init__(self, output):
|
||||
self._output = output
|
||||
def exitE(self, ctx):
|
||||
if ctx.getChildCount()==3:
|
||||
print(ctx.e(0).start.text + ' ' + ctx.e(1).start.text + ' ' + ctx.e()[0].start.text)
|
||||
print(ctx.e(0).start.text + ' ' + ctx.e(1).start.text + ' ' + ctx.e()[0].start.text, file=self._output)
|
||||
else:
|
||||
print(ctx.INT().symbol.text)
|
||||
print(ctx.INT().symbol.text, file=self._output)
|
||||
}
|
||||
>>
|
||||
|
||||
LRWithLabelsListener(X) ::= <<
|
||||
@parser::members {
|
||||
class LeafListener(MockListener):
|
||||
def __init__(self, output):
|
||||
self._output = output
|
||||
def exitCall(self, ctx):
|
||||
print(ctx.e().start.text + ' ' + str(ctx.eList()))
|
||||
print(ctx.e().start.text + ' ' + str(ctx.eList()), file=self._output)
|
||||
def exitInt(self, ctx):
|
||||
print(ctx.INT().symbol.text)
|
||||
print(ctx.INT().symbol.text, file=self._output)
|
||||
}
|
||||
>>
|
||||
|
||||
|
|
@ -232,13 +242,13 @@ def foo():
|
|||
>>
|
||||
|
||||
Declare_foo() ::= <<def foo(self):
|
||||
print('foo')
|
||||
print('foo', file=self._output)
|
||||
>>
|
||||
|
||||
Invoke_foo() ::= "self.foo()"
|
||||
|
||||
Declare_pred() ::= <<def pred(self, v):
|
||||
print('eval=' + str(v).lower())
|
||||
print('eval=' + str(v).lower(), file=self._output)
|
||||
return v
|
||||
|
||||
>>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,309 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
||||
package org.antlr.v4.runtime;
|
||||
|
||||
import org.antlr.v4.runtime.misc.Interval;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.ExpectedException;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
public class TestCodePointCharStream {
|
||||
@Rule
|
||||
public ExpectedException thrown = ExpectedException.none();
|
||||
|
||||
@Test
|
||||
public void emptyBytesHasSize0() {
|
||||
CodePointCharStream s = CharStreams.fromString("");
|
||||
assertEquals(0, s.size());
|
||||
assertEquals(0, s.index());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void emptyBytesLookAheadReturnsEOF() {
|
||||
CodePointCharStream s = CharStreams.fromString("");
|
||||
assertEquals(IntStream.EOF, s.LA(1));
|
||||
assertEquals(0, s.index());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void consumingEmptyStreamShouldThrow() {
|
||||
CodePointCharStream s = CharStreams.fromString("");
|
||||
thrown.expect(IllegalStateException.class);
|
||||
thrown.expectMessage("cannot consume EOF");
|
||||
s.consume();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void singleLatinCodePointHasSize1() {
|
||||
CodePointCharStream s = CharStreams.fromString("X");
|
||||
assertEquals(1, s.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void consumingSingleLatinCodePointShouldMoveIndex() {
|
||||
CodePointCharStream s = CharStreams.fromString("X");
|
||||
assertEquals(0, s.index());
|
||||
s.consume();
|
||||
assertEquals(1, s.index());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void consumingPastSingleLatinCodePointShouldThrow() {
|
||||
CodePointCharStream s = CharStreams.fromString("X");
|
||||
s.consume();
|
||||
thrown.expect(IllegalStateException.class);
|
||||
thrown.expectMessage("cannot consume EOF");
|
||||
s.consume();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void singleLatinCodePointLookAheadShouldReturnCodePoint() {
|
||||
CodePointCharStream s = CharStreams.fromString("X");
|
||||
assertEquals('X', s.LA(1));
|
||||
assertEquals(0, s.index());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void multipleLatinCodePointsLookAheadShouldReturnCodePoints() {
|
||||
CodePointCharStream s = CharStreams.fromString("XYZ");
|
||||
assertEquals('X', s.LA(1));
|
||||
assertEquals(0, s.index());
|
||||
assertEquals('Y', s.LA(2));
|
||||
assertEquals(0, s.index());
|
||||
assertEquals('Z', s.LA(3));
|
||||
assertEquals(0, s.index());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void singleLatinCodePointLookAheadPastEndShouldReturnEOF() {
|
||||
CodePointCharStream s = CharStreams.fromString("X");
|
||||
assertEquals(IntStream.EOF, s.LA(2));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void singleCJKCodePointHasSize1() {
|
||||
CodePointCharStream s = CharStreams.fromString("\u611B");
|
||||
assertEquals(1, s.size());
|
||||
assertEquals(0, s.index());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void consumingSingleCJKCodePointShouldMoveIndex() {
|
||||
CodePointCharStream s = CharStreams.fromString("\u611B");
|
||||
assertEquals(0, s.index());
|
||||
s.consume();
|
||||
assertEquals(1, s.index());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void consumingPastSingleCJKCodePointShouldThrow() {
|
||||
CodePointCharStream s = CharStreams.fromString("\u611B");
|
||||
s.consume();
|
||||
thrown.expect(IllegalStateException.class);
|
||||
thrown.expectMessage("cannot consume EOF");
|
||||
s.consume();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void singleCJKCodePointLookAheadShouldReturnCodePoint() {
|
||||
CodePointCharStream s = CharStreams.fromString("\u611B");
|
||||
assertEquals(0x611B, s.LA(1));
|
||||
assertEquals(0, s.index());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void singleCJKCodePointLookAheadPastEndShouldReturnEOF() {
|
||||
CodePointCharStream s = CharStreams.fromString("\u611B");
|
||||
assertEquals(IntStream.EOF, s.LA(2));
|
||||
assertEquals(0, s.index());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void singleEmojiCodePointHasSize1() {
|
||||
CodePointCharStream s = CharStreams.fromString(
|
||||
new StringBuilder().appendCodePoint(0x1F4A9).toString());
|
||||
assertEquals(1, s.size());
|
||||
assertEquals(0, s.index());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void consumingSingleEmojiCodePointShouldMoveIndex() {
|
||||
CodePointCharStream s = CharStreams.fromString(
|
||||
new StringBuilder().appendCodePoint(0x1F4A9).toString());
|
||||
assertEquals(0, s.index());
|
||||
s.consume();
|
||||
assertEquals(1, s.index());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void consumingPastEndOfEmojiCodePointWithShouldThrow() {
|
||||
CodePointCharStream s = CharStreams.fromString(
|
||||
new StringBuilder().appendCodePoint(0x1F4A9).toString());
|
||||
assertEquals(0, s.index());
|
||||
s.consume();
|
||||
assertEquals(1, s.index());
|
||||
thrown.expect(IllegalStateException.class);
|
||||
thrown.expectMessage("cannot consume EOF");
|
||||
s.consume();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void singleEmojiCodePointLookAheadShouldReturnCodePoint() {
|
||||
CodePointCharStream s = CharStreams.fromString(
|
||||
new StringBuilder().appendCodePoint(0x1F4A9).toString());
|
||||
assertEquals(0x1F4A9, s.LA(1));
|
||||
assertEquals(0, s.index());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void singleEmojiCodePointLookAheadPastEndShouldReturnEOF() {
|
||||
CodePointCharStream s = CharStreams.fromString(
|
||||
new StringBuilder().appendCodePoint(0x1F4A9).toString());
|
||||
assertEquals(IntStream.EOF, s.LA(2));
|
||||
assertEquals(0, s.index());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getTextWithLatin() {
|
||||
CodePointCharStream s = CharStreams.fromString("0123456789");
|
||||
assertEquals("34567", s.getText(Interval.of(3, 7)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getTextWithCJK() {
|
||||
CodePointCharStream s = CharStreams.fromString("01234\u40946789");
|
||||
assertEquals("34\u409467", s.getText(Interval.of(3, 7)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getTextWithEmoji() {
|
||||
CodePointCharStream s = CharStreams.fromString(
|
||||
new StringBuilder("01234")
|
||||
.appendCodePoint(0x1F522)
|
||||
.append("6789")
|
||||
.toString());
|
||||
assertEquals("34\uD83D\uDD2267", s.getText(Interval.of(3, 7)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void toStringWithLatin() {
|
||||
CodePointCharStream s = CharStreams.fromString("0123456789");
|
||||
assertEquals("0123456789", s.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void toStringWithCJK() {
|
||||
CodePointCharStream s = CharStreams.fromString("01234\u40946789");
|
||||
assertEquals("01234\u40946789", s.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void toStringWithEmoji() {
|
||||
CodePointCharStream s = CharStreams.fromString(
|
||||
new StringBuilder("01234")
|
||||
.appendCodePoint(0x1F522)
|
||||
.append("6789")
|
||||
.toString());
|
||||
assertEquals("01234\uD83D\uDD226789", s.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void lookAheadWithLatin() {
|
||||
CodePointCharStream s = CharStreams.fromString("0123456789");
|
||||
assertEquals('5', s.LA(6));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void lookAheadWithCJK() {
|
||||
CodePointCharStream s = CharStreams.fromString("01234\u40946789");
|
||||
assertEquals(0x4094, s.LA(6));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void lookAheadWithEmoji() {
|
||||
CodePointCharStream s = CharStreams.fromString(
|
||||
new StringBuilder("01234")
|
||||
.appendCodePoint(0x1F522)
|
||||
.append("6789")
|
||||
.toString());
|
||||
assertEquals(0x1F522, s.LA(6));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void seekWithLatin() {
|
||||
CodePointCharStream s = CharStreams.fromString("0123456789");
|
||||
s.seek(5);
|
||||
assertEquals('5', s.LA(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void seekWithCJK() {
|
||||
CodePointCharStream s = CharStreams.fromString("01234\u40946789");
|
||||
s.seek(5);
|
||||
assertEquals(0x4094, s.LA(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void seekWithEmoji() {
|
||||
CodePointCharStream s = CharStreams.fromString(
|
||||
new StringBuilder("01234")
|
||||
.appendCodePoint(0x1F522)
|
||||
.append("6789")
|
||||
.toString());
|
||||
s.seek(5);
|
||||
assertEquals(0x1F522, s.LA(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void lookBehindWithLatin() {
|
||||
CodePointCharStream s = CharStreams.fromString("0123456789");
|
||||
s.seek(6);
|
||||
assertEquals('5', s.LA(-1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void lookBehindWithCJK() {
|
||||
CodePointCharStream s = CharStreams.fromString("01234\u40946789");
|
||||
s.seek(6);
|
||||
assertEquals(0x4094, s.LA(-1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void lookBehindWithEmoji() {
|
||||
CodePointCharStream s = CharStreams.fromString(
|
||||
new StringBuilder("01234")
|
||||
.appendCodePoint(0x1F522)
|
||||
.append("6789")
|
||||
.toString());
|
||||
s.seek(6);
|
||||
assertEquals(0x1F522, s.LA(-1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void asciiContentsShouldUse8BitBuffer() {
|
||||
CodePointCharStream s = CharStreams.fromString("hello");
|
||||
assertTrue(s.getInternalStorage() instanceof byte[]);
|
||||
assertEquals(5, s.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void bmpContentsShouldUse16BitBuffer() {
|
||||
CodePointCharStream s = CharStreams.fromString("hello \u4E16\u754C");
|
||||
assertTrue(s.getInternalStorage() instanceof char[]);
|
||||
assertEquals(8, s.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void smpContentsShouldUse32BitBuffer() {
|
||||
CodePointCharStream s = CharStreams.fromString("hello \uD83C\uDF0D");
|
||||
assertTrue(s.getInternalStorage() instanceof int[]);
|
||||
assertEquals(7, s.size());
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
@ -9,7 +9,6 @@ package org.antlr.v4.test.runtime;
|
|||
import org.antlr.v4.Tool;
|
||||
import org.antlr.v4.runtime.misc.Pair;
|
||||
import org.antlr.v4.runtime.misc.Utils;
|
||||
import org.antlr.v4.test.runtime.java.BaseJavaTest;
|
||||
import org.antlr.v4.tool.ANTLRMessage;
|
||||
import org.antlr.v4.tool.DefaultToolListener;
|
||||
import org.junit.Before;
|
||||
|
|
@ -224,7 +223,7 @@ public abstract class BaseRuntimeTest {
|
|||
String... extraOptions)
|
||||
{
|
||||
mkdir(workdir);
|
||||
BaseJavaTest.writeFile(workdir, grammarFileName, grammarStr);
|
||||
writeFile(workdir, grammarFileName, grammarStr);
|
||||
return antlrOnString(workdir, targetName, grammarFileName, defaultListener, extraOptions);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
||||
package org.antlr.v4.test.runtime;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
public final class StreamVacuum implements Runnable {
|
||||
private StringBuilder buf = new StringBuilder();
|
||||
private BufferedReader in;
|
||||
private Thread sucker;
|
||||
public StreamVacuum(InputStream in) {
|
||||
this.in = new BufferedReader( new InputStreamReader(in, StandardCharsets.UTF_8) );
|
||||
}
|
||||
public void start() {
|
||||
sucker = new Thread(this);
|
||||
sucker.start();
|
||||
}
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
TestOutputReading.append(in, buf);
|
||||
}
|
||||
catch (IOException ioe) {
|
||||
System.err.println("can't read output from process");
|
||||
}
|
||||
}
|
||||
/** wait for the thread to finish */
|
||||
public void join() throws InterruptedException {
|
||||
sucker.join();
|
||||
}
|
||||
@Override
|
||||
public String toString() {
|
||||
return buf.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
||||
package org.antlr.v4.test.runtime;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.NoSuchFileException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
public abstract class TestOutputReading {
|
||||
public static void append(BufferedReader in, StringBuilder buf) throws IOException {
|
||||
String line = in.readLine();
|
||||
while (line!=null) {
|
||||
buf.append(line);
|
||||
// NOTE: This appends a newline at EOF
|
||||
// regardless of whether or not the
|
||||
// input actually ended with a
|
||||
// newline.
|
||||
//
|
||||
// We should revisit this and read a
|
||||
// block at a time rather than a line
|
||||
// at a time, and change all tests
|
||||
// which rely on this behavior to
|
||||
// remove the trailing newline at EOF.
|
||||
//
|
||||
// When we fix this, we can remove the
|
||||
// TestOutputReading class entirely.
|
||||
buf.append('\n');
|
||||
line = in.readLine();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read in the UTF-8 bytes at {@code path}, convert all
|
||||
* platform-specific line terminators to NL, and append NL
|
||||
* if the file was non-empty and didn't already end with one.
|
||||
*
|
||||
* {@see StreamVacuum#run()} for why this method exists.
|
||||
*
|
||||
* Returns {@code null} if the file does not exist or the output
|
||||
* was empty.
|
||||
*/
|
||||
public static String read(Path path) throws IOException {
|
||||
// Mimic StreamVacuum.run()'s behavior of replacing all platform-specific
|
||||
// EOL sequences with NL.
|
||||
StringBuilder buf = new StringBuilder();
|
||||
try (BufferedReader in = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
|
||||
append(in, buf);
|
||||
} catch (FileNotFoundException | NoSuchFileException e) {
|
||||
return null;
|
||||
}
|
||||
if (buf.length() > 0) {
|
||||
return buf.toString();
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
@ -36,6 +36,7 @@ import org.antlr.v4.runtime.tree.ParseTree;
|
|||
import org.antlr.v4.semantics.SemanticPipeline;
|
||||
import org.antlr.v4.test.runtime.ErrorQueue;
|
||||
import org.antlr.v4.test.runtime.RuntimeTestSupport;
|
||||
import org.antlr.v4.test.runtime.StreamVacuum;
|
||||
import org.antlr.v4.tool.ANTLRMessage;
|
||||
import org.antlr.v4.tool.DOTGenerator;
|
||||
import org.antlr.v4.tool.Grammar;
|
||||
|
|
@ -46,13 +47,7 @@ import org.stringtemplate.v4.ST;
|
|||
import org.stringtemplate.v4.STGroup;
|
||||
import org.stringtemplate.v4.STGroupString;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.net.URISyntaxException;
|
||||
|
|
@ -70,6 +65,7 @@ import java.util.Set;
|
|||
import java.util.TreeMap;
|
||||
|
||||
import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
|
||||
import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
|
||||
import static org.junit.Assert.assertArrayEquals;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
|
|
@ -486,11 +482,14 @@ public class BaseCppTest implements RuntimeTestSupport {
|
|||
String os = System.getProperty("os.name", "generic").toLowerCase(Locale.ENGLISH);
|
||||
if ((os.indexOf("mac") >= 0) || (os.indexOf("darwin") >= 0)) {
|
||||
detectedOS = "mac";
|
||||
} else if (os.indexOf("win") >= 0) {
|
||||
}
|
||||
else if (os.indexOf("win") >= 0) {
|
||||
detectedOS = "windows";
|
||||
} else if (os.indexOf("nux") >= 0) {
|
||||
}
|
||||
else if (os.indexOf("nux") >= 0) {
|
||||
detectedOS = "linux";
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
detectedOS = "unknown";
|
||||
}
|
||||
}
|
||||
|
|
@ -751,41 +750,6 @@ public class BaseCppTest implements RuntimeTestSupport {
|
|||
}
|
||||
}
|
||||
|
||||
public static class StreamVacuum implements Runnable {
|
||||
StringBuilder buf = new StringBuilder();
|
||||
BufferedReader in;
|
||||
Thread sucker;
|
||||
public StreamVacuum(InputStream in) {
|
||||
this.in = new BufferedReader( new InputStreamReader(in) );
|
||||
}
|
||||
public void start() {
|
||||
sucker = new Thread(this);
|
||||
sucker.start();
|
||||
}
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
String line = in.readLine();
|
||||
while (line!=null) {
|
||||
buf.append(line);
|
||||
buf.append('\n');
|
||||
line = in.readLine();
|
||||
}
|
||||
}
|
||||
catch (IOException ioe) {
|
||||
System.err.println("can't read output from process");
|
||||
}
|
||||
}
|
||||
/** wait for the thread to finish */
|
||||
public void join() throws InterruptedException {
|
||||
sucker.join();
|
||||
}
|
||||
@Override
|
||||
public String toString() {
|
||||
return buf.toString();
|
||||
}
|
||||
}
|
||||
|
||||
protected void checkGrammarSemanticsError(ErrorQueue equeue,
|
||||
GrammarSemanticsMessage expectedMessage)
|
||||
throws Exception
|
||||
|
|
@ -869,21 +833,6 @@ public class BaseCppTest implements RuntimeTestSupport {
|
|||
}
|
||||
}
|
||||
|
||||
public static void writeFile(String dir, String fileName, String content) {
|
||||
try {
|
||||
File f = new File(dir, fileName);
|
||||
FileWriter w = new FileWriter(f);
|
||||
BufferedWriter bw = new BufferedWriter(w);
|
||||
bw.write(content);
|
||||
bw.close();
|
||||
w.close();
|
||||
}
|
||||
catch (IOException ioe) {
|
||||
System.err.println("can't write file");
|
||||
ioe.printStackTrace(System.err);
|
||||
}
|
||||
}
|
||||
|
||||
protected void mkdir(String dir) {
|
||||
File f = new File(dir);
|
||||
f.mkdirs();
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
@ -10,9 +10,10 @@ import org.antlr.v4.runtime.CommonTokenStream;
|
|||
import org.antlr.v4.runtime.Token;
|
||||
import org.antlr.v4.runtime.TokenSource;
|
||||
import org.antlr.v4.runtime.WritableToken;
|
||||
import org.antlr.v4.runtime.misc.Utils;
|
||||
import org.antlr.v4.test.runtime.ErrorQueue;
|
||||
import org.antlr.v4.test.runtime.RuntimeTestSupport;
|
||||
import org.antlr.v4.test.runtime.StreamVacuum;
|
||||
import org.antlr.v4.test.runtime.TestOutputReading;
|
||||
import org.antlr.v4.tool.ANTLRMessage;
|
||||
import org.antlr.v4.tool.GrammarSemanticsMessage;
|
||||
import org.junit.rules.TestRule;
|
||||
|
|
@ -31,7 +32,6 @@ import javax.xml.transform.stream.StreamResult;
|
|||
import javax.xml.xpath.XPathConstants;
|
||||
import javax.xml.xpath.XPathExpression;
|
||||
import javax.xml.xpath.XPathFactory;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
|
|
@ -39,6 +39,7 @@ import java.io.InputStream;
|
|||
import java.io.InputStreamReader;
|
||||
import java.io.OutputStream;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
|
|
@ -49,6 +50,7 @@ import java.util.Map;
|
|||
import java.util.Set;
|
||||
|
||||
import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
|
||||
import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
|
@ -57,6 +59,12 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
|
|||
public static final String newline = System.getProperty("line.separator");
|
||||
public static final String pathSep = System.getProperty("path.separator");
|
||||
|
||||
/**
|
||||
* When {@code true}, on Linux will call dotnet cli toolchain, otherwise
|
||||
* will continue to use mono
|
||||
*/
|
||||
public static final boolean NETSTANDARD = Boolean.parseBoolean(System.getProperty("antlr-csharp-netstandard"));
|
||||
|
||||
/**
|
||||
* When the {@code antlr.preserve-test-dir} runtime property is set to
|
||||
* {@code true}, the temporary directories created by the test run will not
|
||||
|
|
@ -240,7 +248,7 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
|
|||
return stderrDuringParse;
|
||||
}
|
||||
String output = execTest();
|
||||
if ( output.length()==0 ) {
|
||||
if ( output!=null && output.length()==0 ) {
|
||||
output = null;
|
||||
}
|
||||
return output;
|
||||
|
|
@ -351,15 +359,25 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
|
|||
}
|
||||
|
||||
public boolean compile() {
|
||||
try {
|
||||
if(!createProject())
|
||||
return false;
|
||||
if(!buildProject())
|
||||
return false;
|
||||
return true;
|
||||
} catch(Exception e) {
|
||||
return false;
|
||||
}
|
||||
if(!NETSTANDARD) {
|
||||
try {
|
||||
if(!createProject())
|
||||
return false;
|
||||
if(!buildProject())
|
||||
return false;
|
||||
return true;
|
||||
} catch(Exception e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
try {
|
||||
return createDotnetProject() && buildDotnetProject();
|
||||
} catch(Exception e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private File getTestProjectFile() {
|
||||
|
|
@ -405,11 +423,14 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
|
|||
}
|
||||
|
||||
private String locateExec() {
|
||||
return new File(tmpdir, "bin/Release/Test.exe").getAbsolutePath();
|
||||
if (!NETSTANDARD)
|
||||
return new File(tmpdir, "bin/Release/Test.exe").getAbsolutePath();
|
||||
|
||||
return new File(tmpdir, "src/bin/Debug/netcoreapp1.0/Test.dll").getAbsolutePath();
|
||||
}
|
||||
|
||||
private String locateTool(String tool) {
|
||||
String[] roots = { "/opt/local/bin/", "/usr/bin/", "/usr/local/bin/" };
|
||||
String[] roots = { "/opt/local/bin/", "/usr/local/bin/", "/usr/bin/" };
|
||||
for(String root : roots) {
|
||||
if(new File(root + tool).exists())
|
||||
return root + tool;
|
||||
|
|
@ -472,6 +493,105 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
|
|||
}
|
||||
}
|
||||
|
||||
public boolean createDotnetProject() {
|
||||
try {
|
||||
mkdir(tmpdir + "/src");
|
||||
|
||||
// move files to /src, since global.json need to be one level higher
|
||||
File source = new File(tmpdir);
|
||||
File[] files = source.listFiles();
|
||||
for (File thisSource : files) {
|
||||
if (!thisSource.isDirectory()) {
|
||||
File thisDest = new File(tmpdir + "/src/" + thisSource.getName());
|
||||
boolean success = thisSource.renameTo(thisDest);
|
||||
|
||||
if (!success) {
|
||||
throw new RuntimeException("Moving file " + thisSource + " to " + thisDest + " failed.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// save auxiliary files
|
||||
String pack = BaseCSharpTest.class.getPackage().getName().replace(".", "/") + "/";
|
||||
saveResourceAsFile(pack + "global.json", new File(tmpdir, "global.json"));
|
||||
saveResourceAsFile(pack + "project.json", new File(tmpdir, "src/project.json"));
|
||||
return true;
|
||||
}
|
||||
catch(Exception e) {
|
||||
e.printStackTrace(System.err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean buildDotnetProject() {
|
||||
// find runtime package
|
||||
final ClassLoader loader = Thread.currentThread().getContextClassLoader();
|
||||
final URL runtimeProj = loader.getResource("CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.xproj");
|
||||
if ( runtimeProj==null ) {
|
||||
throw new RuntimeException("C# runtime project file not found!");
|
||||
}
|
||||
File runtimeProjFile = new File(runtimeProj.getFile());
|
||||
String runtimeProjPath = runtimeProjFile.getParentFile().getParentFile().getPath();
|
||||
String projectRefPath = runtimeProjPath.substring(0, runtimeProjPath.lastIndexOf("/"));
|
||||
|
||||
// update global.json to reference runtime path
|
||||
try {
|
||||
String content = new java.util.Scanner(new File(tmpdir + "/global.json")).useDelimiter("\\Z").next();
|
||||
content = content.replaceAll("replace_this", projectRefPath);
|
||||
java.io.PrintWriter out = new java.io.PrintWriter(tmpdir + "/global.json");
|
||||
out.write(content);
|
||||
out.close();
|
||||
}
|
||||
catch(Exception e) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// build test
|
||||
String dotnetcli = locateTool("dotnet");
|
||||
String[] args = new String[] {
|
||||
dotnetcli,
|
||||
"restore",
|
||||
".",
|
||||
projectRefPath
|
||||
};
|
||||
|
||||
try {
|
||||
boolean success = runProcess(args, tmpdir);
|
||||
|
||||
args = new String[] {
|
||||
dotnetcli,
|
||||
"build",
|
||||
"src"
|
||||
};
|
||||
success = runProcess(args, tmpdir);
|
||||
}
|
||||
catch(Exception e) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private boolean runProcess(String[] args, String path) throws Exception {
|
||||
ProcessBuilder pb = new ProcessBuilder(args);
|
||||
pb.directory(new File(path));
|
||||
Process process = pb.start();
|
||||
StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
|
||||
StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
|
||||
stdoutVacuum.start();
|
||||
stderrVacuum.start();
|
||||
process.waitFor();
|
||||
stdoutVacuum.join();
|
||||
stderrVacuum.join();
|
||||
boolean success = process.exitValue()==0;
|
||||
if ( !success ) {
|
||||
this.stderrDuringParse = stderrVacuum.toString();
|
||||
System.err.println("runProcess stderrVacuum: "+ this.stderrDuringParse);
|
||||
}
|
||||
return success;
|
||||
}
|
||||
|
||||
private void saveResourceAsFile(String resourceName, File file) throws IOException {
|
||||
InputStream input = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName);
|
||||
if ( input==null ) {
|
||||
|
|
@ -488,26 +608,18 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
|
|||
|
||||
public String execTest() {
|
||||
String exec = locateExec();
|
||||
String[] args = getExecTestArgs(exec);
|
||||
try {
|
||||
File tmpdirFile = new File(tmpdir);
|
||||
Path output = tmpdirFile.toPath().resolve("output");
|
||||
Path errorOutput = tmpdirFile.toPath().resolve("error-output");
|
||||
String[] args = getExecTestArgs(exec, output, errorOutput);
|
||||
ProcessBuilder pb = new ProcessBuilder(args);
|
||||
pb.directory(new File(tmpdir));
|
||||
pb.directory(tmpdirFile);
|
||||
Process process = pb.start();
|
||||
StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
|
||||
StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
|
||||
stdoutVacuum.start();
|
||||
stderrVacuum.start();
|
||||
process.waitFor();
|
||||
stdoutVacuum.join();
|
||||
stderrVacuum.join();
|
||||
String output = stdoutVacuum.toString();
|
||||
if ( output.length()==0 ) {
|
||||
output = null;
|
||||
}
|
||||
if ( stderrVacuum.toString().length()>0 ) {
|
||||
this.stderrDuringParse = stderrVacuum.toString();
|
||||
}
|
||||
return output;
|
||||
String writtenOutput = TestOutputReading.read(output);
|
||||
this.stderrDuringParse = TestOutputReading.read(errorOutput);
|
||||
return writtenOutput;
|
||||
}
|
||||
catch (Exception e) {
|
||||
System.err.println("can't exec recognizer");
|
||||
|
|
@ -516,12 +628,30 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
|
|||
return null;
|
||||
}
|
||||
|
||||
private String[] getExecTestArgs(String exec) {
|
||||
if(isWindows())
|
||||
return new String[] { exec, new File(tmpdir, "input").getAbsolutePath() } ;
|
||||
private String[] getExecTestArgs(String exec, Path output, Path errorOutput) {
|
||||
if ( isWindows() ) {
|
||||
return new String[]{
|
||||
exec, new File(tmpdir, "input").getAbsolutePath(),
|
||||
output.toAbsolutePath().toString(),
|
||||
errorOutput.toAbsolutePath().toString()
|
||||
};
|
||||
}
|
||||
else {
|
||||
String mono = locateTool("mono");
|
||||
return new String[] { mono, exec, new File(tmpdir, "input").getAbsolutePath() };
|
||||
if (!NETSTANDARD) {
|
||||
String mono = locateTool("mono");
|
||||
return new String[] {
|
||||
mono, exec, new File(tmpdir, "input").getAbsolutePath(),
|
||||
output.toAbsolutePath().toString(),
|
||||
errorOutput.toAbsolutePath().toString()
|
||||
};
|
||||
}
|
||||
|
||||
String dotnet = locateTool("dotnet");
|
||||
return new String[] {
|
||||
dotnet, exec, new File(tmpdir, "src/input").getAbsolutePath(),
|
||||
output.toAbsolutePath().toString(),
|
||||
errorOutput.toAbsolutePath().toString()
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -534,41 +664,6 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
|
|||
}
|
||||
|
||||
|
||||
public static class StreamVacuum implements Runnable {
|
||||
StringBuilder buf = new StringBuilder();
|
||||
BufferedReader in;
|
||||
Thread sucker;
|
||||
public StreamVacuum(InputStream in) {
|
||||
this.in = new BufferedReader( new InputStreamReader(in) );
|
||||
}
|
||||
public void start() {
|
||||
sucker = new Thread(this);
|
||||
sucker.start();
|
||||
}
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
String line = in.readLine();
|
||||
while (line!=null) {
|
||||
buf.append(line);
|
||||
buf.append('\n');
|
||||
line = in.readLine();
|
||||
}
|
||||
}
|
||||
catch (IOException ioe) {
|
||||
System.err.println("can't read output from process");
|
||||
}
|
||||
}
|
||||
/** wait for the thread to finish */
|
||||
public void join() throws InterruptedException {
|
||||
sucker.join();
|
||||
}
|
||||
@Override
|
||||
public String toString() {
|
||||
return buf.toString();
|
||||
}
|
||||
}
|
||||
|
||||
protected void checkGrammarSemanticsError(ErrorQueue equeue,
|
||||
GrammarSemanticsMessage expectedMessage)
|
||||
throws Exception
|
||||
|
|
@ -611,16 +706,6 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
|
|||
}
|
||||
}
|
||||
|
||||
public static void writeFile(String dir, String fileName, String content) {
|
||||
try {
|
||||
Utils.writeFile(dir+"/"+fileName, content, "UTF-8");
|
||||
}
|
||||
catch (IOException ioe) {
|
||||
System.err.println("can't write file");
|
||||
ioe.printStackTrace(System.err);
|
||||
}
|
||||
}
|
||||
|
||||
protected void mkdir(String dir) {
|
||||
File f = new File(dir);
|
||||
f.mkdirs();
|
||||
|
|
@ -635,16 +720,23 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
|
|||
"using System;\n" +
|
||||
"using Antlr4.Runtime;\n" +
|
||||
"using Antlr4.Runtime.Tree;\n" +
|
||||
"using System.IO;\n" +
|
||||
"using System.Text;\n" +
|
||||
"\n" +
|
||||
"public class Test {\n" +
|
||||
" public static void Main(string[] args) {\n" +
|
||||
" ICharStream input = new AntlrFileStream(args[0]);\n" +
|
||||
" <lexerName> lex = new <lexerName>(input);\n" +
|
||||
" CommonTokenStream tokens = new CommonTokenStream(lex);\n" +
|
||||
" <createParser>\n"+
|
||||
" parser.BuildParseTree = true;\n" +
|
||||
" ParserRuleContext tree = parser.<parserStartRuleName>();\n" +
|
||||
" ParseTreeWalker.Default.Walk(new TreeShapeListener(), tree);\n" +
|
||||
" var input = CharStreams.fromPath(args[0]);\n" +
|
||||
" using (FileStream fsOut = new FileStream(args[1], FileMode.Create, FileAccess.Write))\n" +
|
||||
" using (FileStream fsErr = new FileStream(args[2], FileMode.Create, FileAccess.Write))\n" +
|
||||
" using (TextWriter output = new StreamWriter(fsOut),\n" +
|
||||
" errorOutput = new StreamWriter(fsErr)) {\n" +
|
||||
" <lexerName> lex = new <lexerName>(input, output, errorOutput);\n" +
|
||||
" CommonTokenStream tokens = new CommonTokenStream(lex);\n" +
|
||||
" <createParser>\n"+
|
||||
" parser.BuildParseTree = true;\n" +
|
||||
" ParserRuleContext tree = parser.<parserStartRuleName>();\n" +
|
||||
" ParseTreeWalker.Default.Walk(new TreeShapeListener(), tree);\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
"}\n" +
|
||||
"\n" +
|
||||
|
|
@ -663,11 +755,11 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
|
|||
" }\n" +
|
||||
"}"
|
||||
);
|
||||
ST createParserST = new ST(" <parserName> parser = new <parserName>(tokens);\n");
|
||||
ST createParserST = new ST(" <parserName> parser = new <parserName>(tokens, output, errorOutput);\n");
|
||||
if ( debug ) {
|
||||
createParserST =
|
||||
new ST(
|
||||
" <parserName> parser = new <parserName>(tokens);\n" +
|
||||
" <parserName> parser = new <parserName>(tokens, output, errorOutput);\n" +
|
||||
" parser.AddErrorListener(new DiagnosticErrorListener());\n");
|
||||
}
|
||||
outputFileST.add("createParser", createParserST);
|
||||
|
|
@ -681,17 +773,24 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
|
|||
ST outputFileST = new ST(
|
||||
"using System;\n" +
|
||||
"using Antlr4.Runtime;\n" +
|
||||
"using System.IO;\n" +
|
||||
"using System.Text;\n" +
|
||||
"\n" +
|
||||
"public class Test {\n" +
|
||||
" public static void Main(string[] args) {\n" +
|
||||
" ICharStream input = new AntlrFileStream(args[0]);\n" +
|
||||
" <lexerName> lex = new <lexerName>(input);\n" +
|
||||
" CommonTokenStream tokens = new CommonTokenStream(lex);\n" +
|
||||
" tokens.Fill();\n" +
|
||||
" foreach (object t in tokens.GetTokens())\n" +
|
||||
" Console.WriteLine(t);\n" +
|
||||
(showDFA?" Console.Write(lex.Interpreter.GetDFA(Lexer.DEFAULT_MODE).ToLexerString());\n":"")+
|
||||
" }\n" +
|
||||
" var input = CharStreams.fromPath(args[0]);\n" +
|
||||
" using (FileStream fsOut = new FileStream(args[1], FileMode.Create, FileAccess.Write))\n" +
|
||||
" using (FileStream fsErr = new FileStream(args[2], FileMode.Create, FileAccess.Write))\n" +
|
||||
" using (TextWriter output = new StreamWriter(fsOut),\n" +
|
||||
" errorOutput = new StreamWriter(fsErr)) {\n" +
|
||||
" <lexerName> lex = new <lexerName>(input, output, errorOutput);\n" +
|
||||
" CommonTokenStream tokens = new CommonTokenStream(lex);\n" +
|
||||
" tokens.Fill();\n" +
|
||||
" foreach (object t in tokens.GetTokens())\n" +
|
||||
" output.WriteLine(t);\n" +
|
||||
(showDFA?" output.Write(lex.Interpreter.GetDFA(Lexer.DEFAULT_MODE).ToLexerString());\n":"")+
|
||||
" }\n" +
|
||||
"}\n" +
|
||||
"}"
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
|
||||
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
||||
* Use of this file is governed by the BSD 3-clause license that
|
||||
* can be found in the LICENSE.txt file in the project root.
|
||||
*/
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue