Merge branch 'master' into Issue_1666

.gitignore

@@ -31,6 +31,9 @@ __pycache__/
 
 # User-specific files (MonoDevelop/Xamarin Studio)
 *.userprefs
+*.user
+.vs/
+project.lock.json
 
 # Build results
 [Dd]ebug/

.travis.yml

@@ -45,12 +45,22 @@ matrix:
     - os: linux
       jdk: oraclejdk7
       env: TARGET=csharp
+    - os: linux
+      jdk: oraclejdk7
+      dist: trusty
+      env: TARGET=dotnet
     - os: linux
       jdk: oraclejdk7
       env: TARGET=python2
     - os: linux
       jdk: oraclejdk7
       env: TARGET=python3
+      addons:
+        apt:
+          sources:
+            - deadsnakes # source required so it finds the package definition below
+          packages:
+            - python3.5
     - os: linux
       jdk: oraclejdk7
       env: TARGET=javascript

.travis/before-install-linux-cpp.sh

@@ -3,6 +3,4 @@
 set -euo pipefail
 
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-sudo add-apt-repository ppa:fkrull/deadsnakes -y
-sudo add-apt-repository ppa:rwky/nodejs -y
 sudo apt-get update -qq

.travis/before-install-linux-csharp.sh

@@ -3,8 +3,6 @@
 set -euo pipefail
 
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-sudo add-apt-repository ppa:fkrull/deadsnakes -y
-sudo add-apt-repository ppa:rwky/nodejs -y
 sudo apt-get update -qq
 echo "deb http://download.mono-project.com/repo/debian wheezy/snapshots/3.12.1 main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list
 sudo apt-get install -qq mono-complete

.travis/before-install-linux-dotnet.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# install dotnet
+sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ trusty main" > /etc/apt/sources.list.d/dotnetdev.list'
+sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 417A0893
+sudo apt-get update
+
+sudo apt-get install dotnet-dev-1.0.0-preview2.1-003177
+
+# install mvn
+wget http://apache.mirrors.lucidnetworks.net/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz && \
+    wget https://www.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz.md5 && \
+    echo "$(cat apache-maven-3.3.9-bin.tar.gz.md5)  apache-maven-3.3.9-bin.tar.gz" > apache-maven-3.3.9-bin.tar.gz.md5 && \
+    md5sum -c *.md5
+
+sudo rm -rf /usr/local/maven/ && sudo mkdir -p /usr/local/maven && \
+    sudo tar xzvf apache-maven-3.3.9-bin.tar.gz -C /usr/local/maven --strip-components=1
+
+mvn -v
+

.travis/before-install-linux-go.sh

@@ -3,8 +3,6 @@
 set -euo pipefail
 
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-sudo add-apt-repository ppa:fkrull/deadsnakes -y
-sudo add-apt-repository ppa:rwky/nodejs -y
 sudo apt-get update -qq
 eval "$(sudo gimme 1.7.3)"
 ( go version ; go env ) || true

.travis/before-install-linux-java.sh

@@ -3,6 +3,4 @@
 set -euo pipefail
 
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-sudo add-apt-repository ppa:fkrull/deadsnakes -y
-sudo add-apt-repository ppa:rwky/nodejs -y
 sudo apt-get update -qq

.travis/before-install-linux-javascript.sh

@@ -3,7 +3,7 @@
 set -euo pipefail
 
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-sudo add-apt-repository ppa:fkrull/deadsnakes -y
-sudo add-apt-repository ppa:rwky/nodejs -y
 sudo apt-get update -qq
+curl -sL https://deb.nodesource.com/setup_0.12 | sudo -E bash -
 sudo apt-get install -qq nodejs
+node --version

.travis/before-install-linux-python2.sh

@@ -3,7 +3,5 @@
 set -euo pipefail
 
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-sudo add-apt-repository ppa:fkrull/deadsnakes -y
-sudo add-apt-repository ppa:rwky/nodejs -y
 sudo apt-get update -qq
 python --version

.travis/before-install-linux-python3.sh

@@ -2,9 +2,4 @@
 
 set -euo pipefail
 
-sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-sudo add-apt-repository ppa:fkrull/deadsnakes -y
-sudo add-apt-repository ppa:rwky/nodejs -y
-sudo apt-get update -qq
-sudo apt-get install -qq python3.5
 python3 --version

.travis/run-tests-dotnet.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=csharp.* -DargLine="-Dantlr-csharp-netstandard=true" test
+

CONTRIBUTING.md

@@ -0,0 +1,10 @@
+# Contributing to ANTLR 4
+
+1. [Fork](https://help.github.com/articles/fork-a-repo) the [antlr/antlr4 repo](https://github.com/antlr/antlr4)
+2. Install and configure [EditorConfig](http://editorconfig.org/) so your text editor or IDE uses the ANTLR 4 coding style
+3. [Build ANTLR 4](doc/building-antlr.md)
+4. [Run the ANTLR project unit tests](doc/antlr-project-testing.md)
+5. Create a [pull request](https://help.github.com/articles/using-pull-requests/) including your change
+
+
+**Note:** You must sign the `contributors.txt` certificate of origin with your pull request if you've not done so before.

LICENSE.txt

@@ -24,3 +24,29 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+=====
+
+MIT License for codepointat.js from https://git.io/codepointat
+MIT License for fromcodepoint.js from https://git.io/vDW1m
+
+Copyright Mathias Bynens <https://mathiasbynens.be/>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

antlr4-maven-plugin/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>org.antlr</groupId>
     <artifactId>antlr4-master</artifactId>
-    <version>4.6</version>
+    <version>4.6.1-SNAPSHOT</version>
   </parent>
   <artifactId>antlr4-maven-plugin</artifactId>
   <packaging>maven-plugin</packaging>

antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java

@@ -150,6 +150,13 @@ public class Antlr4Mojo extends AbstractMojo {
     @Parameter(property = "project", required = true, readonly = true)
     protected MavenProject project;
 
+	/**
+	 * Specifies whether sources are added to the {@code compile} or
+	 * {@code test} scope.
+	 */
+	@Parameter(property = "antlr4.generateTestSources", defaultValue = "false")
+	private boolean generateTestSources;
+
     /**
      * The directory where the ANTLR grammar files ({@code *.g4}) are located.
      */
@@ -190,7 +197,12 @@ public class Antlr4Mojo extends AbstractMojo {
     }
 
     void addSourceRoot(File outputDir) {
-        project.addCompileSourceRoot(outputDir.getPath());
+		if (generateTestSources) {
+			project.addTestCompileSourceRoot(outputDir.getPath());
+		}
+		else {
+			project.addCompileSourceRoot(outputDir.getPath());
+		}
     }
 
     /**

contributors.txt

@@ -127,4 +127,14 @@ YYYY/MM/DD, github id, Full name, email
 2016/12/01, samtatasurya, Samuel Tatasurya, xemradiant@gmail.com
 2016/12/03, redxdev, Samuel Bloomberg, sam@redxdev.com
 2016/12/11, Gaulouis, Gaulouis, gaulouis.com@gmail.com
+2016/12/22, akosthekiss, Akos Kiss, akiss@inf.u-szeged.hu
+2016/12/24, adrpo, Adrian Pop, adrian.pop@liu.se
+2017/01/11, robertbrignull, Robert Brignull, robertbrignull@gmail.com
+2017/01/13, marcelo-rocha, Marcelo Rocha, mcrocha@gmail.com
+2017/01/23, bhamiltoncx, Ben Hamilton, bhamiltoncx+antlr@gmail.com
+2017/01/18, mshockwave, Bekket McClane, yihshyng223@gmail.com
+2017/02/10, lionelplessis, Lionel Plessis, lionelplessis@users.noreply.github.com
+2017/02/14, lecode-official, David Neumann, david.neumann@lecode.de
+2017/02/14, xied75, Dong Xie, xied75@gmail.com
 2017/02/20, Thomasb81, Thomas Burg, thomasb81@gmail.com
+

doc/ace-javascript-target.md

@@ -174,13 +174,20 @@ At this point, you should have an editor which displays an error icon next to th
 
 What remains to be done is have our validate function actually validate the input. Finally ANTLR comes in the picture!
 
-To start with, let's load ANTLR and your parser, listener etc.. Easy, since you could write:
+To start with, let's load ANTLR and your parser, listener etc.. 
+
+The preferred approach for loading parser code is to bundle your parser, [as described here](javascript-target.md).
+You can then load it as part of the importScripts instruction at the start of your worker code.
+
+Another approach is to load it using 'require'. Easy, since you could write:
 
 ```js
 var antlr4 = require('antlr4/index');
 ```
 
-This may work, but it's actually unreliable. The reason is that the require function used by ANTLR, which exactly mimics the NodeJS require function, uses a different syntax than the require function that comes with ACE. So we need to bring in a require function that conforms to the NodeJS syntax. I personally use one that comes from Torben Haase's Honey project, which you can find here. But hey, now we're going to have 2 'require' functions not compatible with each other! Indeed, this is why you need to take special care, as follows:
+This may work, but it's actually unreliable. The reason is that the 'require' function that comes with ACE uses a different syntax than the 'require' function used by ANTLR, which follows the NodeJS 'require' convention. 
+So we need to bring in a NodeJS compatible 'require' function that conforms to the NodeJS syntax. I personally use one that comes from Torben Haase's Honey project, which you can find in li/require.js. 
+But hey, now we're going to have 2 'require' functions not compatible with each other! Indeed, this is why you need to take special care, as follows:
 
 ```js
 // load nodejs compatible require
@@ -191,7 +198,8 @@ importScripts("../lib/require.js");
 var antlr4_require = require;
 require = ace_require;
 ```
-Now it's safe to load antlr, and the parsers generated for your language. Assuming that your language files (generated or hand-built) are in a folder with an index.js file that calls require for each file, your parser loading code can be as simple as follows:
+Now it's safe to load antlr and the parsers generated for your language. 
+Assuming that your language files (generated or hand-built) are in a folder with an index.js file that calls require for each file, your parser loading code can be as simple as follows:
 ```js
 // load antlr4 and myLanguage
 var antlr4, mylanguage;
@@ -204,6 +212,7 @@ try {
 }
 ```
 Please note the try-finally construct. ANTLR uses 'require' synchronously so it's perfectly safe to ignore the ACE 'require' while running ANTLR code. ACE itself does not guarantee synchronous execution, so you are much safer always switching 'require' back to 'ace_require'.
+
 Now detecting deep syntax errors in your code is a task for your ANTLR listener or visitor or whatever piece of code you've delegated this to. We're not going to describe this here, since it would require some knowledge of your language. However, detecting grammar syntax errors is something ANTLR does beautifully (isn't that why you went for ANTLR in the first place?). So what we will illustrate here is how to report grammar syntax errors. I have no doubt that from there, you will be able to extend the validator to suit your specific needs.
 Whenever ANTLR encounters an unexpected token, it fires an error. By default, the error is routed to an error listener which simply writes to the console.
 What we need to do is replace this listener by our own listener, se we can route errors to the ACE editor. First, let's create such a listener:
@@ -243,5 +252,4 @@ var validate = function(input) {
 };
 ```
 You know what? That's it! You now have an ACE editor that does syntax validation using ANTLR! I hope you find this useful, and simple enough to get started.
-What I did not address here is packaging, not something I'm an expert at. The good news is that it makes development simple, since I don't have to run any compilation process. I just edit my code, reload my editor page, and check how it goes.
+WNow wait, hey! How do you debug this? Well, as usual, using Chrome, since no other browser is able to debug worker code. What a shame...
-Now wait, hey! How do you debug this? Well, as usual, using Chrome, since neither Firefox or Safari are able to debug worker code. What a shame...

doc/cpp-target.md

@@ -79,15 +79,10 @@ This example assumes your grammar contains a parser rule named `key` for which t
 There are a couple of things that only the C++ ANTLR target has to deal with. They are described here.
 
 ### Build Aspects
-
+The code generation (by running the ANTLR4 jar) allows to specify 2 values you might find useful for better integration of the generated files into your application (both are optional):
-The code generation (by running the ANTLR4 jar) allows to specify 2 values you
-might find useful for better integration of the generated files into your
-application (both are optional):
 
 * A **namespace**: use the **`-package`** parameter to specify the namespace you want.
-* An **export macro**: especially in VC++ extra work is required to export your classes from a DLL. 
+* An **export macro**: especially in VC++ extra work is required to export your classes from a DLL. This is usually accomplished by a macro that has different values depending on whether you are creating the DLL or import it. The ANTLR4 runtime itself also uses one for its classes:
-This is usually accomplished by a macro that has different values depending on whether 
-you are creating the DLL or import it. The ANTLR4 runtime itself also uses one for its classes:
 
 ```c++
   #ifdef ANTLR4CPP_EXPORTS
@@ -100,30 +95,17 @@ you are creating the DLL or import it. The ANTLR4 runtime itself also uses one f
     #endif
   #endif
 ```
-
+Just like the `ANTLR4CPP_PUBLIC` macro here you can specify your own one for the generated classes using the **`-DexportMacro=...`** command-line parameter or
-Just like the `ANTLR4CPP_PUBLIC` macro here you can specify your own one for
-the generated classes using the **`-DexportMacro=...`** command-line parameter or
 grammar option `options {exportMacro='...';}` in your grammar file.
 
-In order to create a static lib in Visual Studio define the `ANTLR4CPP_STATIC` macro
+In order to create a static lib in Visual Studio define the `ANTLR4CPP_STATIC` macro in addition to the project settings that must be set for a static library (if you compile the runtime yourself).
-in addition to the project settings that must be set for a static library
-(if you compile the runtime yourself).
 
-For gcc and clang it is possible to use the `-fvisibility=hidden` setting to
+For gcc and clang it is possible to use the `-fvisibility=hidden` setting to hide all symbols except those that are made default-visible (which has been defined for all public classes in the runtime).
-hide all symbols except those that are made default-visible (which has been 
-defined for all public classes in the runtime).
 
 ### Memory Management
-
+Since C++ has no built-in memory management we need to take extra care. For that we rely mostly on smart pointers, which however might cause time penalties or memory side effects (like cyclic references) if not used with care. Currently however the memory household looks very stable. Generally, when you see a raw pointer in code consider this as being managed elsewehere. You should never try to manage such a pointer (delete, assign to smart pointer etc.).
-Since C++ has no built-in memory management we need to take extra care.
-For that we rely mostly on smart pointers, which however might cause time 
-penalties or memory side effects (like cyclic references) if not used with care.
-Currently however the memory household looks very stable. Generally, when you
-see a raw pointer in code consider this as being managed elsewehere. You
-should never try to manage such a pointer (delete, assign to smart pointer etc.).
 
 ### Unicode Support
-
 Encoding is mostly an input issue, i.e. when the lexer converts text input into lexer tokens. The parser is completely encoding unaware. However, lexer input in the grammar is defined by character ranges with either a single member (e.g. 'a' or [a] or [abc]), an explicit range (e.g. 'a'..'z' or [a-z]), the full Unicode range (for a wildcard) and the full Unicode range minus a sub range (for negated ranges, e.g. ~[a]). The explicit ranges (including single member ranges) are encoded in the serialized ATN by 16bit numbers, hence cannot reach beyond 0xFFFF (the Unicode BMP), while the implicit ranges can include any value (and hence support the full Unicode set, up to 0x10FFFF).
 
 > An interesting side note here is that the Java target fully supports Unicode as well, despite the inherent limitations from the serialized ATN. That's possible because the Java String class represents characters beyond the BMP as surrogate pairs (two 16bit values) and even reads them as 2 separate input characters. To make this work a character range for an identifier in a grammar must include the surrogate pairs area (for a Java parser).
@@ -133,7 +115,6 @@ The C++ target however always expects UTF-8 input (either in a string or via a w
 The differences in handling characters beyond the BMP leads to a difference between Java and C++ lexers: the character offsets may not concur. This is because Java reads two 16bit values per Unicode char (if that falls into the surrogate area) while a C++ parser only reads one 32bit value. That usually doesn't have practical consequences, but might confuse people when comparing token positions.
 
 ### Named Actions
-
 In order to help customizing the generated files there are a number of additional socalled **named actions**. These actions are tight to specific areas in the generated code and allow to add custom (target specific) code. All targets support these actions 
 
 * @parser::header

doc/index.md

@@ -16,7 +16,7 @@ Links in the documentation refer to various sections of the book but have been r
 <a href=""><img src=images/tpantlr2.png width=120></a>
 <a href=""><img src=images/tpdsl.png width=120></a>
 <a href="https://www.youtube.com/watch?v=OAoA3E-cyug"><img src=images/teronbook.png width=250></a>
-                 
+
 This documentation is a reference and summarizes grammar syntax and the key semantics of ANTLR grammars. The source code for all examples in the book, not just this chapter, are free at the publisher's website. The following video is a general tour of ANTLR 4 and includes a description of how to use parse tree listeners to process Java files easily:
 
 <a href="https://vimeo.com/59285751"><img src=images/tertalk.png width=200></a>
@@ -52,7 +52,7 @@ This documentation is a reference and summarizes grammar syntax and the key sema
 * [Runtime Libraries and Code Generation Targets](targets.md)
 
 * [Parsing binary streams](parsing-binary-files.md)
- 
+
 * [Parser and lexer interpreters](interpreters.md)
 
 * [Resources](resources.md)
@@ -61,6 +61,8 @@ This documentation is a reference and summarizes grammar syntax and the key sema
 
 * [Building ANTLR itself](building-antlr.md)
 
+* [Contributing to ANTLR](/CONTRIBUTING.md)
+
 * [Cutting an ANTLR Release](releasing-antlr.md)
 
 * [ANTLR project unit tests](antlr-project-testing.md)

doc/javascript-target.md

@@ -15,7 +15,7 @@ The tests were conducted using Selenium. No issue was found, so you should find
 
 ## Is NodeJS supported?
 
-The runtime has also been extensively tested against Node.js 0.10.33. No issue was found.
+The runtime has also been extensively tested against Node.js 0.12.7. No issue was found.
 
 ## How to create a JavaScript lexer or parser?
 
@@ -31,7 +31,9 @@ For a full list of antlr4 tool options, please visit the [tool documentation pag
 
 Once you've generated the lexer and/or parser code, you need to download the runtime.
 
-The JavaScript runtime is available from the ANTLR web site [download section](http://www.antlr.org/download/index.html). The runtime is provided in the form of source code, so no additional installation is required.
+The JavaScript runtime is [available from npm](https://www.npmjs.com/package/antlr4).
+
+If you can't use npm, the JavaScript runtime is also available from the ANTLR web site [download section](http://www.antlr.org/download/index.html). The runtime is provided in the form of source code, so no additional installation is required.
 
 We will not document here how to refer to the runtime from your project, since this would differ a lot depending on your project type and IDE. 
 
@@ -47,11 +49,28 @@ However, it would be a bit of a problem when it comes to get it into a browser.
 <script src='lib/myscript.js'>
 ```
 
-In order to avoid having to do this, and also to have the exact same code for browsers and Node.js, we rely on a script which provides the equivalent of the Node.js 'require' function.
+To avoid having doing this, the preferred approach is to bundle antlr4 with your parser code, using webpack.
 
-This script is provided by Torben Haase, and is NOT part of ANTLR JavaScript runtime, although the runtime heavily relies on it. Please note that syntax for 'require' in NodeJS is different from the one implemented by RequireJS and similar frameworks.  
+You can get [information on webpack here](https://webpack.github.io).
 
-So in short, assuming you have at the root of your web site, both the 'antlr4' directory and a 'lib' directory with 'require.js' inside it, all you need to put in your HTML header is the following:
+The steps to create your parsing code are the following:
+ - generate your lexer, parser, listener and visitor using the antlr tool
+ - write your parse tree handling code by providig your custom listener or visitor, and associated code, using 'require' to load antlr.
+ - create an index.js file with the entry point to your parsing code (or several if required).
+ - test your parsing logic thoroughly using node.js
+ 
+You are now ready to bundle your parsing code as follows:
+ - following webpack specs, create a webpack.config file
+ - in the webpack.config file, exclude node.js only modules using: node: { module: "empty", net: "empty", fs: "empty" }
+ - from the cmd line, nag-vigate to the directory containing webpack.config and type: webpack
+ 
+This will produce a single js file containing all your parsing code. Easy to include in your web pages!
+
+If you can't use webpack, you can use the lib/require.js script which implements the Node.js 'require' function in brwsers.
+
+This script is provided by Torben Haase, and is NOT part of ANTLR JavaScript runtime.   
+
+Assuming you have, at the root of your web site, both the 'antlr4' directory and a 'lib' directory with 'require.js' inside it, all you need to put in your HTML header is the following:
 
 ```xml
 <script src='lib/require.js'>
@@ -62,16 +81,6 @@ So in short, assuming you have at the root of your web site, both the 'antlr4' d
 
 This will load the runtime asynchronously.
 
-## How do I get the runtime in Node.js?
-
-Right now, there is no npm package available, so you need to register a link instead. This can be done by running the following command from the antlr4 directory:
-
-```bash
-$ npm link antlr4
-```
-
-This will install antlr4 using the package.json descriptor that comes with the script.
- 
 ## How do I run the generated lexer and/or parser?
 
 Let's suppose that your grammar is named, as above, "MyGrammar". Let's suppose this parser comprises a rule named "StartRule". The tool will have generated for you the following files:

doc/releasing-antlr.md

@@ -33,21 +33,29 @@ Edit the repository looking for 4.5 or whatever and update it. Bump version in t
  * runtime/Python3/src/antlr4/Recognizer.py
  * runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs
  * runtime/CSharp/build/version.ps1
+ * runtime/CSharp/runtime/CSharp/Package.nuspec
  * runtime/JavaScript/src/antlr4/package.json
  * runtime/JavaScript/src/antlr4/Recognizer.js
  * runtime/Cpp/VERSION
  * runtime/Cpp/runtime/src/RuntimeMetaData.cpp
  * runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake
+ * runtime/Cpp/demo/generate.cmd
+ * runtime/Go/antlr/recognizer.go
+ * runtime/Swift/Antlr4/org/antlr/v4/runtime/RuntimeMetaData.swift
+ * tool/src/org/antlr/v4/codegen/target/GoTarget.java
  * tool/src/org/antlr/v4/codegen/target/CppTarget.java
  * tool/src/org/antlr/v4/codegen/target/CSharpTarget.java
  * tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java
  * tool/src/org/antlr/v4/codegen/target/Python2Target.java
  * tool/src/org/antlr/v4/codegen/target/Python3Target.java
+ * tool/src/org/antlr/v4/codegen/target/SwiftTarget.java
+ * tool/src/org/antlr/v4/codegen/Target.java
+ * tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg
  
 Here is a simple script to display any line from the critical files with, say, `4.5` in it:
 
 ```bash
-find /tmp/antlr4 -type f -exec grep -l '4\.5' {} \;
+find tool runtime -type f -exec grep -l '4\.6' {} \;
 ```
 
 Commit to repository.
@@ -301,7 +309,7 @@ rm Antlr4.Runtime/obj/net20/Release/Antlr4.Runtime.dll
 # build
 xbuild /p:Configuration=Release Antlr4.Runtime/Antlr4.Runtime.mono.csproj
 # zip it up to get a version number on zip filename
-zip --junk-paths /tmp/antlr-csharp-runtime-4.6.zip Antlr4.Runtime/bin/net35/Release/Antlr4.Runtime.dll
+zip --junk-paths /tmp/antlr-csharp-runtime-4.6.zip Antlr4.Runtime/obj/net20/Release/Antlr4.Runtime.Standard.dll
 cp /tmp/antlr-csharp-runtime-4.6.zip ~/antlr/sites/website-antlr4/download
 ```
 
@@ -358,7 +366,7 @@ python setup.py register -r pypi
 python setup.py sdist bdist_wininst upload -r pypi
 ```
 
-Add links to the artifacts from download.html
+There are links to the artifacts in [download.html](http://www.antlr.org/download.html) already.
 
 ### C++
 
@@ -378,6 +386,7 @@ On a Mac (with XCode 7+ installed):
 ```bash
 cd runtime/Cpp
 ./deploy-macos.sh
+cp antlr4-cpp-runtime-macos.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.6-macos.zip
 ```
 
 On any Mac or Linux machine:
@@ -385,6 +394,7 @@ On any Mac or Linux machine:
 ```bash
 cd runtime/Cpp
 ./deploy-source.sh
+cp antlr4-cpp-runtime-source.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.6-source.zip
 ```
 
 On a Windows machine the build scripts checks if VS 2013 and/or VS 2015 are installed and builds binaries for each, if found. This script requires 7z to be installed (http://7-zip.org).
@@ -392,15 +402,17 @@ On a Windows machine the build scripts checks if VS 2013 and/or VS 2015 are inst
 ```bash
 cd runtime/Cpp
 deploy-windows.cmd
+cp antlr4-cpp-runtime-vs2015.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.6-vs2015.zip
 ```
 
 Move target to website (**_rename to a specific ANTLR version first if needed_**):
 
 ```bash
 pushd ~/antlr/sites/website-antlr4/download
-git add antlr4cpp-runtime-macos.zip
+# vi index.html
-git add antlr4cpp-runtime-windows.zip
+git add antlr4cpp-runtime-4.6-macos.zip
-git add antlr4cpp-runtime-source.zip
+git add antlr4cpp-runtime-4.6-windows.zip
+git add antlr4cpp-runtime-4.6-source.zip
 git commit -a -m 'update C++ runtime'
 git push origin gh-pages
 popd
@@ -408,7 +420,7 @@ popd
 
 ## Update javadoc for runtime and tool
 
-First gen javadoc:
+First, gen javadoc:
 
 ```bash
 $ cd antlr4

pom.xml

@@ -13,7 +13,7 @@
   </parent>
   <groupId>org.antlr</groupId>
   <artifactId>antlr4-master</artifactId>
-  <version>4.6</version>
+  <version>4.6.1-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>ANTLR 4</name>

runtime-testsuite/annotations/pom.xml

@@ -9,7 +9,7 @@
   <parent>
     <groupId>org.antlr</groupId>
     <artifactId>antlr4-master</artifactId>
-    <version>4.6</version>
+    <version>4.6.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
   <artifactId>antlr4-runtime-test-annotations</artifactId>

runtime-testsuite/pom.xml

@@ -10,7 +10,7 @@
 	<parent>
 		<groupId>org.antlr</groupId>
 		<artifactId>antlr4-master</artifactId>
-		<version>4.6</version>
+		<version>4.6.1-SNAPSHOT</version>
 	</parent>
 	<artifactId>antlr4-runtime-testsuite</artifactId>
 	<name>ANTLR 4 Runtime Tests (2nd generation)</name>
@@ -95,6 +95,8 @@
 				<artifactId>maven-surefire-plugin</artifactId>
 				<version>2.19.1</version>
 				<configuration>
+					<!-- SUREFIRE-951: file.encoding cannot be set via systemPropertyVariables -->
+					<argLine>-Dfile.encoding=UTF-8</argLine>
 					<includes>
 						<include>**/csharp/Test*.java</include>
 						<include>**/java/Test*.java</include>
@@ -118,6 +120,24 @@
 					</execution>
 				</executions>
 			</plugin>
+			<plugin>
+				<groupId>org.antlr</groupId>
+				<artifactId>antlr4-maven-plugin</artifactId>
+				<version>${project.version}</version>
+				<executions>
+					<execution>
+						<goals>
+							<goal>antlr4</goal>
+						</goals>
+						<configuration>
+							<sourceDirectory>${basedir}/test</sourceDirectory>
+							<outputDirectory>${project.build.directory}/generated-test-sources/antlr4</outputDirectory>
+							<visitor>true</visitor>
+							<generateTestSources>true</generateTestSources>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
 		</plugins>
 	</build>
 

runtime-testsuite/processors/pom.xml

@@ -9,7 +9,7 @@
 	<parent>
 		<groupId>org.antlr</groupId>
 		<artifactId>antlr4-master</artifactId>
-		<version>4.6</version>
+		<version>4.6.1-SNAPSHOT</version>
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 	<artifactId>antlr4-runtime-test-annotation-processors</artifactId>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CSharp.test.stg

@@ -1,6 +1,6 @@
-writeln(s) ::= <<Console.WriteLine(<s>);>>
+writeln(s) ::= <<Output.WriteLine(<s>);>>
-write(s) ::= <<Console.Write(<s>);>>
+write(s) ::= <<Output.Write(<s>);>>
-writeList(s) ::= <<Console.WriteLine(<s; separator="+">);>>
+writeList(s) ::= <<Output.WriteLine(<s; separator="+">);>>
 
 False() ::= "false"
 
@@ -176,8 +176,14 @@ public class PositionAdjustingLexerATNSimulator : LexerATNSimulator {
 BasicListener(X) ::= <<
 @parser::members {
 public class LeafListener : TBaseListener {
+	private readonly TextWriter Output;
+
+	public LeafListener(TextWriter output) {
+		Output = output;
+	}
+
 	public override void VisitTerminal(ITerminalNode node) {
-		Console.WriteLine(node.Symbol.Text);
+		Output.WriteLine(node.Symbol.Text);
 	}
 }
 }
@@ -185,7 +191,7 @@ public class LeafListener : TBaseListener {
 
 WalkListener(s) ::= <<
 ParseTreeWalker walker = new ParseTreeWalker();
-walker.Walk(new LeafListener(), <s>);
+walker.Walk(new LeafListener(Output), <s>);
 >>
 
 TreeNodeWithAltNumField(X) ::= <<
@@ -204,6 +210,12 @@ public class MyRuleNode : ParserRuleContext {
 TokenGetterListener(X) ::= <<
 @parser::members {
 public class LeafListener : TBaseListener {
+	private readonly TextWriter Output;
+
+	public LeafListener(TextWriter output) {
+		Output = output;
+	}
+
 	public override void ExitA(TParser.AContext ctx) {
 		if (ctx.ChildCount==2)
 		{
@@ -214,11 +226,11 @@ public class LeafListener : TBaseListener {
 			}
 			sb.Length = sb.Length - 2;
 			sb.Append ("]");
-			Console.Write ("{0} {1} {2}", ctx.INT (0).Symbol.Text,
+			Output.Write ("{0} {1} {2}", ctx.INT (0).Symbol.Text,
 				ctx.INT (1).Symbol.Text, sb.ToString());
 		}
 		else
-			Console.WriteLine(ctx.ID().Symbol);
+			Output.WriteLine(ctx.ID().Symbol);
 	}
 }
 }
@@ -227,12 +239,18 @@ public class LeafListener : TBaseListener {
 RuleGetterListener(X) ::= <<
 @parser::members {
 public class LeafListener : TBaseListener {
+	private readonly TextWriter Output;
+
+	public LeafListener(TextWriter output) {
+		Output = output;
+	}
+
 	public override void ExitA(TParser.AContext ctx) {
 		if (ctx.ChildCount==2) {
-			Console.Write("{0} {1} {2}",ctx.b(0).Start.Text,
+			Output.Write("{0} {1} {2}",ctx.b(0).Start.Text,
 				ctx.b(1).Start.Text,ctx.b()[0].Start.Text);
 		} else
-			Console.WriteLine(ctx.b(0).Start.Text);
+			Output.WriteLine(ctx.b(0).Start.Text);
 	}
 }
 }
@@ -242,12 +260,18 @@ public class LeafListener : TBaseListener {
 LRListener(X) ::= <<
 @parser::members {
 public class LeafListener : TBaseListener {
+	private readonly TextWriter Output;
+
+	public LeafListener(TextWriter output) {
+		Output = output;
+	}
+
 	public override void ExitE(TParser.EContext ctx) {
 		if (ctx.ChildCount==3) {
-			Console.Write("{0} {1} {2}\n",ctx.e(0).Start.Text,
+			Output.Write("{0} {1} {2}\n",ctx.e(0).Start.Text,
 				ctx.e(1).Start.Text, ctx.e()[0].Start.Text);
 		} else
-			Console.WriteLine(ctx.INT().Symbol.Text);
+			Output.WriteLine(ctx.INT().Symbol.Text);
 	}
 }
 }
@@ -256,24 +280,22 @@ public class LeafListener : TBaseListener {
 LRWithLabelsListener(X) ::= <<
 @parser::members {
 public class LeafListener : TBaseListener {
+	private readonly TextWriter Output;
+
+	public LeafListener(TextWriter output) {
+		Output = output;
+	}
+
 	public override void ExitCall(TParser.CallContext ctx) {
-		Console.Write("{0} {1}",ctx.e().Start.Text,ctx.eList());
+		Output.Write("{0} {1}",ctx.e().Start.Text,ctx.eList());
 	}
 	public override void ExitInt(TParser.IntContext ctx) {
-		Console.WriteLine(ctx.INT().Symbol.Text);
+		Output.WriteLine(ctx.INT().Symbol.Text);
 	}
 }
 }
 >>
 
-ImportVisitor(X) ::= ""
-BasicVisitor(X) ::= ""
-WalkVisitor(s) ::= ""
-LRWithLabelsVisitor(X) ::= ""
-RuleGetterVisitor(X) ::= ""
-LRVisitor(x) ::= ""
-TokenGetterVisitor(x) ::= ""
-
 DeclareContextListGettersFunction() ::= <<
 void foo() {
 	SContext s = null;
@@ -282,12 +304,12 @@ void foo() {
 }
 >>
 
-Declare_foo() ::= <<public void foo() {Console.WriteLine("foo");}>>
+Declare_foo() ::= <<public void foo() {Output.WriteLine("foo");}>>
 
 Invoke_foo() ::= "this.foo();"
 
 Declare_pred() ::= <<bool pred(bool v) {
-	Console.WriteLine("eval="+v.ToString().ToLower());
+	Output.WriteLine("eval="+v.ToString().ToLower());
 	return v;
 }
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Cpp.test.stg

@@ -243,14 +243,6 @@ public:
 }
 >>
 
-ImportVisitor(X) ::= ""
-BasicVisitor(X) ::= ""
-WalkVisitor(s) ::= ""
-LRWithLabelsVisitor(X) ::= ""
-RuleGetterVisitor(X) ::= ""
-LRVisitor(x) ::= ""
-TokenGetterVisitor(x) ::= ""
-
 DeclareContextListGettersFunction() ::= <<
 void foo() {
   SContext *s;

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Go.test.stg

@@ -298,14 +298,6 @@ func (*LeafListener) ExitInt(ctx *IntContext) {
 }
 >>
 
-ImportVisitor(X) ::= ""
-BasicVisitor(X) ::= ""
-WalkVisitor(s) ::= ""
-LRWithLabelsVisitor(X) ::= ""
-RuleGetterVisitor(X) ::= ""
-LRVisitor(x) ::= ""
-TokenGetterVisitor(x) ::= ""
-
 DeclareContextListGettersFunction() ::= <<
 func foo() {
 	// TODO
@@ -333,5 +325,5 @@ func pred(v bool) bool {
 
 Invoke_pred(v) ::= <<pred(<v>)>>
 ContextRuleFunction(ctx, rule) ::= "<ctx>.<rule>"
-StringType() ::= "String"
+StringType() ::= "string"
 ContextMember(ctx, subctx, member) ::= "<ctx>.<subctx>.<member; format={cap}>"

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Java.test.stg

@@ -262,14 +262,6 @@ public static class LeafListener extends TBaseListener {
 }
 >>
 
-ImportVisitor(X) ::= ""
-BasicVisitor(X) ::= ""
-WalkVisitor(s) ::= ""
-LRWithLabelsVisitor(X) ::= ""
-RuleGetterVisitor(X) ::= ""
-LRVisitor(x) ::= ""
-TokenGetterVisitor(x) ::= ""
-
 DeclareContextListGettersFunction() ::= <<
 void foo() {
 	SContext s = null;

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Node.test.stg

@@ -76,8 +76,6 @@ var <X>Listener = require('./<X>Listener').<X>Listener;
 }
 >>
 
-ImportVisitor(X) ::= <<var <X>Visitor = require('./<X>Visitor').<X>Visitor;>>
-
 GetExpectedTokenNames() ::= "this.getExpectedTokens().toString(this.literalNames)"
 
 RuleInvocationStack() ::= "antlr4.Utils.arrayToString(this.getRuleInvocationStack())"
@@ -182,24 +180,6 @@ var walker = new antlr4.tree.ParseTreeWalker();
 walker.walk(new this.LeafListener(), <s>);
 >>
 
-BasicVisitor(X) ::= <<
-this.LeafVisitor = function() {
-    this.visitTerminal = function(node) {
-        return node.symbol.text;
-    };
-    return this;
-};
-this.LeafVisitor.prototype = Object.create(<X>Visitor.prototype);
-this.LeafVisitor.prototype.constructor = this.LeafVisitor;
-
->>
-
-WalkVisitor(s) ::= <<
-var visitor = new this.LeafVisitor();
-console.log(<s>.accept(visitor));
->>
-
-
 TreeNodeWithAltNumField(X) ::= <<
 @parser::header {
 MyRuleNode = function(parent, invokingState) {
@@ -236,24 +216,6 @@ this.LeafListener.prototype.constructor = this.LeafListener;
 }
 >>
 
-TokenGetterVisitor(X) ::= <<
-this.LeafVisitor = function() {
-    this.visitA = function(ctx) {
-        var str;
-        if(ctx.getChildCount()===2) {
-            str = ctx.INT(0).symbol.text + ' ' + ctx.INT(1).symbol.text + ' ' + antlr4.Utils.arrayToString(ctx.INT());
-        } else {
-            str = ctx.ID().symbol.toString();
-        }
-        return this.visitChildren(ctx) + str;
-    };
-    return this;
-};
-this.LeafVisitor.prototype = Object.create(<X>Visitor.prototype);
-this.LeafVisitor.prototype.constructor = this.LeafVisitor;
-
->>
-
 RuleGetterListener(X) ::= <<
 @parser::members {
 this.LeafListener = function() {
@@ -273,25 +235,6 @@ this.LeafListener.prototype.constructor = this.LeafListener;
 }
 >>
 
-RuleGetterVisitor(X) ::= <<
-this.LeafVisitor = function() {
-    this.visitA = function(ctx) {
-        var str;
-        if(ctx.getChildCount()===2) {
-            str = ctx.b(0).start.text + ' ' + ctx.b(1).start.text + ' ' + ctx.b()[0].start.text;
-        } else {
-            str = ctx.b(0).start.text;
-        }
-        return this.visitChildren(ctx) + str;
-    };
-    return this;
-};
-this.LeafVisitor.prototype = Object.create(<X>Visitor.prototype);
-this.LeafVisitor.prototype.constructor = this.LeafVisitor;
-
->>
-
-
 LRListener(X) ::= <<
 @parser::members {
 this.LeafListener = function() {
@@ -311,24 +254,6 @@ this.LeafListener.prototype.constructor = this.LeafListener;
 }
 >>
 
-LRVisitor(X) ::= <<
-this.LeafVisitor = function() {
-    this.visitE = function(ctx) {
-        var str;
-        if(ctx.getChildCount()===3) {
-            str = ctx.e(0).start.text + ' ' + ctx.e(1).start.text + ' ' + ctx.e()[0].start.text;
-        } else {
-            str = ctx.INT().symbol.text;
-        }
-        return this.visitChildren(ctx) + str;
-    };
-    return this;
-};
-this.LeafVisitor.prototype = Object.create(<X>Visitor.prototype);
-this.LeafVisitor.prototype.constructor = this.LeafVisitor;
-
->>
-
 LRWithLabelsListener(X) ::= <<
 @parser::members {
 this.LeafListener = function() {
@@ -347,23 +272,6 @@ this.LeafListener.prototype.constructor = this.LeafListener;
 }
 >>
 
-LRWithLabelsVisitor(X) ::= <<
-this.LeafVisitor = function() {
-    this.visitCall = function(ctx) {
-        var str = ctx.e().start.text + ' ' + ctx.eList();
-        return this.visitChildren(ctx) + str;
-    };
-    this.visitInt = function(ctx) {
-        var str = ctx.INT().symbol.text;
-        return this.visitChildren(ctx) + str;
-    };
-    return this;
-};
-this.LeafVisitor.prototype = Object.create(<X>Visitor.prototype);
-this.LeafVisitor.prototype.constructor = this.LeafVisitor;
-
->>
-
 DeclareContextListGettersFunction() ::= <<
 	function foo() {
 		var s = new SContext();

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python2.test.stg

@@ -1,6 +1,6 @@
-writeln(s) ::= <<print(<s>)>>
+writeln(s) ::= <<print(<s>, file=self._output)>>
-write(s) ::= <<print(<s>,end='')>>
+write(s) ::= <<print(<s>,end='', file=self._output)>>
-writeList(s) ::= <<print(<s: {v | str(<v>)}; separator="+">)>>
+writeList(s) ::= <<print(<s: {v | str(<v>)}; separator="+">, file=self._output)>>
 
 False() ::= "False"
 
@@ -152,14 +152,16 @@ else:
     from <X>Listener import <X>Listener
 
 class LeafListener(TListener):
+    def __init__(self, output):
+        self._output = output
     def visitTerminal(self, node):
-        print(node.symbol.text)
+        print(node.symbol.text, file=self._output)
 }
 >>
 
 WalkListener(s) ::= <<
 walker = ParseTreeWalker()
-walker.walk(TParser.LeafListener(), <s>)
+walker.walk(TParser.LeafListener(self._output), <s>)
 >>
 
 TreeNodeWithAltNumField(X) ::= <<
@@ -183,11 +185,13 @@ else:
     from <X>Listener import <X>Listener
 
 class LeafListener(TListener):
+    def __init__(self, output):
+        self._output = output
     def exitA(self, ctx):
         if ctx.getChildCount()==2:
-            print(ctx.INT(0).symbol.text + ' ' + ctx.INT(1).symbol.text + ' ' + str_list(ctx.INT()))
+            print(ctx.INT(0).symbol.text + ' ' + ctx.INT(1).symbol.text + ' ' + str_list(ctx.INT()), file=self._output)
         else:
-            print(str(ctx.ID().symbol))
+            print(str(ctx.ID().symbol), file=self._output)
 }
 >>
 
@@ -199,11 +203,13 @@ else:
     from <X>Listener import <X>Listener
 
 class LeafListener(TListener):
+    def __init__(self, output):
+        self._output = output
     def exitA(self, ctx):
         if ctx.getChildCount()==2:
-            print(ctx.b(0).start.text + ' ' + ctx.b(1).start.text + ' ' + ctx.b()[0].start.text)
+            print(ctx.b(0).start.text + ' ' + ctx.b(1).start.text + ' ' + ctx.b()[0].start.text, file=self._output)
         else:
-            print(ctx.b(0).start.text)
+            print(ctx.b(0).start.text, file=self._output)
 }
 >>
 
@@ -216,11 +222,13 @@ else:
     from <X>Listener import <X>Listener
 
 class LeafListener(TListener):
+    def __init__(self, output):
+        self._output = output
     def exitE(self, ctx):
         if ctx.getChildCount()==3:
-            print(ctx.e(0).start.text + ' ' + ctx.e(1).start.text + ' ' + ctx.e()[0].start.text)
+            print(ctx.e(0).start.text + ' ' + ctx.e(1).start.text + ' ' + ctx.e()[0].start.text, file=self._output)
         else:
-            print(ctx.INT().symbol.text)
+            print(ctx.INT().symbol.text, file=self._output)
 }
 >>
 
@@ -232,21 +240,15 @@ else:
     from <X>Listener import <X>Listener
 
 class LeafListener(TListener):
+    def __init__(self, output):
+        self._output = output
     def exitCall(self, ctx):
-        print(ctx.e().start.text + ' ' + str(ctx.eList()))
+        print(ctx.e().start.text + ' ' + str(ctx.eList()), file=self._output)
     def exitInt(self, ctx):
-        print(ctx.INT().symbol.text)
+        print(ctx.INT().symbol.text, file=self._output)
 }
 >>
 
-ImportVisitor(X) ::= ""
-BasicVisitor(X) ::= ""
-WalkVisitor(s) ::= ""
-LRWithLabelsVisitor(X) ::= ""
-RuleGetterVisitor(X) ::= ""
-LRVisitor(x) ::= ""
-TokenGetterVisitor(x) ::= ""
-
 DeclareContextListGettersFunction() ::= <<
 def foo():
     s = SContext()
@@ -255,13 +257,13 @@ def foo():
 >>
 
 Declare_foo() ::= <<def foo(self):
-	print('foo')
+	print('foo', file=self._output)
 >>
 
 Invoke_foo() ::= "self.foo()"
 
 Declare_pred() ::= <<def pred(self, v):
-	print('eval=' + str(v).lower())
+	print('eval=' + str(v).lower(), file=self._output)
 	return v
 
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Python3.test.stg

@@ -1,6 +1,6 @@
-writeln(s) ::= <<print(<s>)>>
+writeln(s) ::= <<print(<s>, file=self._output)>>
-write(s) ::= <<print(<s>,end='')>>
+write(s) ::= <<print(<s>,end='',file=self._output)>>
-writeList(s) ::= <<print(<s: {v | str(<v>)}; separator="+">)>>
+writeList(s) ::= <<print(<s: {v | str(<v>)}; separator="+">, file=self._output)>>
 
 False() ::= "False"
 
@@ -152,8 +152,10 @@ def isIdentifierChar(c):
 BasicListener(X) ::= <<
 @parser::members {
 class LeafListener(MockListener):
+    def __init__(self, output):
+        self._output = output
     def visitTerminal(self, node):
-        print(node.symbol.text)
+        print(node.symbol.text, file=self._output)
 }
 >>
 
@@ -164,7 +166,7 @@ else:
     from TListener import TListener
 TParser.LeafListener.__bases__ = (TListener,)
 walker = ParseTreeWalker()
-walker.walk(TParser.LeafListener(), <s>)
+walker.walk(TParser.LeafListener(self._output), <s>)
 >>
 
 TreeNodeWithAltNumField(X) ::= <<
@@ -183,22 +185,26 @@ class MyRuleNode(ParserRuleContext):
 TokenGetterListener(X) ::= <<
 @parser::members {
 class LeafListener(MockListener):
+    def __init__(self, output):
+        self._output = output
     def exitA(self, ctx):
         if ctx.getChildCount()==2:
-            print(ctx.INT(0).symbol.text + ' ' + ctx.INT(1).symbol.text + ' ' + str_list(ctx.INT()))
+            print(ctx.INT(0).symbol.text + ' ' + ctx.INT(1).symbol.text + ' ' + str_list(ctx.INT()), file=self._output)
         else:
-            print(str(ctx.ID().symbol))
+            print(str(ctx.ID().symbol), file=self._output)
 }
 >>
 
 RuleGetterListener(X) ::= <<
 @parser::members {
 class LeafListener(MockListener):
+    def __init__(self, output):
+        self._output = output
     def exitA(self, ctx):
         if ctx.getChildCount()==2:
-            print(ctx.b(0).start.text + ' ' + ctx.b(1).start.text + ' ' + ctx.b()[0].start.text)
+            print(ctx.b(0).start.text + ' ' + ctx.b(1).start.text + ' ' + ctx.b()[0].start.text, file=self._output)
         else:
-            print(ctx.b(0).start.text)
+            print(ctx.b(0).start.text, file=self._output)
 }
 >>
 
@@ -206,32 +212,28 @@ class LeafListener(MockListener):
 LRListener(X) ::= <<
 @parser::members {
 class LeafListener(MockListener):
+    def __init__(self, output):
+        self._output = output
     def exitE(self, ctx):
         if ctx.getChildCount()==3:
-            print(ctx.e(0).start.text + ' ' + ctx.e(1).start.text + ' ' + ctx.e()[0].start.text)
+            print(ctx.e(0).start.text + ' ' + ctx.e(1).start.text + ' ' + ctx.e()[0].start.text, file=self._output)
         else:
-            print(ctx.INT().symbol.text)
+            print(ctx.INT().symbol.text, file=self._output)
 }
 >>
 
 LRWithLabelsListener(X) ::= <<
 @parser::members {
 class LeafListener(MockListener):
+    def __init__(self, output):
+        self._output = output
     def exitCall(self, ctx):
-        print(ctx.e().start.text + ' ' + str(ctx.eList()))
+        print(ctx.e().start.text + ' ' + str(ctx.eList()), file=self._output)
     def exitInt(self, ctx):
-        print(ctx.INT().symbol.text)
+        print(ctx.INT().symbol.text, file=self._output)
 }
 >>
 
-ImportVisitor(X) ::= ""
-BasicVisitor(X) ::= ""
-WalkVisitor(s) ::= ""
-LRWithLabelsVisitor(X) ::= ""
-RuleGetterVisitor(X) ::= ""
-LRVisitor(x) ::= ""
-TokenGetterVisitor(x) ::= ""
-
 DeclareContextListGettersFunction() ::= <<
 def foo():
     s = SContext()
@@ -240,13 +242,13 @@ def foo():
 >>
 
 Declare_foo() ::= <<def foo(self):
-	print('foo')
+	print('foo', file=self._output)
 >>
 
 Invoke_foo() ::= "self.foo()"
 
 Declare_pred() ::= <<def pred(self, v):
-	print('eval=' + str(v).lower())
+	print('eval=' + str(v).lower(), file=self._output)
 	return v
 
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Swift.test.stg

@@ -271,14 +271,6 @@ open class LeafListener: TBaseListener {
 }
 >>
 
-ImportVisitor(X) ::= ""
-BasicVisitor(X) ::= ""
-WalkVisitor(s) ::= ""
-LRWithLabelsVisitor(X) ::= ""
-RuleGetterVisitor(X) ::= ""
-LRVisitor(x) ::= ""
-TokenGetterVisitor(x) ::= ""
-
 DeclareContextListGettersFunction() ::= <<
 func foo() {
 	//let s: SContext? = nil

runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java

@@ -224,7 +224,7 @@ public abstract class BaseRuntimeTest {
 	                                       String... extraOptions)
 	{
 		mkdir(workdir);
-		BaseJavaTest.writeFile(workdir, grammarFileName, grammarStr);
+		writeFile(workdir, grammarFileName, grammarStr);
 		return antlrOnString(workdir, targetName, grammarFileName, defaultListener, extraOptions);
 	}
 

runtime-testsuite/test/org/antlr/v4/test/runtime/StreamVacuum.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+package org.antlr.v4.test.runtime;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public final class StreamVacuum implements Runnable {
+	private StringBuilder buf = new StringBuilder();
+	private BufferedReader in;
+	private Thread sucker;
+	public StreamVacuum(InputStream in) {
+		this.in = new BufferedReader( new InputStreamReader(in, StandardCharsets.UTF_8) );
+	}
+	public void start() {
+		sucker = new Thread(this);
+		sucker.start();
+	}
+	@Override
+	public void run() {
+		try {
+			TestOutputReading.append(in, buf);
+		}
+		catch (IOException ioe) {
+			System.err.println("can't read output from process");
+		}
+	}
+	/** wait for the thread to finish */
+	public void join() throws InterruptedException {
+		sucker.join();
+	}
+	@Override
+	public String toString() {
+		return buf.toString();
+	}
+}

runtime-testsuite/test/org/antlr/v4/test/runtime/TestOutputReading.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+package org.antlr.v4.test.runtime;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+public abstract class TestOutputReading {
+	public static void append(BufferedReader in, StringBuilder buf) throws IOException {
+		String line = in.readLine();
+		while (line!=null) {
+			buf.append(line);
+			// NOTE: This appends a newline at EOF
+			// regardless of whether or not the
+			// input actually ended with a
+			// newline.
+			//
+			// We should revisit this and read a
+			// block at a time rather than a line
+			// at a time, and change all tests
+			// which rely on this behavior to
+			// remove the trailing newline at EOF.
+			//
+			// When we fix this, we can remove the
+			// TestOutputReading class entirely.
+			buf.append('\n');
+			line = in.readLine();
+		}
+	}
+
+	/**
+	 * Read in the UTF-8 bytes at {@code path}, convert all
+	 * platform-specific line terminators to NL, and append NL
+	 * if the file was non-empty and didn't already end with one.
+	 *
+	 * {@see StreamVacuum#run()} for why this method exists.
+	 *
+	 * Returns {@code null} if the file does not exist or the output
+	 * was empty.
+	 */
+	public static String read(Path path) throws IOException {
+		// Mimic StreamVacuum.run()'s behavior of replacing all platform-specific
+		// EOL sequences with NL.
+		StringBuilder buf = new StringBuilder();
+		try (BufferedReader in = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
+			append(in, buf);
+		} catch (FileNotFoundException | NoSuchFileException e) {
+			return null;
+		}
+		if (buf.length() > 0) {
+			return buf.toString();
+		} else {
+			return null;
+		}
+	}
+}

runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/BaseCppTest.java

@@ -36,6 +36,7 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.semantics.SemanticPipeline;
 import org.antlr.v4.test.runtime.ErrorQueue;
 import org.antlr.v4.test.runtime.RuntimeTestSupport;
+import org.antlr.v4.test.runtime.StreamVacuum;
 import org.antlr.v4.tool.ANTLRMessage;
 import org.antlr.v4.tool.DOTGenerator;
 import org.antlr.v4.tool.Grammar;
@@ -46,13 +47,7 @@ import org.stringtemplate.v4.ST;
 import org.stringtemplate.v4.STGroup;
 import org.stringtemplate.v4.STGroupString;
 
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
 import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.net.URISyntaxException;
@@ -70,6 +65,7 @@ import java.util.Set;
 import java.util.TreeMap;
 
 import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
+import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -751,41 +747,6 @@ public class BaseCppTest implements RuntimeTestSupport {
 		}
 	}
 
-	public static class StreamVacuum implements Runnable {
-		StringBuilder buf = new StringBuilder();
-		BufferedReader in;
-		Thread sucker;
-		public StreamVacuum(InputStream in) {
-			this.in = new BufferedReader( new InputStreamReader(in) );
-		}
-		public void start() {
-			sucker = new Thread(this);
-			sucker.start();
-		}
-		@Override
-		public void run() {
-			try {
-				String line = in.readLine();
-				while (line!=null) {
-					buf.append(line);
-					buf.append('\n');
-					line = in.readLine();
-				}
-			}
-			catch (IOException ioe) {
-				System.err.println("can't read output from process");
-			}
-		}
-		/** wait for the thread to finish */
-		public void join() throws InterruptedException {
-			sucker.join();
-		}
-		@Override
-		public String toString() {
-			return buf.toString();
-		}
-	}
-
 	protected void checkGrammarSemanticsError(ErrorQueue equeue,
 	                                          GrammarSemanticsMessage expectedMessage)
 		throws Exception
@@ -869,21 +830,6 @@ public class BaseCppTest implements RuntimeTestSupport {
 		}
 	}
 
-	public static void writeFile(String dir, String fileName, String content) {
-		try {
-			File f = new File(dir, fileName);
-			FileWriter w = new FileWriter(f);
-			BufferedWriter bw = new BufferedWriter(w);
-			bw.write(content);
-			bw.close();
-			w.close();
-		}
-		catch (IOException ioe) {
-			System.err.println("can't write file");
-			ioe.printStackTrace(System.err);
-		}
-	}
-
 	protected void mkdir(String dir) {
 		File f = new File(dir);
 		f.mkdirs();

runtime-testsuite/test/org/antlr/v4/test/runtime/cpp/TestVisitors.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-package org.antlr.v4.test.runtime.cpp;
-
-import org.antlr.v4.test.runtime.BaseRuntimeTest;
-import org.antlr.v4.test.runtime.RuntimeTestDescriptor;
-import org.antlr.v4.test.runtime.descriptors.VisitorsDescriptors;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class TestVisitors extends BaseRuntimeTest {
-	public TestVisitors(RuntimeTestDescriptor descriptor) {
-		super(descriptor,new BaseCppTest());
-	}
-
-	@Parameterized.Parameters(name="{0}")
-	public static RuntimeTestDescriptor[] getAllTestDescriptors() {
-		return BaseRuntimeTest.getRuntimeTestDescriptors(VisitorsDescriptors.class, "Cpp");
-	}
-}

runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java

@@ -10,9 +10,10 @@ import org.antlr.v4.runtime.CommonTokenStream;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.TokenSource;
 import org.antlr.v4.runtime.WritableToken;
-import org.antlr.v4.runtime.misc.Utils;
 import org.antlr.v4.test.runtime.ErrorQueue;
 import org.antlr.v4.test.runtime.RuntimeTestSupport;
+import org.antlr.v4.test.runtime.StreamVacuum;
+import org.antlr.v4.test.runtime.TestOutputReading;
 import org.antlr.v4.tool.ANTLRMessage;
 import org.antlr.v4.tool.GrammarSemanticsMessage;
 import org.junit.rules.TestRule;
@@ -31,7 +32,6 @@ import javax.xml.transform.stream.StreamResult;
 import javax.xml.xpath.XPathConstants;
 import javax.xml.xpath.XPathExpression;
 import javax.xml.xpath.XPathFactory;
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -39,6 +39,7 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.net.URL;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -49,6 +50,7 @@ import java.util.Map;
 import java.util.Set;
 
 import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
+import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -57,6 +59,12 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
 	public static final String newline = System.getProperty("line.separator");
 	public static final String pathSep = System.getProperty("path.separator");
 
+    /**
+     * When {@code true}, on Linux will call dotnet cli toolchain, otherwise
+     * will continue to use mono
+     */
+    public static final boolean NETSTANDARD = Boolean.parseBoolean(System.getProperty("antlr-csharp-netstandard"));
+
 	/**
 	 * When the {@code antlr.preserve-test-dir} runtime property is set to
 	 * {@code true}, the temporary directories created by the test run will not
@@ -240,7 +248,7 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
 			return stderrDuringParse;
 		}
 		String output = execTest();
-		if ( output.length()==0 ) {
+		if ( output!=null && output.length()==0 ) {
 			output = null;
 		}
 		return output;
@@ -351,15 +359,25 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
 	}
 
 	public boolean compile() {
-		try {
+        if(!NETSTANDARD) {
-			if(!createProject())
+            try {
-				return false;
+                if(!createProject())
-			if(!buildProject())
+                    return false;
-				return false;
+                if(!buildProject())
-			return true;
+                    return false;
-		} catch(Exception e) {
+                return true;
-			return false;
+            } catch(Exception e) {
-		}
+                return false;
+            }
+        }
+        else
+        {
+            try {
+                return createDotnetProject() && buildDotnetProject();
+            } catch(Exception e) {
+                return false;
+            }
+        }
 	}
 
 	private File getTestProjectFile() {
@@ -405,11 +423,14 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
 	}
 
 	private String locateExec() {
-		return new File(tmpdir, "bin/Release/Test.exe").getAbsolutePath();
+        if (!NETSTANDARD)
+            return new File(tmpdir, "bin/Release/Test.exe").getAbsolutePath();
+
+        return new File(tmpdir, "src/bin/Debug/netcoreapp1.0/Test.dll").getAbsolutePath();
 	}
 
 	private String locateTool(String tool) {
-		String[] roots = { "/opt/local/bin/", "/usr/bin/", "/usr/local/bin/" };
+		String[] roots = { "/opt/local/bin/", "/usr/local/bin/", "/usr/bin/" };
 		for(String root : roots) {
 			if(new File(root + tool).exists())
 				return root + tool;
@@ -472,6 +493,105 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
 		}
 	}
 
+    public boolean createDotnetProject() {
+        try {
+            mkdir(tmpdir + "/src");
+
+            // move files to /src, since global.json need to be one level higher
+            File source = new File(tmpdir);
+            File[] files = source.listFiles();
+            for (File thisSource : files) {
+                if (!thisSource.isDirectory()) {
+                    File thisDest = new File(tmpdir + "/src/" + thisSource.getName());
+                    boolean success = thisSource.renameTo(thisDest);
+
+                    if (!success) {
+                        throw new RuntimeException("Moving file " + thisSource + " to " + thisDest + " failed.");
+                    }
+                }
+            }
+
+            // save auxiliary files
+            String pack = BaseCSharpTest.class.getPackage().getName().replace(".", "/") + "/";
+            saveResourceAsFile(pack + "global.json", new File(tmpdir, "global.json"));
+            saveResourceAsFile(pack + "project.json", new File(tmpdir, "src/project.json"));
+            return true;
+        }
+        catch(Exception e) {
+            e.printStackTrace(System.err);
+            return false;
+        }
+    }
+
+    public boolean buildDotnetProject() {
+        // find runtime package
+        final ClassLoader loader = Thread.currentThread().getContextClassLoader();
+        final URL runtimeProj = loader.getResource("CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.xproj");
+        if ( runtimeProj==null ) {
+            throw new RuntimeException("C# runtime project file not found!");
+        }
+        File runtimeProjFile = new File(runtimeProj.getFile());
+        String runtimeProjPath = runtimeProjFile.getParentFile().getParentFile().getPath();
+        String projectRefPath = runtimeProjPath.substring(0, runtimeProjPath.lastIndexOf("/"));
+
+        // update global.json to reference runtime path
+        try {
+            String content = new java.util.Scanner(new File(tmpdir + "/global.json")).useDelimiter("\\Z").next();
+            content = content.replaceAll("replace_this", projectRefPath);
+            java.io.PrintWriter out = new java.io.PrintWriter(tmpdir + "/global.json");
+            out.write(content);
+            out.close();
+        }
+        catch(Exception e) {
+            return false;
+        }
+
+        // build test
+        String dotnetcli = locateTool("dotnet");
+        String[] args = new String[] {
+            dotnetcli,
+            "restore",
+            ".",
+            projectRefPath
+        };
+
+        try {
+            boolean success = runProcess(args, tmpdir);
+
+            args = new String[] {
+                dotnetcli,
+                "build",
+                "src"
+            };
+            success = runProcess(args, tmpdir);
+        }
+        catch(Exception e) {
+            return false;
+        }
+
+
+        return true;
+    }
+
+    private boolean runProcess(String[] args, String path) throws Exception {
+        ProcessBuilder pb = new ProcessBuilder(args);
+        pb.directory(new File(path));
+        Process process = pb.start();
+        StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
+        StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
+        stdoutVacuum.start();
+        stderrVacuum.start();
+        process.waitFor();
+        stdoutVacuum.join();
+        stderrVacuum.join();
+        boolean success = process.exitValue()==0;
+        if ( !success ) {
+            this.stderrDuringParse = stderrVacuum.toString();
+            System.err.println("runProcess stderrVacuum: "+ this.stderrDuringParse);
+        }
+        return success;
+    }
+
 	private void saveResourceAsFile(String resourceName, File file) throws IOException {
 		InputStream input = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName);
 		if ( input==null ) {
@@ -488,26 +608,18 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
 
 	public String execTest() {
 		String exec = locateExec();
-		String[] args = getExecTestArgs(exec);
 		try {
+			File tmpdirFile = new File(tmpdir);
+			Path output = tmpdirFile.toPath().resolve("output");
+			Path errorOutput = tmpdirFile.toPath().resolve("error-output");
+			String[] args = getExecTestArgs(exec, output, errorOutput);
 			ProcessBuilder pb = new ProcessBuilder(args);
-			pb.directory(new File(tmpdir));
+			pb.directory(tmpdirFile);
 			Process process = pb.start();
-			StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
-			StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
-			stdoutVacuum.start();
-			stderrVacuum.start();
 			process.waitFor();
-			stdoutVacuum.join();
+			String writtenOutput = TestOutputReading.read(output);
-			stderrVacuum.join();
+			this.stderrDuringParse = TestOutputReading.read(errorOutput);
-			String output = stdoutVacuum.toString();
+			return writtenOutput;
-			if ( output.length()==0 ) {
-				output = null;
-			}
-			if ( stderrVacuum.toString().length()>0 ) {
-				this.stderrDuringParse = stderrVacuum.toString();
-			}
-			return output;
 		}
 		catch (Exception e) {
 			System.err.println("can't exec recognizer");
@@ -516,12 +628,30 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
 		return null;
 	}
 
-	private String[] getExecTestArgs(String exec) {
+	private String[] getExecTestArgs(String exec, Path output, Path errorOutput) {
-		if(isWindows())
+		if ( isWindows() ) {
-			return new String[] { exec, new File(tmpdir, "input").getAbsolutePath() } ;
+			return new String[]{
+				exec, new File(tmpdir, "input").getAbsolutePath(),
+				output.toAbsolutePath().toString(),
+				errorOutput.toAbsolutePath().toString()
+			};
+		}
 		else {
-			String mono = locateTool("mono");
+			if (!NETSTANDARD) {
-			return new String[] { mono, exec, new File(tmpdir, "input").getAbsolutePath() };
+				String mono = locateTool("mono");
+				return new String[] {
+					mono, exec, new File(tmpdir, "input").getAbsolutePath(),
+					output.toAbsolutePath().toString(),
+					errorOutput.toAbsolutePath().toString()
+				};
+			}
+
+			String dotnet = locateTool("dotnet");
+			return new String[] {
+				dotnet, exec, new File(tmpdir, "src/input").getAbsolutePath(),
+				output.toAbsolutePath().toString(),
+				errorOutput.toAbsolutePath().toString()
+			};
 		}
 	}
 
@@ -534,41 +664,6 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
 	}
 
 
-	public static class StreamVacuum implements Runnable {
-		StringBuilder buf = new StringBuilder();
-		BufferedReader in;
-		Thread sucker;
-		public StreamVacuum(InputStream in) {
-			this.in = new BufferedReader( new InputStreamReader(in) );
-		}
-		public void start() {
-			sucker = new Thread(this);
-			sucker.start();
-		}
-		@Override
-		public void run() {
-			try {
-				String line = in.readLine();
-				while (line!=null) {
-					buf.append(line);
-					buf.append('\n');
-					line = in.readLine();
-				}
-			}
-			catch (IOException ioe) {
-				System.err.println("can't read output from process");
-			}
-		}
-		/** wait for the thread to finish */
-		public void join() throws InterruptedException {
-			sucker.join();
-		}
-		@Override
-		public String toString() {
-			return buf.toString();
-		}
-	}
-
 	protected void checkGrammarSemanticsError(ErrorQueue equeue,
 	                                          GrammarSemanticsMessage expectedMessage)
 		throws Exception
@@ -611,16 +706,6 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
 		}
 	}
 
-	public static void writeFile(String dir, String fileName, String content) {
-		try {
-			Utils.writeFile(dir+"/"+fileName, content, "UTF-8");
-		}
-		catch (IOException ioe) {
-			System.err.println("can't write file");
-			ioe.printStackTrace(System.err);
-		}
-	}
-
 	protected void mkdir(String dir) {
 		File f = new File(dir);
 		f.mkdirs();
@@ -635,16 +720,23 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
 			"using System;\n" +
 				"using Antlr4.Runtime;\n" +
 				"using Antlr4.Runtime.Tree;\n" +
+				"using System.IO;\n" +
+				"using System.Text;\n" +
 				"\n" +
 				"public class Test {\n" +
 				"    public static void Main(string[] args) {\n" +
-				"        ICharStream input = new AntlrFileStream(args[0]);\n" +
+				"        string inputData = File.ReadAllText(args[0], Encoding.UTF8);\n" +
-				"        <lexerName> lex = new <lexerName>(input);\n" +
+				"        using (TextWriter output = new StreamWriter(args[1]),\n" +
-				"        CommonTokenStream tokens = new CommonTokenStream(lex);\n" +
+				"                          errorOutput = new StreamWriter(args[2])) {\n" +
-				"        <createParser>\n"+
+				"                CodePointCharStream input = new CodePointCharStream(inputData);\n" +
-				"		 parser.BuildParseTree = true;\n" +
+				"                input.name = args[0];\n" +
-				"        ParserRuleContext tree = parser.<parserStartRuleName>();\n" +
+				"                <lexerName> lex = new <lexerName>(input, output, errorOutput);\n" +
-				"        ParseTreeWalker.Default.Walk(new TreeShapeListener(), tree);\n" +
+				"                CommonTokenStream tokens = new CommonTokenStream(lex);\n" +
+				"                <createParser>\n"+
+				"			 parser.BuildParseTree = true;\n" +
+				"                ParserRuleContext tree = parser.<parserStartRuleName>();\n" +
+				"                ParseTreeWalker.Default.Walk(new TreeShapeListener(), tree);\n" +
+				"        }\n" +
 				"    }\n" +
 				"}\n" +
 				"\n" +
@@ -663,11 +755,11 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
 				"	}\n" +
 				"}"
 		);
-		ST createParserST = new ST("        <parserName> parser = new <parserName>(tokens);\n");
+		ST createParserST = new ST("        <parserName> parser = new <parserName>(tokens, output, errorOutput);\n");
 		if ( debug ) {
 			createParserST =
 				new ST(
-					"        <parserName> parser = new <parserName>(tokens);\n" +
+					"        <parserName> parser = new <parserName>(tokens, output, errorOutput);\n" +
 						"        parser.AddErrorListener(new DiagnosticErrorListener());\n");
 		}
 		outputFileST.add("createParser", createParserST);
@@ -681,17 +773,23 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
 		ST outputFileST = new ST(
 			"using System;\n" +
 				"using Antlr4.Runtime;\n" +
+				"using System.IO;\n" +
+				"using System.Text;\n" +
 				"\n" +
 				"public class Test {\n" +
 				"    public static void Main(string[] args) {\n" +
-				"        ICharStream input = new AntlrFileStream(args[0]);\n" +
+				"        string inputData = File.ReadAllText(args[0], Encoding.UTF8);\n" +
-				"        <lexerName> lex = new <lexerName>(input);\n" +
+				"        ICharStream input = new CodePointCharStream(inputData);\n" +
-				"        CommonTokenStream tokens = new CommonTokenStream(lex);\n" +
+				"        using (TextWriter output = new StreamWriter(args[1]),\n" +
-				"        tokens.Fill();\n" +
+				"                          errorOutput = new StreamWriter(args[2])) {\n" +
-				"        foreach (object t in tokens.GetTokens())\n" +
+					"        <lexerName> lex = new <lexerName>(input, output, errorOutput);\n" +
-				"			Console.WriteLine(t);\n" +
+					"        CommonTokenStream tokens = new CommonTokenStream(lex);\n" +
-				(showDFA?"        Console.Write(lex.Interpreter.GetDFA(Lexer.DEFAULT_MODE).ToLexerString());\n":"")+
+					"        tokens.Fill();\n" +
-				"    }\n" +
+					"        foreach (object t in tokens.GetTokens())\n" +
+					"			output.WriteLine(t);\n" +
+					(showDFA?"        output.Write(lex.Interpreter.GetDFA(Lexer.DEFAULT_MODE).ToLexerString());\n":"")+
+					"    }\n" +
+					"}\n" +
 				"}"
 		);
 

runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/TestVisitors.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-package org.antlr.v4.test.runtime.csharp;
-
-import org.antlr.v4.test.runtime.BaseRuntimeTest;
-import org.antlr.v4.test.runtime.RuntimeTestDescriptor;
-import org.antlr.v4.test.runtime.descriptors.VisitorsDescriptors;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class TestVisitors extends BaseRuntimeTest {
-	public TestVisitors(RuntimeTestDescriptor descriptor) {
-		super(descriptor,new BaseCSharpTest());
-	}
-
-	@Parameterized.Parameters(name="{0}")
-	public static RuntimeTestDescriptor[] getAllTestDescriptors() {
-		return BaseRuntimeTest.getRuntimeTestDescriptors(VisitorsDescriptors.class, "CSharp");
-	}
-}

runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/global.json

@@ -0,0 +1,6 @@
+{
+  "projects": [
+    "replace_this"
+  ]
+}
+

runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/project.json

@@ -0,0 +1,27 @@
+{
+  "version": "1.0.0-*",
+  "frameworks": {
+    "netcoreapp1.0": {
+      "imports": [
+        "dnxcore50"
+      ]
+    }
+  },
+  "dependencies": {
+    "Microsoft.NETCore.App": {
+      "type": "platform",
+      "version": "1.1.0"
+    },
+    "CSharp": {
+      "target": "Project",
+      "version": "*"
+    }
+  },
+  "buildOptions": {
+    "emitEntryPoint": true,
+    "outputName": "Test",
+    "nowarn": [
+      "CS3021"
+    ]
+  }
+}

runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/ParseTreesDescriptors.java

@@ -112,7 +112,7 @@ public class ParseTreesDescriptors {
 
 		@Override
 		public boolean ignore(String targetName) {
-			return !targetName.matches("Java|Python2|Python3|Node|Swift");
+			return !targetName.matches("Java|Python2|Python3|Node|Swift|CSharp");
 		}
 	}
 

runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/ParserErrorsDescriptors.java

@@ -135,7 +135,7 @@ public class ParserErrorsDescriptors {
 	public static class InvalidEmptyInput extends BaseParserTestDescriptor {
 		public String input = "";
 		public String output = null;
-		public String errors = "line 1:0 missing ID at '<EOF>'\n";
+		public String errors = "line 1:0 mismatched input '<EOF>' expecting ID\n";
 		public String startRule = "start";
 		public String grammarName = "T";
 
@@ -414,7 +414,7 @@ public class ParserErrorsDescriptors {
 
 		/**
 		 grammar T;
-		 a : 'a' 'b'* ;
+		 a : 'a' 'b'* EOF ;
 		 */
 		@CommentHasStringValue
 		public String grammar;
@@ -436,7 +436,7 @@ public class ParserErrorsDescriptors {
 
 		/**
 		 grammar T;
-		 a : 'a' ('b'|'z'{<Pass()>})*;
+		 a : 'a' ('b'|'z'{<Pass()>})* EOF ;
 		 */
 		@CommentHasStringValue
 		public String grammar;

runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/ParserExecDescriptors.java

@@ -753,6 +753,83 @@ public class ParserExecDescriptors {
 		 */
 		@CommentHasStringValue
 		public String grammar;
+	}
 
+	/**
+	 * This is a regression test for antlr/antlr4#1545, case 1.
+	 */
+	public static class OpenDeviceStatement_Case1 extends BaseParserTestDescriptor {
+		public String input = "OPEN DEVICE DEVICE";
+		public String output = "OPEN DEVICE DEVICE\n";
+		public String errors = null;
+		public String startRule = "statement";
+		public String grammarName = "OpenDeviceStatement";
+
+		/**
+		 grammar OpenDeviceStatement;
+		 program : statement+ '.' ;
+
+		 statement : 'OPEN' ( 'DEVICE' (  OPT1  |  OPT2  |  OPT3  )? )+ {<writeln("$text")>} ;
+
+		 OPT1 : 'OPT-1';
+		 OPT2 : 'OPT-2';
+		 OPT3 : 'OPT-3';
+
+		 WS : (' '|'\n')+ -> channel(HIDDEN);
+		 */
+		@CommentHasStringValue
+		public String grammar;
+	}
+
+	/**
+	 * This is a regression test for antlr/antlr4#1545, case 2.
+	 */
+	public static class OpenDeviceStatement_Case2 extends BaseParserTestDescriptor {
+		public String input = "OPEN DEVICE DEVICE";
+		public String output = "OPEN DEVICE DEVICE\n";
+		public String errors = null;
+		public String startRule = "statement";
+		public String grammarName = "OpenDeviceStatement";
+
+		/**
+		 grammar OpenDeviceStatement;
+		 program : statement+ '.' ;
+
+		 statement : 'OPEN' ( 'DEVICE' (  (OPT1)  |  OPT2  |  OPT3  )? )+ {<writeln("$text")>} ;
+
+		 OPT1 : 'OPT-1';
+		 OPT2 : 'OPT-2';
+		 OPT3 : 'OPT-3';
+
+		 WS : (' '|'\n')+ -> channel(HIDDEN);
+		 */
+		@CommentHasStringValue
+		public String grammar;
+	}
+
+	/**
+	 * This is a regression test for antlr/antlr4#1545, case 3.
+	 */
+	public static class OpenDeviceStatement_Case3 extends BaseParserTestDescriptor {
+		public String input = "OPEN DEVICE DEVICE.";
+		public String output = "OPEN DEVICE DEVICE\n";
+		public String errors = null;
+		public String startRule = "statement";
+		public String grammarName = "OpenDeviceStatement";
+
+		/**
+		 grammar OpenDeviceStatement;
+		 program : statement+ '.' ;
+
+		 statement : 'OPEN' ( 'DEVICE' (  (OPT1)  |  OPT2  |  OPT3  )? )+ {<writeln("$text")>} ;
+
+		 OPT1 : 'OPT-1';
+		 OPT2 : 'OPT-2';
+		 OPT3 : 'OPT-3';
+
+		 WS : (' '|'\n')+ -> channel(HIDDEN);
+		 */
+		@CommentHasStringValue
+		public String grammar;
 	}
 }

runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/PerformanceDescriptors.java

@@ -113,7 +113,7 @@ public class PerformanceDescriptors {
 
 		@Override
 		public boolean ignore(String targetName) {
-			return !Arrays.asList("Java", "CSharp", "Python2", "Python3", "Node").contains(targetName);
+			return !Arrays.asList("Java", "CSharp", "Python2", "Python3", "Node", "Cpp").contains(targetName);
 		}
 	}
 
@@ -199,7 +199,7 @@ public class PerformanceDescriptors {
 		@Override
 		public boolean ignore(String targetName) {
 			// passes, but still too slow in Python and JavaScript
-			return !Arrays.asList("Java", "CSharp").contains(targetName);
+			return !Arrays.asList("Java", "CSharp", "Cpp").contains(targetName);
 		}
 	}
 

runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/VisitorsDescriptors.java

@@ -1,273 +0,0 @@
-/*
- * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-package org.antlr.v4.test.runtime.descriptors;
-
-import org.antlr.v4.test.runtime.BaseParserTestDescriptor;
-import org.antlr.v4.test.runtime.CommentHasStringValue;
-
-import java.util.Arrays;
-
-import static org.antlr.v4.test.runtime.BaseRuntimeTest.JavaScriptTargets;
-
-public class VisitorsDescriptors {
-	public static boolean isJavaScriptTarget(String targetName) {
-		boolean isJavaScriptTarget = Arrays.binarySearch(JavaScriptTargets, targetName)>=0;
-		return isJavaScriptTarget;
-	}
-
-	public static class Basic extends BaseParserTestDescriptor {
-		public String input = "1 2";
-		/**
-		(a 1 2)
-		[ '1', '2' ]
-		 */
-		@CommentHasStringValue
-		public String output;
-
-		public String errors = null;
-		public String startRule = "s";
-		public String grammarName = "T";
-
-		/**
-		 grammar T;
-		 @parser::header {
-		 <ImportVisitor(grammarName)>
-		 }
-
-		 @parser::members {
-		 <BasicVisitor(grammarName)>
-		 }
-
-		 s
-		 @after {
-		 <ToStringTree("$ctx.r"):writeln()>
-		 <WalkVisitor("$ctx.r")>
-		 }
-		   : r=a ;
-		 a : INT INT
-		   | ID
-		   ;
-		 MULT: '*' ;
-		 ADD : '+' ;
-		 INT : [0-9]+ ;
-		 ID  : [a-z]+ ;
-		 WS : [ \t\n]+ -> skip ;
-		 */
-		@CommentHasStringValue
-		public String grammar;
-
-		@Override
-		public boolean ignore(String targetName) { return !isJavaScriptTarget(targetName); }
-	}
-
-	public static class LR extends BaseParserTestDescriptor {
-		public String input = "1+2*3";
-		/**
-		(e (e 1) + (e (e 2) * (e 3)))
-		1,,2,,32 3 21 2 1
-		 */
-		@CommentHasStringValue
-		public String output;
-
-		public String errors = null;
-		public String startRule = "s";
-		public String grammarName = "T";
-
-		/**
-		 grammar T;
-		 @parser::header {
-		 <ImportVisitor(grammarName)>
-		 }
-
-		 @parser::members {
-		 <LRVisitor(grammarName)>
-		 }
-
-		 s
-		 @after {
-		 <ToStringTree("$ctx.r"):writeln()>
-		 <WalkVisitor("$ctx.r")>
-		 }
-		 	: r=e ;
-		 e : e op='*' e
-		 	| e op='+' e
-		 	| INT
-		 	;
-		 MULT: '*' ;
-		 ADD : '+' ;
-		 INT : [0-9]+ ;
-		 ID  : [a-z]+ ;
-		 WS : [ \t\n]+ -> skip ;
-		 */
-		@CommentHasStringValue
-		public String grammar;
-
-		@Override
-		public boolean ignore(String targetName) { return !isJavaScriptTarget(targetName); }
-	}
-
-	public static class LRWithLabels extends BaseParserTestDescriptor {
-		public String input = "1(2,3)";
-		/**
-		(e (e 1) ( (eList (e 2) , (e 3)) ))
-		1,,2,,3,1 [13 6]
-		 */
-		@CommentHasStringValue
-		public String output;
-
-		public String errors = null;
-		public String startRule = "s";
-		public String grammarName = "T";
-
-		/**
-		 grammar T;
-		 @parser::header {
-		 <ImportVisitor(grammarName)>
-		 }
-
-		 @parser::members {
-		 <LRWithLabelsVisitor(grammarName)>
-		 }
-
-		 s
-		 @after {
-		 <ToStringTree("$ctx.r"):writeln()>
-		 <WalkVisitor("$ctx.r")>
-		 }
-		   : r=e ;
-		 e : e '(' eList ')' # Call
-		   | INT             # Int
-		   ;
-		 eList : e (',' e)* ;
-		 MULT: '*' ;
-		 ADD : '+' ;
-		 INT : [0-9]+ ;
-		 ID  : [a-z]+ ;
-		 WS : [ \t\n]+ -> skip ;
-		 */
-		@CommentHasStringValue
-		public String grammar;
-
-		@Override
-		public boolean ignore(String targetName) { return !isJavaScriptTarget(targetName); }
-	}
-
-	public static abstract class RuleGetters extends BaseParserTestDescriptor {
-		public String errors = null;
-		public String startRule = "s";
-		public String grammarName = "T";
-
-		/**
-		 grammar T;
-		 @parser::header {
-		 <ImportVisitor(grammarName)>
-		 }
-
-		 @parser::members {
-		 <RuleGetterVisitor(grammarName)>
-		 }
-
-		 s
-		 @after {
-		 <ToStringTree("$ctx.r"):writeln()>
-		 <WalkVisitor("$ctx.r")>
-		 }
-		   : r=a ;
-		 a : b b		// forces list
-		   | b		// a list still
-		   ;
-		 b : ID | INT;
-		 MULT: '*' ;
-		 ADD : '+' ;
-		 INT : [0-9]+ ;
-		 ID  : [a-z]+ ;
-		 WS : [ \t\n]+ -> skip ;
-		 */
-		@CommentHasStringValue
-		public String grammar;
-
-		@Override
-		public boolean ignore(String targetName) { return !isJavaScriptTarget(targetName); }
-	}
-
-	public static class RuleGetters_1 extends RuleGetters {
-		public String input = "1 2";
-		/**
-		(a (b 1) (b 2))
-		,1 2 1
-		 */
-		@CommentHasStringValue
-		public String output;
-	}
-
-	public static class RuleGetters_2 extends RuleGetters {
-		public String input = "abc";
-		/**
-		(a (b abc))
-		abc
-		 */
-		@CommentHasStringValue
-		public String output;
-	}
-
-	public static abstract class TokenGetters extends BaseParserTestDescriptor {
-		public String errors = null;
-		public String startRule = "s";
-		public String grammarName = "T";
-
-		/**
-		 grammar T;
-		 @parser::header {
-		 <ImportVisitor(grammarName)>
-		 }
-
-		 @parser::members {
-		 <TokenGetterVisitor(grammarName)>
-		 }
-
-		 s
-		 @after {
-		 <ToStringTree("$ctx.r"):writeln()>
-		 <WalkVisitor("$ctx.r")>
-		 }
-		   : r=a ;
-		 a : INT INT
-		   | ID
-		   ;
-		 MULT: '*' ;
-		 ADD : '+' ;
-		 INT : [0-9]+ ;
-		 ID  : [a-z]+ ;
-		 WS : [ \t\n]+ -> skip ;
-		 */
-		@CommentHasStringValue
-		public String grammar;
-
-		@Override
-		public boolean ignore(String targetName) { return !isJavaScriptTarget(targetName); }
-	}
-
-	public static class TokenGetters_1 extends TokenGetters {
-		public String input = "1 2";
-		/**
-		(a 1 2)
-		,1 2 [1, 2]
-		 */
-		@CommentHasStringValue
-		public String output;
-	}
-
-	public static class TokenGetters_2 extends TokenGetters {
-		public String input = "abc";
-		/**
-		(a abc)
-		[@0,0:2='abc',<4>,1:0]
-		 */
-		@CommentHasStringValue
-		public String output;
-	}
-}

runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java

@@ -33,6 +33,7 @@ import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.semantics.SemanticPipeline;
 import org.antlr.v4.test.runtime.ErrorQueue;
 import org.antlr.v4.test.runtime.RuntimeTestSupport;
+import org.antlr.v4.test.runtime.StreamVacuum;
 import org.antlr.v4.tool.ANTLRMessage;
 import org.antlr.v4.tool.DOTGenerator;
 import org.antlr.v4.tool.Grammar;
@@ -43,8 +44,6 @@ import org.stringtemplate.v4.ST;
 import org.stringtemplate.v4.STGroup;
 import org.stringtemplate.v4.STGroupString;
 
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileFilter;
 import java.io.FileInputStream;
@@ -52,7 +51,6 @@ import java.io.FileOutputStream;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.net.URL;
 import java.util.ArrayList;
@@ -70,6 +68,7 @@ import static junit.framework.TestCase.assertFalse;
 import static junit.framework.TestCase.assertNotNull;
 import static junit.framework.TestCase.assertTrue;
 import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
+import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
 import static org.junit.Assert.assertArrayEquals;
 
 public class BaseGoTest implements RuntimeTestSupport {
@@ -309,7 +308,7 @@ public class BaseGoTest implements RuntimeTestSupport {
 		boolean success = rawGenerateAndBuildRecognizer(grammarFileName,
 		                                                grammarStr, null, lexerName, "-no-listener");
 		assertTrue(success);
-		writeFile(overall_tmpdir, "input", input);
+		writeFile(overall_tmpdir.toString(), "input", input);
 		writeLexerTestFile(lexerName, showDFA);
 		String output = execModule("Test.go");
 		return output;
@@ -338,7 +337,7 @@ public class BaseGoTest implements RuntimeTestSupport {
 		boolean success = rawGenerateAndBuildRecognizer(grammarFileName,
 		                                                grammarStr, parserName, lexerName, "-visitor");
 		assertTrue(success);
-		writeFile(overall_tmpdir, "input", input);
+		writeFile(overall_tmpdir.toString(), "input", input);
 		rawBuildRecognizerTestFile(parserName, lexerName, listenerName,
 		                           visitorName, startRuleName, showDiagnosticErrors);
 		return execRecognizer();
@@ -571,45 +570,6 @@ public class BaseGoTest implements RuntimeTestSupport {
 		}
 	}
 
-	public static class StreamVacuum implements Runnable {
-		StringBuilder buf = new StringBuilder();
-		BufferedReader in;
-		Thread sucker;
-
-		public StreamVacuum(InputStream in) {
-			this.in = new BufferedReader(new InputStreamReader(in));
-		}
-
-		public void start() {
-			sucker = new Thread(this);
-			sucker.start();
-		}
-
-		@Override
-		public void run() {
-			try {
-				String line = in.readLine();
-				while (line != null) {
-					buf.append(line);
-					buf.append('\n');
-					line = in.readLine();
-				}
-			} catch (IOException ioe) {
-				System.err.println("can't read output from process");
-			}
-		}
-
-		/** wait for the thread to finish */
-		public void join() throws InterruptedException {
-			sucker.join();
-		}
-
-		@Override
-		public String toString() {
-			return buf.toString();
-		}
-	}
-
 	protected void checkGrammarSemanticsError(ErrorQueue equeue,
 	                                          GrammarSemanticsMessage expectedMessage) throws Exception {
 		ANTLRMessage foundMsg = null;
@@ -700,35 +660,6 @@ public class BaseGoTest implements RuntimeTestSupport {
 		}
 	}
 
-	public static void writeFile(File dir, String fileName, String content) {
-		try {
-			File f = new File(dir, fileName);
-			FileWriter w = new FileWriter(f);
-			BufferedWriter bw = new BufferedWriter(w);
-			bw.write(content);
-			bw.close();
-			w.close();
-		} catch (IOException ioe) {
-			System.err.println("can't write file");
-			ioe.printStackTrace(System.err);
-		}
-	}
-
-	public static void writeFile(String dir, String fileName, InputStream content) {
-		try {
-			File f = new File(dir, fileName);
-			OutputStream output = new FileOutputStream(f);
-			while(content.available()>0) {
-				int b = content.read();
-				output.write(b);
-			}
-			output.close();
-		} catch (IOException ioe) {
-			System.err.println("can't write file");
-			ioe.printStackTrace(System.err);
-		}
-	}
-
 	protected void mkdir(File dir) {
 		dir.mkdirs();
 	}
@@ -785,7 +716,7 @@ public class BaseGoTest implements RuntimeTestSupport {
 		outputFileST.add("listenerName", listenerName);
 		outputFileST.add("visitorName", visitorName);
 		outputFileST.add("parserStartRuleName", parserStartRuleName.substring(0, 1).toUpperCase() + parserStartRuleName.substring(1) );
-		writeFile(overall_tmpdir, "Test.go", outputFileST.render());
+		writeFile(overall_tmpdir.toString(), "Test.go", outputFileST.render());
 	}
 
 
@@ -813,7 +744,7 @@ public class BaseGoTest implements RuntimeTestSupport {
 				+ "}\n"
 				+ "\n");
 		outputFileST.add("lexerName", lexerName);
-		writeFile(overall_tmpdir, "Test.go", outputFileST.render());
+		writeFile(overall_tmpdir.toString(), "Test.go", outputFileST.render());
 	}
 
 	public void writeRecognizer(String parserName, String lexerName,

runtime-testsuite/test/org/antlr/v4/test/runtime/go/TestVisitors.java

@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-package org.antlr.v4.test.runtime.go;
-
-import org.antlr.v4.test.runtime.BaseRuntimeTest;
-import org.antlr.v4.test.runtime.RuntimeTestDescriptor;
-import org.antlr.v4.test.runtime.descriptors.VisitorsDescriptors;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class TestVisitors extends BaseRuntimeTest {
-	public TestVisitors(RuntimeTestDescriptor descriptor) {
-		super(descriptor,new BaseGoTest());
-	}
-
-	@BeforeClass
-	public static void groupSetUp() throws Exception { BaseGoTest.groupSetUp(); }
-
-	@AfterClass
-	public static void groupTearDown() throws Exception { BaseGoTest.groupTearDown(); }
-
-	@Parameterized.Parameters(name="{0}")
-	public static RuntimeTestDescriptor[] getAllTestDescriptors() {
-		return BaseRuntimeTest.getRuntimeTestDescriptors(VisitorsDescriptors.class, "Go");
-	}
-}

runtime-testsuite/test/org/antlr/v4/test/runtime/java/BaseJavaTest.java

@@ -40,6 +40,7 @@ import org.antlr.v4.semantics.SemanticPipeline;
 import org.antlr.v4.test.runtime.BaseRuntimeTest;
 import org.antlr.v4.test.runtime.ErrorQueue;
 import org.antlr.v4.test.runtime.RuntimeTestSupport;
+import org.antlr.v4.test.runtime.StreamVacuum;
 import org.antlr.v4.tool.ANTLRMessage;
 import org.antlr.v4.tool.Grammar;
 import org.antlr.v4.tool.GrammarSemanticsMessage;
@@ -53,7 +54,6 @@ import javax.tools.JavaCompiler;
 import javax.tools.JavaFileObject;
 import javax.tools.StandardJavaFileManager;
 import javax.tools.ToolProvider;
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -81,6 +81,7 @@ import static junit.framework.TestCase.assertEquals;
 import static junit.framework.TestCase.assertFalse;
 import static junit.framework.TestCase.assertNotNull;
 import static junit.framework.TestCase.assertTrue;
+import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
 import static org.junit.Assert.assertArrayEquals;
 
 public class BaseJavaTest implements RuntimeTestSupport {
@@ -702,6 +703,7 @@ public class BaseJavaTest implements RuntimeTestSupport {
 		try {
 			String[] args = new String[] {
 				"java", "-classpath", tmpdir+pathSep+CLASSPATH,
+				"-Dfile.encoding=UTF-8",
 				className, new File(tmpdir, "input").getAbsolutePath()
 			};
 //			String cmdLine = Utils.join(args, " ");
@@ -823,41 +825,6 @@ public class BaseJavaTest implements RuntimeTestSupport {
 		}
 	}
 
-	public static class StreamVacuum implements Runnable {
-		StringBuilder buf = new StringBuilder();
-		BufferedReader in;
-		Thread sucker;
-		public StreamVacuum(InputStream in) {
-			this.in = new BufferedReader( new InputStreamReader(in) );
-		}
-		public void start() {
-			sucker = new Thread(this);
-			sucker.start();
-		}
-		@Override
-		public void run() {
-			try {
-				String line = in.readLine();
-				while (line!=null) {
-					buf.append(line);
-					buf.append('\n');
-					line = in.readLine();
-				}
-			}
-			catch (IOException ioe) {
-				System.err.println("can't read output from process");
-			}
-		}
-		/** wait for the thread to finish */
-		public void join() throws InterruptedException {
-			sucker.join();
-		}
-		@Override
-		public String toString() {
-			return buf.toString();
-		}
-	}
-
 	protected void checkGrammarSemanticsError(ErrorQueue equeue,
 											  GrammarSemanticsMessage expectedMessage)
 		throws Exception
@@ -941,16 +908,6 @@ public class BaseJavaTest implements RuntimeTestSupport {
         }
     }
 
-	public static void writeFile(String dir, String fileName, String content) {
-		try {
-			Utils.writeFile(dir+"/"+fileName, content, "UTF-8");
-		}
-		catch (IOException ioe) {
-			System.err.println("can't write file");
-			ioe.printStackTrace(System.err);
-		}
-	}
-
 	protected void writeTestFile(String parserName,
 								 String lexerName,
 								 String parserStartRuleName,
@@ -961,11 +918,12 @@ public class BaseJavaTest implements RuntimeTestSupport {
 			"import org.antlr.v4.runtime.*;\n" +
 			"import org.antlr.v4.runtime.tree.*;\n" +
 			"import org.antlr.v4.runtime.atn.*;\n" +
+			"import java.nio.file.Paths;\n"+
 			"import java.util.Arrays;\n"+
 			"\n" +
 			"public class Test {\n" +
 			"    public static void main(String[] args) throws Exception {\n" +
-			"        CharStream input = new ANTLRFileStream(args[0]);\n" +
+			"        CharStream input = CharStreams.createWithUTF8(Paths.get(args[0]));\n" +
 			"        <lexerName> lex = new <lexerName>(input);\n" +
 			"        CommonTokenStream tokens = new CommonTokenStream(lex);\n" +
 			"        <createParser>\n"+
@@ -1017,11 +975,12 @@ public class BaseJavaTest implements RuntimeTestSupport {
 
 	protected void writeLexerTestFile(String lexerName, boolean showDFA) {
 		ST outputFileST = new ST(
+			"import java.nio.file.Paths;\n" +
 			"import org.antlr.v4.runtime.*;\n" +
 			"\n" +
 			"public class Test {\n" +
 			"    public static void main(String[] args) throws Exception {\n" +
-			"        CharStream input = new ANTLRFileStream(args[0]);\n" +
+			"        CharStream input = CharStreams.createWithUTF8(Paths.get(args[0]));\n" +
 			"        <lexerName> lex = new <lexerName>(input);\n" +
 			"        CommonTokenStream tokens = new CommonTokenStream(lex);\n" +
 			"        tokens.fill();\n" +

runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestCharStreams.java

@@ -0,0 +1,172 @@
+/*
+ * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+package org.antlr.v4.test.runtime.java;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+
+import java.nio.channels.SeekableByteChannel;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import java.util.Arrays;
+
+import org.antlr.v4.runtime.CharStreams;
+import org.antlr.v4.runtime.CodePointCharStream;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+
+public class TestCharStreams {
+	@Rule
+	public TemporaryFolder folder = new TemporaryFolder();
+
+	@Rule
+	public ExpectedException thrown = ExpectedException.none();
+
+	@Test
+	public void createWithBMPStringHasExpectedSize() {
+		CodePointCharStream s = CharStreams.createWithString("hello");
+		assertEquals(5, s.size());
+		assertEquals(0, s.index());
+		assertEquals("hello", s.toString());
+	}
+
+	@Test
+	public void createWithSMPStringHasExpectedSize() {
+		CodePointCharStream s = CharStreams.createWithString(
+				"hello \uD83C\uDF0E");
+		assertEquals(7, s.size());
+		assertEquals(0, s.index());
+		assertEquals("hello \uD83C\uDF0E", s.toString());
+	}
+
+	@Test
+	public void createWithBMPUTF8PathHasExpectedSize() throws Exception {
+		Path p = folder.newFile().toPath();
+		Files.write(p, "hello".getBytes(StandardCharsets.UTF_8));
+		CodePointCharStream s = CharStreams.createWithUTF8(p);
+		assertEquals(5, s.size());
+		assertEquals(0, s.index());
+		assertEquals("hello", s.toString());
+		assertEquals(p.toString(), s.getSourceName());
+	}
+
+	@Test
+	public void createWithSMPUTF8PathHasExpectedSize() throws Exception {
+		Path p = folder.newFile().toPath();
+		Files.write(p, "hello \uD83C\uDF0E".getBytes(StandardCharsets.UTF_8));
+		CodePointCharStream s = CharStreams.createWithUTF8(p);
+		assertEquals(7, s.size());
+		assertEquals(0, s.index());
+		assertEquals("hello \uD83C\uDF0E", s.toString());
+		assertEquals(p.toString(), s.getSourceName());
+	}
+
+	@Test
+	public void createWithBMPUTF8InputStreamHasExpectedSize() throws Exception {
+		Path p = folder.newFile().toPath();
+		Files.write(p, "hello".getBytes(StandardCharsets.UTF_8));
+		try (InputStream is = Files.newInputStream(p)) {
+			CodePointCharStream s = CharStreams.createWithUTF8Stream(is);
+			assertEquals(5, s.size());
+			assertEquals(0, s.index());
+			assertEquals("hello", s.toString());
+		}
+	}
+
+	@Test
+	public void createWithSMPUTF8InputStreamHasExpectedSize() throws Exception {
+		Path p = folder.newFile().toPath();
+		Files.write(p, "hello \uD83C\uDF0E".getBytes(StandardCharsets.UTF_8));
+		try (InputStream is = Files.newInputStream(p)) {
+			CodePointCharStream s = CharStreams.createWithUTF8Stream(is);
+			assertEquals(7, s.size());
+			assertEquals(0, s.index());
+			assertEquals("hello \uD83C\uDF0E", s.toString());
+		}
+	}
+
+	@Test
+	public void createWithBMPUTF8ChannelHasExpectedSize() throws Exception {
+		Path p = folder.newFile().toPath();
+		Files.write(p, "hello".getBytes(StandardCharsets.UTF_8));
+		try (SeekableByteChannel c = Files.newByteChannel(p)) {
+			CodePointCharStream s = CharStreams.createWithUTF8Channel(
+					c, 4096, CodingErrorAction.REPLACE, "foo");
+			assertEquals(5, s.size());
+			assertEquals(0, s.index());
+			assertEquals("hello", s.toString());
+			assertEquals("foo", s.getSourceName());
+		}
+	}
+
+	@Test
+	public void createWithSMPUTF8ChannelHasExpectedSize() throws Exception {
+		Path p = folder.newFile().toPath();
+		Files.write(p, "hello \uD83C\uDF0E".getBytes(StandardCharsets.UTF_8));
+		try (SeekableByteChannel c = Files.newByteChannel(p)) {
+			CodePointCharStream s = CharStreams.createWithUTF8Channel(
+					c, 4096, CodingErrorAction.REPLACE, "foo");
+			assertEquals(7, s.size());
+			assertEquals(0, s.index());
+			assertEquals("hello \uD83C\uDF0E", s.toString());
+			assertEquals("foo", s.getSourceName());
+		}
+	}
+
+	@Test
+	public void createWithInvalidUTF8BytesChannelReplacesWithSubstCharInReplaceMode()
+		throws Exception {
+		Path p = folder.newFile().toPath();
+		byte[] toWrite = new byte[] { (byte)0xCA, (byte)0xFE, (byte)0xFE, (byte)0xED };
+		Files.write(p, toWrite);
+		try (SeekableByteChannel c = Files.newByteChannel(p)) {
+			CodePointCharStream s = CharStreams.createWithUTF8Channel(
+					c, 4096, CodingErrorAction.REPLACE, "foo");
+			assertEquals(3, s.size());
+			assertEquals(0, s.index());
+			assertEquals("\uFFFD\uFFFD\uFFFD", s.toString());
+		}
+	}
+
+	@Test
+	public void createWithInvalidUTF8BytesThrowsInReportMode() throws Exception {
+		Path p = folder.newFile().toPath();
+		byte[] toWrite = new byte[] { (byte)0xCA, (byte)0xFE };
+		Files.write(p, toWrite);
+		try (SeekableByteChannel c = Files.newByteChannel(p)) {
+			thrown.expect(CharacterCodingException.class);
+			CharStreams.createWithUTF8Channel(c, 4096, CodingErrorAction.REPORT, "foo");
+		}
+	}
+
+	@Test
+	public void createWithSMPUTF8SequenceStraddlingBufferBoundary() throws Exception {
+		Path p = folder.newFile().toPath();
+		Files.write(p, "hello \uD83C\uDF0E".getBytes(StandardCharsets.UTF_8));
+		try (SeekableByteChannel c = Files.newByteChannel(p)) {
+			CodePointCharStream s = CharStreams.createWithUTF8Channel(
+					c,
+					// Note this buffer size ensures the SMP code point
+					// straddles the boundary of two buffers
+					8,
+					CodingErrorAction.REPLACE,
+					"foo");
+			assertEquals(7, s.size());
+			assertEquals(0, s.index());
+			assertEquals("hello \uD83C\uDF0E", s.toString());
+		}
+	}
+}

runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestCodePointCharStream.java

@@ -0,0 +1,294 @@
+/*
+ * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+package org.antlr.v4.test.runtime.java;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.nio.IntBuffer;
+
+import org.antlr.v4.runtime.CharStreams;
+import org.antlr.v4.runtime.CodePointCharStream;
+import org.antlr.v4.runtime.IntStream;
+
+import org.antlr.v4.runtime.misc.Interval;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class TestCodePointCharStream {
+	@Rule
+	public ExpectedException thrown = ExpectedException.none();
+
+	@Test
+	public void emptyBytesHasSize0() {
+		CodePointCharStream s = CharStreams.createWithString("");
+		assertEquals(0, s.size());
+		assertEquals(0, s.index());
+	}
+
+	@Test
+	public void emptyBytesLookAheadReturnsEOF() {
+		CodePointCharStream s = CharStreams.createWithString("");
+		assertEquals(IntStream.EOF, s.LA(1));
+		assertEquals(0, s.index());
+	}
+
+	@Test
+	public void consumingEmptyStreamShouldThrow() {
+		CodePointCharStream s = CharStreams.createWithString("");
+		thrown.expect(IllegalStateException.class);
+		thrown.expectMessage("cannot consume EOF");
+		s.consume();
+	}
+
+	@Test
+	public void singleLatinCodePointHasSize1() {
+		CodePointCharStream s = CharStreams.createWithString("X");
+		assertEquals(1, s.size());
+	}
+
+	@Test
+	public void consumingSingleLatinCodePointShouldMoveIndex() {
+		CodePointCharStream s = CharStreams.createWithString("X");
+		assertEquals(0, s.index());
+		s.consume();
+		assertEquals(1, s.index());
+	}
+
+	@Test
+	public void consumingPastSingleLatinCodePointShouldThrow() {
+		CodePointCharStream s = CharStreams.createWithString("X");
+		s.consume();
+		thrown.expect(IllegalStateException.class);
+		thrown.expectMessage("cannot consume EOF");
+		s.consume();
+	}
+
+	@Test
+	public void singleLatinCodePointLookAheadShouldReturnCodePoint() {
+		CodePointCharStream s = CharStreams.createWithString("X");
+		assertEquals('X', s.LA(1));
+		assertEquals(0, s.index());
+	}
+
+	@Test
+	public void multipleLatinCodePointsLookAheadShouldReturnCodePoints() {
+		CodePointCharStream s = CharStreams.createWithString("XYZ");
+		assertEquals('X', s.LA(1));
+		assertEquals(0, s.index());
+		assertEquals('Y', s.LA(2));
+		assertEquals(0, s.index());
+		assertEquals('Z', s.LA(3));
+		assertEquals(0, s.index());
+	}
+
+	@Test
+	public void singleLatinCodePointLookAheadPastEndShouldReturnEOF() {
+		CodePointCharStream s = CharStreams.createWithString("X");
+		assertEquals(IntStream.EOF, s.LA(2));
+	}
+
+	@Test
+	public void singleCJKCodePointHasSize1() {
+		CodePointCharStream s = CharStreams.createWithString("\u611B");
+		assertEquals(1, s.size());
+		assertEquals(0, s.index());
+	}
+
+	@Test
+	public void consumingSingleCJKCodePointShouldMoveIndex() {
+		CodePointCharStream s = CharStreams.createWithString("\u611B");
+		assertEquals(0, s.index());
+		s.consume();
+		assertEquals(1, s.index());
+	}
+
+	@Test
+	public void consumingPastSingleCJKCodePointShouldThrow() {
+		CodePointCharStream s = CharStreams.createWithString("\u611B");
+		s.consume();
+		thrown.expect(IllegalStateException.class);
+		thrown.expectMessage("cannot consume EOF");
+		s.consume();
+	}
+
+	@Test
+	public void singleCJKCodePointLookAheadShouldReturnCodePoint() {
+		CodePointCharStream s = CharStreams.createWithString("\u611B");
+		assertEquals(0x611B, s.LA(1));
+		assertEquals(0, s.index());
+	}
+
+	@Test
+	public void singleCJKCodePointLookAheadPastEndShouldReturnEOF() {
+		CodePointCharStream s = CharStreams.createWithString("\u611B");
+		assertEquals(IntStream.EOF, s.LA(2));
+		assertEquals(0, s.index());
+	}
+
+	@Test
+	public void singleEmojiCodePointHasSize1() {
+		CodePointCharStream s = CharStreams.createWithString(
+				new StringBuilder().appendCodePoint(0x1F4A9).toString());
+		assertEquals(1, s.size());
+		assertEquals(0, s.index());
+	}
+
+	@Test
+	public void consumingSingleEmojiCodePointShouldMoveIndex() {
+		CodePointCharStream s = CharStreams.createWithString(
+				new StringBuilder().appendCodePoint(0x1F4A9).toString());
+		assertEquals(0, s.index());
+		s.consume();
+		assertEquals(1, s.index());
+	}
+
+	@Test
+	public void consumingPastEndOfEmojiCodePointWithShouldThrow() {
+		CodePointCharStream s = CharStreams.createWithString(
+				new StringBuilder().appendCodePoint(0x1F4A9).toString());
+		assertEquals(0, s.index());
+		s.consume();
+		assertEquals(1, s.index());
+		thrown.expect(IllegalStateException.class);
+		thrown.expectMessage("cannot consume EOF");
+		s.consume();
+	}
+
+	@Test
+	public void singleEmojiCodePointLookAheadShouldReturnCodePoint() {
+		CodePointCharStream s = CharStreams.createWithString(
+				new StringBuilder().appendCodePoint(0x1F4A9).toString());
+		assertEquals(0x1F4A9, s.LA(1));
+		assertEquals(0, s.index());
+	}
+
+	@Test
+	public void singleEmojiCodePointLookAheadPastEndShouldReturnEOF() {
+		CodePointCharStream s = CharStreams.createWithString(
+				new StringBuilder().appendCodePoint(0x1F4A9).toString());
+		assertEquals(IntStream.EOF, s.LA(2));
+		assertEquals(0, s.index());
+	}
+
+	@Test
+	public void getTextWithLatin() {
+		CodePointCharStream s = CharStreams.createWithString("0123456789");
+		assertEquals("34567", s.getText(Interval.of(3, 7)));
+	}
+
+	@Test
+	public void getTextWithCJK() {
+		CodePointCharStream s = CharStreams.createWithString("01234\u40946789");
+		assertEquals("34\u409467", s.getText(Interval.of(3, 7)));
+	}
+
+	@Test
+	public void getTextWithEmoji() {
+		CodePointCharStream s = CharStreams.createWithString(
+				new StringBuilder("01234")
+					.appendCodePoint(0x1F522)
+					.append("6789")
+					.toString());
+		assertEquals("34\uD83D\uDD2267", s.getText(Interval.of(3, 7)));
+	}
+
+	@Test
+	public void toStringWithLatin() {
+		CodePointCharStream s = CharStreams.createWithString("0123456789");
+		assertEquals("0123456789", s.toString());
+	}
+
+	@Test
+	public void toStringWithCJK() {
+		CodePointCharStream s = CharStreams.createWithString("01234\u40946789");
+		assertEquals("01234\u40946789", s.toString());
+	}
+
+	@Test
+	public void toStringWithEmoji() {
+		CodePointCharStream s = CharStreams.createWithString(
+				new StringBuilder("01234")
+					.appendCodePoint(0x1F522)
+					.append("6789")
+					.toString());
+		assertEquals("01234\uD83D\uDD226789", s.toString());
+	}
+
+	@Test
+	public void lookAheadWithLatin() {
+		CodePointCharStream s = CharStreams.createWithString("0123456789");
+		assertEquals('5', s.LA(6));
+	}
+
+	@Test
+	public void lookAheadWithCJK() {
+		CodePointCharStream s = CharStreams.createWithString("01234\u40946789");
+		assertEquals(0x4094, s.LA(6));
+	}
+
+	@Test
+	public void lookAheadWithEmoji() {
+		CodePointCharStream s = CharStreams.createWithString(
+				new StringBuilder("01234")
+					.appendCodePoint(0x1F522)
+					.append("6789")
+					.toString());
+		assertEquals(0x1F522, s.LA(6));
+	}
+
+	@Test
+	public void seekWithLatin() {
+		CodePointCharStream s = CharStreams.createWithString("0123456789");
+		s.seek(5);
+		assertEquals('5', s.LA(1));
+	}
+
+	@Test
+	public void seekWithCJK() {
+		CodePointCharStream s = CharStreams.createWithString("01234\u40946789");
+		s.seek(5);
+		assertEquals(0x4094, s.LA(1));
+	}
+
+	@Test
+	public void seekWithEmoji() {
+		CodePointCharStream s = CharStreams.createWithString(
+				new StringBuilder("01234")
+					.appendCodePoint(0x1F522)
+					.append("6789")
+					.toString());
+		s.seek(5);
+		assertEquals(0x1F522, s.LA(1));
+	}
+
+	@Test
+	public void lookBehindWithLatin() {
+		CodePointCharStream s = CharStreams.createWithString("0123456789");
+		s.seek(6);
+		assertEquals('5', s.LA(-1));
+	}
+
+	@Test
+	public void lookBehindWithCJK() {
+		CodePointCharStream s = CharStreams.createWithString("01234\u40946789");
+		s.seek(6);
+		assertEquals(0x4094, s.LA(-1));
+	}
+
+	@Test
+	public void lookBehindWithEmoji() {
+		CodePointCharStream s = CharStreams.createWithString(
+				new StringBuilder("01234")
+					.appendCodePoint(0x1F522)
+					.append("6789")
+					.toString());
+		s.seek(6);
+		assertEquals(0x1F522, s.LA(-1));
+	}
+}

runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestIntegerList.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+package org.antlr.v4.test.runtime.java;
+
+import static org.junit.Assert.assertArrayEquals;
+
+import org.antlr.v4.runtime.misc.IntegerList;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class TestIntegerList {
+	@Rule
+	public ExpectedException thrown = ExpectedException.none();
+
+	@Test
+	public void emptyListToEmptyCharArray() {
+		IntegerList l = new IntegerList();
+		assertArrayEquals(new char[0], l.toCharArray());
+	}
+
+	@Test
+	public void negativeIntegerToCharArrayThrows() {
+		IntegerList l = new IntegerList();
+		l.add(-42);
+		thrown.expect(IllegalArgumentException.class);
+		l.toCharArray();
+	}
+
+	@Test
+	public void surrogateRangeIntegerToCharArray() {
+		IntegerList l = new IntegerList();
+		// Java allows dangling surrogates, so (currently) we do
+		// as well. We could change this if desired.
+		l.add(0xDC00);
+		char expected[] = new char[] { 0xDC00 };
+		assertArrayEquals(expected, l.toCharArray());
+	}
+
+	@Test
+	public void tooLargeIntegerToCharArrayThrows() {
+		IntegerList l = new IntegerList();
+		l.add(0x110000);
+		thrown.expect(IllegalArgumentException.class);
+		l.toCharArray();
+	}
+
+	@Test
+	public void unicodeBMPIntegerListToCharArray() {
+		IntegerList l = new IntegerList();
+		l.add(0x35);
+		l.add(0x4E94);
+		l.add(0xFF15);
+		char expected[] = new char[] { 0x35, 0x4E94, 0xFF15 };
+		assertArrayEquals(expected, l.toCharArray());
+	}
+
+	@Test
+	public void unicodeSMPIntegerListToCharArray() {
+		IntegerList l = new IntegerList();
+		l.add(0x104A5);
+		l.add(0x116C5);
+		l.add(0x1D7FB);
+		char expected[] = new char[] { 0xD801, 0xDCA5, 0xD805, 0xDEC5, 0xD835, 0xDFFB };
+		assertArrayEquals(expected, l.toCharArray());
+	}
+}

runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestUTF8CodePointDecoder.java

@@ -0,0 +1,162 @@
+/*
+ * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+package org.antlr.v4.test.runtime.java;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.IntBuffer;
+
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
+
+import org.antlr.v4.runtime.UTF8CodePointDecoder;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class TestUTF8CodePointDecoder {
+	@Rule
+	public ExpectedException thrown = ExpectedException.none();
+
+	@Test
+	public void decodeEmptyByteBufferWritesNothing() throws Exception {
+		UTF8CodePointDecoder decoder = new UTF8CodePointDecoder(CodingErrorAction.REPLACE);
+		ByteBuffer utf8BytesIn = ByteBuffer.allocate(0);
+		IntBuffer codePointsOut = IntBuffer.allocate(0);
+		IntBuffer result = decoder.decodeCodePointsFromBuffer(
+				utf8BytesIn,
+				codePointsOut,
+				true);
+		result.flip();
+		assertEquals(0, result.remaining());
+	}
+
+	@Test
+	public void decodeLatinByteBufferWritesCodePoint() throws Exception {
+		UTF8CodePointDecoder decoder = new UTF8CodePointDecoder(CodingErrorAction.REPLACE);
+		ByteBuffer utf8BytesIn = StandardCharsets.UTF_8.encode("X");
+		IntBuffer codePointsOut = IntBuffer.allocate(1);
+		IntBuffer result = decoder.decodeCodePointsFromBuffer(
+				utf8BytesIn,
+				codePointsOut,
+				true);
+		result.flip();
+		assertEquals(1, result.remaining());
+		assertEquals('X', result.get(0));
+	}
+
+	@Test
+	public void decodeCyrillicByteBufferWritesCodePoint() throws Exception {
+		UTF8CodePointDecoder decoder = new UTF8CodePointDecoder(CodingErrorAction.REPLACE);
+		ByteBuffer utf8BytesIn = StandardCharsets.UTF_8.encode("\u042F");
+		IntBuffer codePointsOut = IntBuffer.allocate(1);
+		IntBuffer result = decoder.decodeCodePointsFromBuffer(
+				utf8BytesIn,
+				codePointsOut,
+				true);
+		result.flip();
+		assertEquals(1, result.remaining());
+		assertEquals(0x042F, result.get(0));
+	}
+
+	@Test
+	public void decodeCJKByteBufferWritesCodePoint() throws Exception {
+		UTF8CodePointDecoder decoder = new UTF8CodePointDecoder(CodingErrorAction.REPLACE);
+		ByteBuffer utf8BytesIn = StandardCharsets.UTF_8.encode("\u611B");
+		IntBuffer codePointsOut = IntBuffer.allocate(1);
+		IntBuffer result = decoder.decodeCodePointsFromBuffer(
+				utf8BytesIn,
+				codePointsOut,
+				true);
+		result.flip();
+		assertEquals(1, result.remaining());
+		assertEquals(0x611B, result.get(0));
+	}
+
+	@Test
+	public void decodeEmojiByteBufferWritesCodePoint() throws Exception {
+		UTF8CodePointDecoder decoder = new UTF8CodePointDecoder(CodingErrorAction.REPLACE);
+		ByteBuffer utf8BytesIn = StandardCharsets.UTF_8.encode(
+				new StringBuilder().appendCodePoint(0x1F4A9).toString()
+		);
+		IntBuffer codePointsOut = IntBuffer.allocate(1);
+		IntBuffer result = decoder.decodeCodePointsFromBuffer(
+				utf8BytesIn,
+				codePointsOut,
+				true);
+		result.flip();
+		assertEquals(1, result.remaining());
+		assertEquals(0x1F4A9, result.get(0));
+	}
+
+	@Test
+	public void decodingInvalidLeadInReplaceModeWritesSubstitutionCharacter() throws Exception {
+		UTF8CodePointDecoder decoder = new UTF8CodePointDecoder(CodingErrorAction.REPLACE);
+		ByteBuffer utf8BytesIn = ByteBuffer.wrap(new byte[] { (byte)0xF8 });
+		IntBuffer codePointsOut = IntBuffer.allocate(1);
+		IntBuffer result = decoder.decodeCodePointsFromBuffer(utf8BytesIn, codePointsOut, true);
+		result.flip();
+		assertEquals(1, result.remaining());
+		assertEquals(0xFFFD, result.get(0));
+	}
+
+	@Test
+	public void decodingInvalidLeadInReportModeThrows() throws Exception {
+		UTF8CodePointDecoder decoder = new UTF8CodePointDecoder(CodingErrorAction.REPORT);
+		ByteBuffer utf8BytesIn = ByteBuffer.wrap(new byte[] { (byte)0xF8 });
+		IntBuffer codePointsOut = IntBuffer.allocate(1);
+		thrown.expect(CharacterCodingException.class);
+		thrown.expectMessage("Invalid UTF-8 leading byte 0xF8");
+		decoder.decodeCodePointsFromBuffer(utf8BytesIn, codePointsOut, true);
+	}
+
+	@Test
+	public void decodingInvalidTrailInReplaceModeWritesSubstitutionCharacter() throws Exception {
+		UTF8CodePointDecoder decoder = new UTF8CodePointDecoder(CodingErrorAction.REPLACE);
+		ByteBuffer utf8BytesIn = ByteBuffer.wrap(new byte[] { (byte)0xC0, (byte)0xC0 });
+		IntBuffer codePointsOut = IntBuffer.allocate(1);
+		IntBuffer result = decoder.decodeCodePointsFromBuffer(utf8BytesIn, codePointsOut, true);
+		result.flip();
+		assertEquals(1, result.remaining());
+		assertEquals(0xFFFD, result.get(0));
+	}
+
+	@Test
+	public void decodingInvalidTrailInReportModeThrows() throws Exception {
+		UTF8CodePointDecoder decoder = new UTF8CodePointDecoder(CodingErrorAction.REPORT);
+		ByteBuffer utf8BytesIn = ByteBuffer.wrap(new byte[] { (byte)0xC0, (byte)0xC0 });
+		IntBuffer codePointsOut = IntBuffer.allocate(1);
+		thrown.expect(CharacterCodingException.class);
+		thrown.expectMessage("Invalid UTF-8 trailing byte 0xC0");
+		decoder.decodeCodePointsFromBuffer(utf8BytesIn, codePointsOut, true);
+	}
+
+	@Test
+	public void decodingNonShortestFormInReplaceModeWritesSubstitutionCharacter() throws Exception {
+		UTF8CodePointDecoder decoder = new UTF8CodePointDecoder(CodingErrorAction.REPLACE);
+		// 0xC1 0x9C would decode to \ (U+005C) if we didn't have this check
+		ByteBuffer utf8BytesIn = ByteBuffer.wrap(new byte[] { (byte)0xC1, (byte)0x9C });
+		IntBuffer codePointsOut = IntBuffer.allocate(1);
+		IntBuffer result = decoder.decodeCodePointsFromBuffer(utf8BytesIn, codePointsOut, true);
+		result.flip();
+		assertEquals(1, result.remaining());
+		assertEquals(0xFFFD, result.get(0));
+	}
+
+	@Test
+	public void decodingNonShortestFormInReportModeThrows() throws Exception {
+		UTF8CodePointDecoder decoder = new UTF8CodePointDecoder(CodingErrorAction.REPORT);
+		// 0xC1 0x9C would decode to \ (U+005C) if we didn't have this check
+		ByteBuffer utf8BytesIn = ByteBuffer.wrap(new byte[] { (byte)0xC1, (byte)0x9C });
+		IntBuffer codePointsOut = IntBuffer.allocate(1);
+		thrown.expect(CharacterCodingException.class);
+		thrown.expectMessage("Code point 92 is out of expected range 128..2047");
+		decoder.decodeCodePointsFromBuffer(utf8BytesIn, codePointsOut, true);
+	}
+}

runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestVisitors.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-package org.antlr.v4.test.runtime.java;
-
-import org.antlr.v4.test.runtime.BaseRuntimeTest;
-import org.antlr.v4.test.runtime.RuntimeTestDescriptor;
-import org.antlr.v4.test.runtime.descriptors.VisitorsDescriptors;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class TestVisitors extends BaseRuntimeTest {
-	public TestVisitors(RuntimeTestDescriptor descriptor) {
-		super(descriptor,new BaseJavaTest());
-	}
-
-	@Parameterized.Parameters(name="{0}")
-	public static RuntimeTestDescriptor[] getAllTestDescriptors() {
-		return BaseRuntimeTest.getRuntimeTestDescriptors(VisitorsDescriptors.class, "Java");
-	}
-}

runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestExpectedTokens.java

@@ -1,3 +1,8 @@
+/*
+ * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
 package org.antlr.v4.test.runtime.java.api;
 
 import org.antlr.v4.runtime.ParserRuleContext;

runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestTokenStream.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+package org.antlr.v4.test.runtime.java.api;
+
+import org.antlr.v4.runtime.ANTLRInputStream;
+import org.antlr.v4.runtime.BufferedTokenStream;
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.TokenStream;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class contains tests for specific API functionality in {@link TokenStream} and derived types.
+ */
+public class TestTokenStream {
+
+	/**
+	 * This is a targeted regression test for antlr/antlr4#1584 ({@link BufferedTokenStream} cannot be reused after EOF).
+	 */
+	@Test
+	public void testBufferedTokenStreamReuseAfterFill() {
+		CharStream firstInput = new ANTLRInputStream("A");
+		BufferedTokenStream tokenStream = new BufferedTokenStream(new VisitorBasicLexer(firstInput));
+		tokenStream.fill();
+		Assert.assertEquals(2, tokenStream.size());
+		Assert.assertEquals(VisitorBasicLexer.A, tokenStream.get(0).getType());
+		Assert.assertEquals(Token.EOF, tokenStream.get(1).getType());
+
+		CharStream secondInput = new ANTLRInputStream("AA");
+		tokenStream.setTokenSource(new VisitorBasicLexer(secondInput));
+		tokenStream.fill();
+		Assert.assertEquals(3, tokenStream.size());
+		Assert.assertEquals(VisitorBasicLexer.A, tokenStream.get(0).getType());
+		Assert.assertEquals(VisitorBasicLexer.A, tokenStream.get(1).getType());
+		Assert.assertEquals(Token.EOF, tokenStream.get(2).getType());
+	}
+
+}

runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/TestVisitors.java

@@ -0,0 +1,237 @@
+/*
+ * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+package org.antlr.v4.test.runtime.java.api;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.antlr.v4.runtime.ANTLRInputStream;
+import org.antlr.v4.runtime.BaseErrorListener;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.RecognitionException;
+import org.antlr.v4.runtime.Recognizer;
+import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor;
+import org.antlr.v4.runtime.tree.ErrorNode;
+import org.antlr.v4.runtime.tree.RuleNode;
+import org.antlr.v4.runtime.tree.TerminalNode;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestVisitors {
+
+	/**
+	 * This test verifies the basic behavior of visitors, with an emphasis on
+	 * {@link AbstractParseTreeVisitor#visitTerminal}.
+	 */
+	@Test
+	public void testVisitTerminalNode() {
+		String input = "A";
+		VisitorBasicLexer lexer = new VisitorBasicLexer(new ANTLRInputStream(input));
+		VisitorBasicParser parser = new VisitorBasicParser(new CommonTokenStream(lexer));
+
+		VisitorBasicParser.SContext context = parser.s();
+		Assert.assertEquals("(s A <EOF>)", context.toStringTree(parser));
+
+		VisitorBasicVisitor<String> listener = new VisitorBasicBaseVisitor<String>() {
+			@Override
+			public String visitTerminal(TerminalNode node) {
+				return node.getSymbol().toString() + "\n";
+			}
+
+			@Override
+			protected String defaultResult() {
+				return "";
+			}
+
+			@Override
+			protected String aggregateResult(String aggregate, String nextResult) {
+				return aggregate + nextResult;
+			}
+		};
+
+		String result = listener.visit(context);
+		String expected =
+			"[@0,0:0='A',<1>,1:0]\n" +
+			"[@1,1:0='<EOF>',<-1>,1:1]\n";
+		Assert.assertEquals(expected, result);
+	}
+
+	/**
+	 * This test verifies the basic behavior of visitors, with an emphasis on
+	 * {@link AbstractParseTreeVisitor#visitErrorNode}.
+	 */
+	@Test
+	public void testVisitErrorNode() {
+		String input = "";
+		VisitorBasicLexer lexer = new VisitorBasicLexer(new ANTLRInputStream(input));
+		VisitorBasicParser parser = new VisitorBasicParser(new CommonTokenStream(lexer));
+
+		final List<String> errors = new ArrayList<>();
+		parser.removeErrorListeners();
+		parser.addErrorListener(new BaseErrorListener() {
+			@Override
+			public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) {
+				errors.add("line " + line + ":" + charPositionInLine + " " + msg);
+			}
+		});
+
+		VisitorBasicParser.SContext context = parser.s();
+		Assert.assertEquals("(s <missing 'A'> <EOF>)", context.toStringTree(parser));
+		Assert.assertEquals(1, errors.size());
+		Assert.assertEquals("line 1:0 missing 'A' at '<EOF>'", errors.get(0));
+
+		VisitorBasicVisitor<String> listener = new VisitorBasicBaseVisitor<String>() {
+			@Override
+			public String visitErrorNode(ErrorNode node) {
+				return "Error encountered: " + node.getSymbol();
+			}
+
+			@Override
+			protected String defaultResult() {
+				return "";
+			}
+
+			@Override
+			protected String aggregateResult(String aggregate, String nextResult) {
+				return aggregate + nextResult;
+			}
+		};
+
+		String result = listener.visit(context);
+		String expected = "Error encountered: [@-1,-1:-1='<missing 'A'>',<1>,1:0]";
+		Assert.assertEquals(expected, result);
+	}
+
+	/**
+	 * This test verifies that {@link AbstractParseTreeVisitor#visitChildren} does not call
+	 * {@link ParseTreeVisitor#visit} after {@link AbstractParseTreeVisitor#shouldVisitNextChild} returns
+	 * {@code false}.
+	 */
+	@Test
+	public void testShouldNotVisitEOF() {
+		String input = "A";
+		VisitorBasicLexer lexer = new VisitorBasicLexer(new ANTLRInputStream(input));
+		VisitorBasicParser parser = new VisitorBasicParser(new CommonTokenStream(lexer));
+
+		VisitorBasicParser.SContext context = parser.s();
+		Assert.assertEquals("(s A <EOF>)", context.toStringTree(parser));
+
+		VisitorBasicVisitor<String> listener = new VisitorBasicBaseVisitor<String>() {
+			@Override
+			public String visitTerminal(TerminalNode node) {
+				return node.getSymbol().toString() + "\n";
+			}
+
+			@Override
+			protected boolean shouldVisitNextChild(RuleNode node, String currentResult) {
+				return currentResult == null || currentResult.isEmpty();
+			}
+		};
+
+		String result = listener.visit(context);
+		String expected = "[@0,0:0='A',<1>,1:0]\n";
+		Assert.assertEquals(expected, result);
+	}
+
+	/**
+	 * This test verifies that {@link AbstractParseTreeVisitor#shouldVisitNextChild} is called before visiting the first
+	 * child. It also verifies that {@link AbstractParseTreeVisitor#defaultResult} provides the default return value for
+	 * visiting a tree.
+	 */
+	@Test
+	public void testShouldNotVisitTerminal() {
+		String input = "A";
+		VisitorBasicLexer lexer = new VisitorBasicLexer(new ANTLRInputStream(input));
+		VisitorBasicParser parser = new VisitorBasicParser(new CommonTokenStream(lexer));
+
+		VisitorBasicParser.SContext context = parser.s();
+		Assert.assertEquals("(s A <EOF>)", context.toStringTree(parser));
+
+		VisitorBasicVisitor<String> listener = new VisitorBasicBaseVisitor<String>() {
+			@Override
+			public String visitTerminal(TerminalNode node) {
+				throw new RuntimeException("Should not be reachable");
+			}
+
+			@Override
+			protected String defaultResult() {
+				return "default result";
+			}
+
+			@Override
+			protected boolean shouldVisitNextChild(RuleNode node, String currentResult) {
+				return false;
+			}
+		};
+
+		String result = listener.visit(context);
+		String expected = "default result";
+		Assert.assertEquals(expected, result);
+	}
+
+	/**
+	 * This test verifies that the visitor correctly dispatches calls for labeled outer alternatives.
+	 */
+	@Test
+	public void testCalculatorVisitor() {
+		String input = "2 + 8 / 2";
+		VisitorCalcLexer lexer = new VisitorCalcLexer(new ANTLRInputStream(input));
+		VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer));
+
+		VisitorCalcParser.SContext context = parser.s();
+		Assert.assertEquals("(s (expr (expr 2) + (expr (expr 8) / (expr 2))) <EOF>)", context.toStringTree(parser));
+
+		VisitorCalcVisitor<Integer> listener = new VisitorCalcBaseVisitor<Integer>() {
+			@Override
+			public Integer visitS(VisitorCalcParser.SContext ctx) {
+				return visit(ctx.expr());
+			}
+
+			@Override
+			public Integer visitNumber(VisitorCalcParser.NumberContext ctx) {
+				return Integer.valueOf(ctx.INT().getText());
+			}
+
+			@Override
+			public Integer visitMultiply(VisitorCalcParser.MultiplyContext ctx) {
+				Integer left = visit(ctx.expr(0));
+				Integer right = visit(ctx.expr(1));
+				if (ctx.MUL() != null) {
+					return left * right;
+				}
+				else {
+					return left / right;
+				}
+			}
+
+			@Override
+			public Integer visitAdd(VisitorCalcParser.AddContext ctx) {
+				Integer left = visit(ctx.expr(0));
+				Integer right = visit(ctx.expr(1));
+				if (ctx.ADD() != null) {
+					return left + right;
+				}
+				else {
+					return left - right;
+				}
+			}
+
+			@Override
+			protected Integer defaultResult() {
+				throw new RuntimeException("Should not be reachable");
+			}
+
+			@Override
+			protected Integer aggregateResult(Integer aggregate, Integer nextResult) {
+				throw new RuntimeException("Should not be reachable");
+			}
+		};
+
+		int result = listener.visit(context);
+		int expected = 6;
+		Assert.assertEquals(expected, result);
+	}
+
+}

runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/VisitorBasic.g4

@@ -0,0 +1,7 @@
+grammar VisitorBasic;
+
+s
+	:	'A' EOF
+	;
+
+A : 'A';

runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/VisitorCalc.g4

@@ -0,0 +1,18 @@
+grammar VisitorCalc;
+
+s
+	:	expr EOF
+	;
+
+expr
+	:	INT						# number
+	|	expr (MUL | DIV) expr	# multiply
+	|	expr (ADD | SUB) expr	# add
+	;
+
+INT : [0-9]+;
+MUL : '*';
+DIV : '/';
+ADD : '+';
+SUB : '-';
+WS : [ \t]+ -> channel(HIDDEN);

runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/browser/BaseBrowserTest.java

@@ -53,13 +53,8 @@ import org.stringtemplate.v4.ST;
 import org.stringtemplate.v4.STGroup;
 import org.stringtemplate.v4.STGroupString;
 
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.net.BindException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -72,6 +67,7 @@ import java.util.Set;
 import java.util.TreeMap;
 
 import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
+import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -528,41 +524,6 @@ public abstract class BaseBrowserTest implements RuntimeTestSupport {
 		}
 	}
 
-	public static class StreamVacuum implements Runnable {
-		StringBuilder buf = new StringBuilder();
-		BufferedReader in;
-		Thread sucker;
-		public StreamVacuum(InputStream in) {
-			this.in = new BufferedReader( new InputStreamReader(in) );
-		}
-		public void start() {
-			sucker = new Thread(this);
-			sucker.start();
-		}
-		@Override
-		public void run() {
-			try {
-				String line = in.readLine();
-				while (line!=null) {
-					buf.append(line);
-					buf.append('\n');
-					line = in.readLine();
-				}
-			}
-			catch (IOException ioe) {
-				System.err.println("can't read output from process");
-			}
-		}
-		/** wait for the thread to finish */
-		public void join() throws InterruptedException {
-			sucker.join();
-		}
-		@Override
-		public String toString() {
-			return buf.toString();
-		}
-	}
-
 	protected void checkGrammarSemanticsError(ErrorQueue equeue,
 	                                          GrammarSemanticsMessage expectedMessage)
 		throws Exception
@@ -646,21 +607,6 @@ public abstract class BaseBrowserTest implements RuntimeTestSupport {
 		}
 	}
 
-	public static void writeFile(String dir, String fileName, String content) {
-		try {
-			File f = new File(dir, fileName);
-			FileWriter w = new FileWriter(f);
-			BufferedWriter bw = new BufferedWriter(w);
-			bw.write(content);
-			bw.close();
-			w.close();
-		}
-		catch (IOException ioe) {
-			System.err.println("can't write file");
-			ioe.printStackTrace(System.err);
-		}
-	}
-
 	protected void mkdir(String dir) {
 		File f = new File(dir);
 		f.mkdirs();
@@ -732,7 +678,7 @@ public abstract class BaseBrowserTest implements RuntimeTestSupport {
 			"			test = function() {\r\n" +
 			"				document.getElementById('output').value = ''\r\n" +
 			"				var input = document.getElementById('input').value;\r\n" +
-			"    			var stream = new antlr4.InputStream(input);\n" +
+			"			var stream = new antlr4.InputStream(input, true);\n" +
 			"    			var lexer = new " + lexerName + "." + lexerName + "(stream);\n" +
 			"				lexer._listeners = [new listener()];\r\n" +
 			"    			var tokens = new antlr4.CommonTokenStream(lexer);\n" +
@@ -791,7 +737,7 @@ public abstract class BaseBrowserTest implements RuntimeTestSupport {
 			"			test = function() {\r\n" +
 			"				document.getElementById('output').value = ''\r\n" +
 			"				var input = document.getElementById('input').value;\r\n" +
-			"    			var chars = new antlr4.InputStream(input);\r\n" +
+			"			var chars = new antlr4.InputStream(input, true);\r\n" +
 			"    			var lexer = new " + lexerName + "." + lexerName + "(chars);\r\n" +
 			"				lexer._listeners = [new listener()];\r\n" +
 			"    			var stream = new antlr4.CommonTokenStream(lexer);\r\n" +

runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/chrome/TestVisitors.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-package org.antlr.v4.test.runtime.javascript.chrome;
-
-import org.antlr.v4.test.runtime.BaseRuntimeTest;
-import org.antlr.v4.test.runtime.RuntimeTestDescriptor;
-import org.antlr.v4.test.runtime.descriptors.VisitorsDescriptors;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class TestVisitors extends BaseRuntimeTest {
-	public TestVisitors(RuntimeTestDescriptor descriptor) {
-		super(descriptor,new BaseChromeTest());
-	}
-
-	@Parameterized.Parameters(name="{0}")
-	public static RuntimeTestDescriptor[] getAllTestDescriptors() {
-		return BaseRuntimeTest.getRuntimeTestDescriptors(VisitorsDescriptors.class, "Chrome");
-	}
-}

runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/explorer/TestVisitors.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-package org.antlr.v4.test.runtime.javascript.explorer;
-
-import org.antlr.v4.test.runtime.BaseRuntimeTest;
-import org.antlr.v4.test.runtime.RuntimeTestDescriptor;
-import org.antlr.v4.test.runtime.descriptors.VisitorsDescriptors;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class TestVisitors extends BaseRuntimeTest {
-	public TestVisitors(RuntimeTestDescriptor descriptor) {
-		super(descriptor,new BaseExplorerTest());
-	}
-
-	@Parameterized.Parameters(name="{0}")
-	public static RuntimeTestDescriptor[] getAllTestDescriptors() {
-		return BaseRuntimeTest.getRuntimeTestDescriptors(VisitorsDescriptors.class, "Explorer");
-	}
-}

runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/firefox/TestVisitors.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-package org.antlr.v4.test.runtime.javascript.firefox;
-
-import org.antlr.v4.test.runtime.BaseRuntimeTest;
-import org.antlr.v4.test.runtime.RuntimeTestDescriptor;
-import org.antlr.v4.test.runtime.descriptors.VisitorsDescriptors;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class TestVisitors extends BaseRuntimeTest {
-	public TestVisitors(RuntimeTestDescriptor descriptor) {
-		super(descriptor,new BaseFirefoxTest());
-	}
-
-	@Parameterized.Parameters(name="{0}")
-	public static RuntimeTestDescriptor[] getAllTestDescriptors() {
-		return BaseRuntimeTest.getRuntimeTestDescriptors(VisitorsDescriptors.class, "Firefox");
-	}
-}

runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/BaseNodeTest.java

@@ -33,6 +33,7 @@ import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.semantics.SemanticPipeline;
 import org.antlr.v4.test.runtime.ErrorQueue;
 import org.antlr.v4.test.runtime.RuntimeTestSupport;
+import org.antlr.v4.test.runtime.StreamVacuum;
 import org.antlr.v4.tool.ANTLRMessage;
 import org.antlr.v4.tool.DOTGenerator;
 import org.antlr.v4.tool.Grammar;
@@ -43,15 +44,8 @@ import org.stringtemplate.v4.ST;
 import org.stringtemplate.v4.STGroup;
 import org.stringtemplate.v4.STGroupString;
 
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
 import java.io.File;
-import java.io.FileOutputStream;
 import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -64,6 +58,7 @@ import java.util.Set;
 import java.util.TreeMap;
 
 import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
+import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -524,45 +519,6 @@ public class BaseNodeTest implements RuntimeTestSupport {
 		}
 	}
 
-	public static class StreamVacuum implements Runnable {
-		StringBuilder buf = new StringBuilder();
-		BufferedReader in;
-		Thread sucker;
-
-		public StreamVacuum(InputStream in) {
-			this.in = new BufferedReader(new InputStreamReader(in));
-		}
-
-		public void start() {
-			sucker = new Thread(this);
-			sucker.start();
-		}
-
-		@Override
-		public void run() {
-			try {
-				String line = in.readLine();
-				while (line != null) {
-					buf.append(line);
-					buf.append('\n');
-					line = in.readLine();
-				}
-			} catch (IOException ioe) {
-				System.err.println("can't read output from process");
-			}
-		}
-
-		/** wait for the thread to finish */
-		public void join() throws InterruptedException {
-			sucker.join();
-		}
-
-		@Override
-		public String toString() {
-			return buf.toString();
-		}
-	}
-
 	protected void checkGrammarSemanticsError(ErrorQueue equeue,
 			GrammarSemanticsMessage expectedMessage) throws Exception {
 		ANTLRMessage foundMsg = null;
@@ -653,35 +609,6 @@ public class BaseNodeTest implements RuntimeTestSupport {
 		}
 	}
 
-	public static void writeFile(String dir, String fileName, String content) {
-		try {
-			File f = new File(dir, fileName);
-			FileWriter w = new FileWriter(f);
-			BufferedWriter bw = new BufferedWriter(w);
-			bw.write(content);
-			bw.close();
-			w.close();
-		} catch (IOException ioe) {
-			System.err.println("can't write file");
-			ioe.printStackTrace(System.err);
-		}
-	}
-
-	public static void writeFile(String dir, String fileName, InputStream content) {
-		try {
-			File f = new File(dir, fileName);
-			OutputStream output = new FileOutputStream(f);
-			while(content.available()>0) {
-				int b = content.read();
-				output.write(b);
-			}
-			output.close();
-		} catch (IOException ioe) {
-			System.err.println("can't write file");
-			ioe.printStackTrace(System.err);
-		}
-	}
-
 	protected void mkdir(String dir) {
 		File f = new File(dir);
 		f.mkdirs();
@@ -716,7 +643,7 @@ public class BaseNodeTest implements RuntimeTestSupport {
 						+ "};\n"
 						+ "\n"
 						+ "function main(argv) {\n"
-						+ "    var input = new antlr4.FileStream(argv[2]);\n"
+						+ "    var input = new antlr4.FileStream(argv[2], true);\n"
 						+ "    var lexer = new <lexerName>.<lexerName>(input);\n"
 						+ "    var stream = new antlr4.CommonTokenStream(lexer);\n"
 						+ "<createParser>"
@@ -752,7 +679,7 @@ public class BaseNodeTest implements RuntimeTestSupport {
 						+ "var <lexerName> = require('./<lexerName>');\n"
 						+ "\n"
 						+ "function main(argv) {\n"
-						+ "    var input = new antlr4.FileStream(argv[2]);\n"
+						+ "    var input = new antlr4.FileStream(argv[2], true);\n"
 						+ "    var lexer = new <lexerName>.<lexerName>(input);\n"
 						+ "    var stream = new antlr4.CommonTokenStream(lexer);\n"
 						+ "    stream.fill();\n"

runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/TestVisitors.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-package org.antlr.v4.test.runtime.javascript.node;
-
-import org.antlr.v4.test.runtime.BaseRuntimeTest;
-import org.antlr.v4.test.runtime.RuntimeTestDescriptor;
-import org.antlr.v4.test.runtime.descriptors.VisitorsDescriptors;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class TestVisitors extends BaseRuntimeTest {
-	public TestVisitors(RuntimeTestDescriptor descriptor) {
-		super(descriptor,new BaseNodeTest());
-	}
-
-	@Parameterized.Parameters(name="{0}")
-	public static RuntimeTestDescriptor[] getAllTestDescriptors() {
-		return BaseRuntimeTest.getRuntimeTestDescriptors(VisitorsDescriptors.class, "Node");
-	}
-}

runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/safari/TestVisitors.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-package org.antlr.v4.test.runtime.javascript.safari;
-
-import org.antlr.v4.test.runtime.BaseRuntimeTest;
-import org.antlr.v4.test.runtime.RuntimeTestDescriptor;
-import org.antlr.v4.test.runtime.descriptors.VisitorsDescriptors;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class TestVisitors extends BaseRuntimeTest {
-	public TestVisitors(RuntimeTestDescriptor descriptor) {
-		super(descriptor,new BaseSafariTest());
-	}
-
-	@Parameterized.Parameters(name="{0}")
-	public static RuntimeTestDescriptor[] getAllTestDescriptors() {
-		return BaseRuntimeTest.getRuntimeTestDescriptors(VisitorsDescriptors.class, "Safari");
-	}
-}

runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java

@@ -36,6 +36,8 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.semantics.SemanticPipeline;
 import org.antlr.v4.test.runtime.ErrorQueue;
 import org.antlr.v4.test.runtime.RuntimeTestSupport;
+import org.antlr.v4.test.runtime.StreamVacuum;
+import org.antlr.v4.test.runtime.TestOutputReading;
 import org.antlr.v4.tool.ANTLRMessage;
 import org.antlr.v4.tool.DOTGenerator;
 import org.antlr.v4.tool.Grammar;
@@ -49,13 +51,8 @@ import org.stringtemplate.v4.ST;
 import org.stringtemplate.v4.STGroup;
 import org.stringtemplate.v4.STGroupString;
 
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
 import java.io.File;
-import java.io.FileWriter;
+import java.nio.file.Path;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.net.URL;
@@ -70,6 +67,7 @@ import java.util.Set;
 import java.util.TreeMap;
 
 import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
+import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -488,24 +486,21 @@ public abstract class BasePythonTest implements RuntimeTestSupport {
 	public String execModule(String fileName) {
 		String pythonPath = locatePython();
 		String runtimePath = locateRuntime();
-		String modulePath = new File(new File(tmpdir), fileName).getAbsolutePath();
+		File tmpdirFile = new File(tmpdir);
-		String inputPath = new File(new File(tmpdir), "input").getAbsolutePath();
+		String modulePath = new File(tmpdirFile, fileName).getAbsolutePath();
+		String inputPath = new File(tmpdirFile, "input").getAbsolutePath();
+		Path outputPath = tmpdirFile.toPath().resolve("output").toAbsolutePath();
 		try {
-			ProcessBuilder builder = new ProcessBuilder( pythonPath, modulePath, inputPath );
+			ProcessBuilder builder = new ProcessBuilder( pythonPath, modulePath, inputPath, outputPath.toString() );
 			builder.environment().put("PYTHONPATH",runtimePath);
-			builder.directory(new File(tmpdir));
+			builder.environment().put("PYTHONIOENCODING", "utf-8");
+			builder.directory(tmpdirFile);
 			Process process = builder.start();
-			StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
 			StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
-			stdoutVacuum.start();
 			stderrVacuum.start();
 			process.waitFor();
-			stdoutVacuum.join();
 			stderrVacuum.join();
-			String output = stdoutVacuum.toString();
+			String output = TestOutputReading.read(outputPath);
-			if ( output.length()==0 ) {
-				output = null;
-			}
 			if ( stderrVacuum.toString().length()>0 ) {
 				this.stderrDuringParse = stderrVacuum.toString();
 			}
@@ -651,41 +646,6 @@ public abstract class BasePythonTest implements RuntimeTestSupport {
 		}
 	}
 
-	public static class StreamVacuum implements Runnable {
-		StringBuilder buf = new StringBuilder();
-		BufferedReader in;
-		Thread sucker;
-		public StreamVacuum(InputStream in) {
-			this.in = new BufferedReader( new InputStreamReader(in) );
-		}
-		public void start() {
-			sucker = new Thread(this);
-			sucker.start();
-		}
-		@Override
-		public void run() {
-			try {
-				String line = in.readLine();
-				while (line!=null) {
-					buf.append(line);
-					buf.append('\n');
-					line = in.readLine();
-				}
-			}
-			catch (IOException ioe) {
-				System.err.println("can't read output from process");
-			}
-		}
-		/** wait for the thread to finish */
-		public void join() throws InterruptedException {
-			sucker.join();
-		}
-		@Override
-		public String toString() {
-			return buf.toString();
-		}
-	}
-
 	protected void checkGrammarSemanticsError(ErrorQueue equeue,
 	                                          GrammarSemanticsMessage expectedMessage)
 		throws Exception
@@ -769,21 +729,6 @@ public abstract class BasePythonTest implements RuntimeTestSupport {
 		}
 	}
 
-	public static void writeFile(String dir, String fileName, String content) {
-		try {
-			File f = new File(dir, fileName);
-			FileWriter w = new FileWriter(f);
-			BufferedWriter bw = new BufferedWriter(w);
-			bw.write(content);
-			bw.close();
-			w.close();
-		}
-		catch (IOException ioe) {
-			System.err.println("can't write file");
-			ioe.printStackTrace(System.err);
-		}
-	}
-
 	protected void mkdir(String dir) {
 		File f = new File(dir);
 		f.mkdirs();

runtime-testsuite/test/org/antlr/v4/test/runtime/python2/BasePython2Test.java

@@ -9,6 +9,8 @@ package org.antlr.v4.test.runtime.python2;
 import org.antlr.v4.test.runtime.python.BasePythonTest;
 import org.stringtemplate.v4.ST;
 
+import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
+
 public class BasePython2Test extends BasePythonTest {
 
 	@Override
@@ -26,16 +28,18 @@ public class BasePython2Test extends BasePythonTest {
 		ST outputFileST = new ST(
 				"from __future__ import print_function\n"
 						+ "import sys\n"
+						+ "import codecs\n"
 						+ "from antlr4 import *\n"
 						+ "from <lexerName> import <lexerName>\n"
 						+ "\n"
 						+ "def main(argv):\n"
-						+ "    input = FileStream(argv[1])\n"
+						+ "    input = FileStream(argv[1], encoding='utf-8', errors='replace')\n"
-						+ "    lexer = <lexerName>(input)\n"
+						+ "    with codecs.open(argv[2], 'w', 'utf-8', 'replace') as output:\n"
-						+ "    stream = CommonTokenStream(lexer)\n"
+						+ "        lexer = <lexerName>(input, output)\n"
-						+ "    stream.fill()\n"
+						+ "        stream = CommonTokenStream(lexer)\n"
-						+ "    [ print(str(t)) for t in stream.tokens ]\n"
+						+ "        stream.fill()\n"
-						+ (showDFA ? "    print(lexer._interp.decisionToDFA[Lexer.DEFAULT_MODE].toLexerString(), end='')\n"
+						+ "        [ print(t, file=output) for t in stream.tokens ]\n"
+						+ (showDFA ? "        print(lexer._interp.decisionToDFA[Lexer.DEFAULT_MODE].toLexerString(), end='', file=output)\n"
 								: "") + "\n" + "if __name__ == '__main__':\n"
 						+ "    main(sys.argv)\n" + "\n");
 		outputFileST.add("lexerName", lexerName);
@@ -50,6 +54,7 @@ public class BasePython2Test extends BasePythonTest {
 			parserStartRuleName += "()";
 		ST outputFileST = new ST(
 				"import sys\n"
+						+ "import codecs\n"
 						+ "from antlr4 import *\n"
 						+ "from <lexerName> import <lexerName>\n"
 						+ "from <parserName> import <parserName>\n"
@@ -74,20 +79,21 @@ public class BasePython2Test extends BasePythonTest {
 						+ "                raise IllegalStateException(\"Invalid parse tree shape detected.\")\n"
 						+ "\n"
 						+ "def main(argv):\n"
-						+ "    input = FileStream(argv[1])\n"
+						+ "    input = FileStream(argv[1], encoding='utf-8', errors='replace')\n"
-						+ "    lexer = <lexerName>(input)\n"
+						+ "    with codecs.open(argv[2], 'w', 'utf-8', 'replace') as output:\n"
-						+ "    stream = CommonTokenStream(lexer)\n"
+						+ "        lexer = <lexerName>(input, output)\n"
+						+ "        stream = CommonTokenStream(lexer)\n"
 						+ "<createParser>"
-						+ "    parser.buildParseTrees = True\n"
+						+ "        parser.buildParseTrees = True\n"
-						+ "    tree = parser.<parserStartRuleName>\n"
+						+ "        tree = parser.<parserStartRuleName>\n"
-						+ "    ParseTreeWalker.DEFAULT.walk(TreeShapeListener(), tree)\n"
+						+ "        ParseTreeWalker.DEFAULT.walk(TreeShapeListener(), tree)\n"
 						+ "\n" + "if __name__ == '__main__':\n"
 						+ "    main(sys.argv)\n" + "\n");
-		String stSource = "    parser = <parserName>(stream)\n";
+		String stSource = "        parser = <parserName>(stream, output)\n";
 		if(debug)
-			stSource += "    parser.addErrorListener(DiagnosticErrorListener())\n";
+			stSource += "        parser.addErrorListener(DiagnosticErrorListener())\n";
 		if(trace)
-			stSource += "    parser.setTrace(True)\n";
+			stSource += "        parser.setTrace(True)\n";
 		ST createParserST = new ST(stSource);
 		outputFileST.add("createParser", createParserST);
 		outputFileST.add("parserName", parserName);

runtime-testsuite/test/org/antlr/v4/test/runtime/python2/TestVisitors.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-package org.antlr.v4.test.runtime.python2;
-
-import org.antlr.v4.test.runtime.BaseRuntimeTest;
-import org.antlr.v4.test.runtime.RuntimeTestDescriptor;
-import org.antlr.v4.test.runtime.descriptors.VisitorsDescriptors;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class TestVisitors extends BaseRuntimeTest {
-	public TestVisitors(RuntimeTestDescriptor descriptor) {
-		super(descriptor,new BasePython2Test());
-	}
-
-	@Parameterized.Parameters(name="{0}")
-	public static RuntimeTestDescriptor[] getAllTestDescriptors() {
-		return BaseRuntimeTest.getRuntimeTestDescriptors(VisitorsDescriptors.class, "Python2");
-	}
-}

runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java

@@ -8,6 +8,8 @@ package org.antlr.v4.test.runtime.python3;
 import org.antlr.v4.test.runtime.python.BasePythonTest;
 import org.stringtemplate.v4.ST;
 
+import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
+
 public class BasePython3Test extends BasePythonTest {
 
 	@Override
@@ -24,16 +26,18 @@ public class BasePython3Test extends BasePythonTest {
 	protected void writeLexerTestFile(String lexerName, boolean showDFA) {
 		ST outputFileST = new ST(
 				"import sys\n"
+						+ "import codecs\n"
 						+ "from antlr4 import *\n"
 						+ "from <lexerName> import <lexerName>\n"
 						+ "\n"
 						+ "def main(argv):\n"
-						+ "    input = FileStream(argv[1])\n"
+						+ "    input = FileStream(argv[1], encoding='utf-8', errors='replace')\n"
-						+ "    lexer = <lexerName>(input)\n"
+						+ "    with codecs.open(argv[2], 'w', 'utf-8', 'replace') as output:\n"
-						+ "    stream = CommonTokenStream(lexer)\n"
+						+ "        lexer = <lexerName>(input, output)\n"
-						+ "    stream.fill()\n"
+						+ "        stream = CommonTokenStream(lexer)\n"
-						+ "    [ print(str(t)) for t in stream.tokens ]\n"
+						+ "        stream.fill()\n"
-						+ (showDFA ? "    print(lexer._interp.decisionToDFA[Lexer.DEFAULT_MODE].toLexerString(), end='')\n"
+						+ "        [ print(t, file=output) for t in stream.tokens ]\n"
+						+ (showDFA ? "        print(lexer._interp.decisionToDFA[Lexer.DEFAULT_MODE].toLexerString(), end='', file=output)\n"
 								: "") + "\n" + "if __name__ == '__main__':\n"
 						+ "    main(sys.argv)\n" + "\n");
 		outputFileST.add("lexerName", lexerName);
@@ -48,6 +52,7 @@ public class BasePython3Test extends BasePythonTest {
 			parserStartRuleName += "()";
 		ST outputFileST = new ST(
 				"import sys\n"
+						+ "import codecs\n"
 						+ "from antlr4 import *\n"
 						+ "from <lexerName> import <lexerName>\n"
 						+ "from <parserName> import <parserName>\n"
@@ -72,20 +77,21 @@ public class BasePython3Test extends BasePythonTest {
 						+ "                raise IllegalStateException(\"Invalid parse tree shape detected.\")\n"
 						+ "\n"
 						+ "def main(argv):\n"
-						+ "    input = FileStream(argv[1])\n"
+						+ "    input = FileStream(argv[1], encoding='utf-8', errors='replace')\n"
-						+ "    lexer = <lexerName>(input)\n"
+						+ "    with codecs.open(argv[2], 'w', 'utf-8', 'replace') as output:\n"
-						+ "    stream = CommonTokenStream(lexer)\n"
+						+ "        lexer = <lexerName>(input, output)\n"
+						+ "        stream = CommonTokenStream(lexer)\n"
 						+ "<createParser>"
-						+ "    parser.buildParseTrees = True\n"
+						+ "        parser.buildParseTrees = True\n"
-						+ "    tree = parser.<parserStartRuleName>\n"
+						+ "        tree = parser.<parserStartRuleName>\n"
-						+ "    ParseTreeWalker.DEFAULT.walk(TreeShapeListener(), tree)\n"
+						+ "        ParseTreeWalker.DEFAULT.walk(TreeShapeListener(), tree)\n"
 						+ "\n" + "if __name__ == '__main__':\n"
 						+ "    main(sys.argv)\n" + "\n");
-		String stSource = "    parser = <parserName>(stream)\n";
+		String stSource = "        parser = <parserName>(stream, output)\n";
 		if (debug)
-			stSource += "    parser.addErrorListener(DiagnosticErrorListener())\n";
+			stSource += "        parser.addErrorListener(DiagnosticErrorListener())\n";
 		if (trace)
-			stSource += "    parser.setTrace(True)\n";
+			stSource += "        parser.setTrace(True)\n";
 		ST createParserST = new ST(stSource);
 		outputFileST.add("createParser", createParserST);
 		outputFileST.add("parserName", parserName);

runtime-testsuite/test/org/antlr/v4/test/runtime/python3/TestVisitors.java

@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-package org.antlr.v4.test.runtime.python3;
-
-import org.antlr.v4.test.runtime.BaseRuntimeTest;
-import org.antlr.v4.test.runtime.RuntimeTestDescriptor;
-import org.antlr.v4.test.runtime.descriptors.VisitorsDescriptors;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class TestVisitors extends BaseRuntimeTest {
-	public TestVisitors(RuntimeTestDescriptor descriptor) {
-		super(descriptor,new BasePython3Test());
-	}
-
-	@Parameterized.Parameters(name="{0}")
-	public static RuntimeTestDescriptor[] getAllTestDescriptors() {
-		return BaseRuntimeTest.getRuntimeTestDescriptors(VisitorsDescriptors.class, "Python3");
-	}
-}

runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java

@@ -9,15 +9,11 @@ package org.antlr.v4.test.runtime.swift;
 import org.antlr.v4.Tool;
 import org.antlr.v4.test.runtime.ErrorQueue;
 import org.antlr.v4.test.runtime.RuntimeTestSupport;
+import org.antlr.v4.test.runtime.StreamVacuum;
 import org.stringtemplate.v4.ST;
 
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
 import java.io.File;
-import java.io.FileWriter;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -27,6 +23,7 @@ import java.util.List;
 import java.util.Set;
 
 import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
+import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
 import static org.junit.Assert.assertTrue;
 
 public class BaseSwiftTest implements RuntimeTestSupport {
@@ -57,7 +54,7 @@ public class BaseSwiftTest implements RuntimeTestSupport {
 		//add antlr.swift
 		final ClassLoader loader = Thread.currentThread().getContextClassLoader();
 
-		final URL swiftRuntime = loader.getResource("Swift/Antlr4");
+		final URL swiftRuntime = loader.getResource("Swift/Sources/Antlr4");
 		if (swiftRuntime == null) {
 			throw new RuntimeException("Swift runtime file not found at:" + swiftRuntime.getPath());
 		}
@@ -118,49 +115,6 @@ public class BaseSwiftTest implements RuntimeTestSupport {
 		return runProcess(argsString, ANTLR_FRAMEWORK_DIR);
 	}
 
-	public static class StreamVacuum implements Runnable {
-		StringBuilder buf = new StringBuilder();
-		BufferedReader in;
-		Thread sucker;
-
-		public StreamVacuum(InputStream in) {
-			this.in = new BufferedReader(new InputStreamReader(in));
-		}
-
-		public void start() {
-			sucker = new Thread(this);
-			sucker.start();
-		}
-
-		@Override
-		public void run() {
-			try {
-				String line = in.readLine();
-				while (line != null) {
-					buf.append(line);
-					buf.append('\n');
-					line = in.readLine();
-				}
-			}
-			catch (IOException ioe) {
-				System.err.println("can't read output from process");
-				ioe.printStackTrace(System.err);
-			}
-		}
-
-		/**
-		 * wait for the thread to finish
-		 */
-		public void join() throws InterruptedException {
-			sucker.join();
-		}
-
-		@Override
-		public String toString() {
-			return buf.toString();
-		}
-	}
-
 	public String tmpdir = null;
 
 	/**
@@ -364,20 +318,6 @@ public class BaseSwiftTest implements RuntimeTestSupport {
 		return execTest();
 	}
 
-	public static void writeFile(String dir, String fileName, String content) {
-		try {
-			File f = new File(dir, fileName);
-			FileWriter w = new FileWriter(f);
-			BufferedWriter bw = new BufferedWriter(w);
-			bw.write(content);
-			bw.close();
-			w.close();
-		} catch (IOException ioe) {
-			System.err.println("can't write file");
-			ioe.printStackTrace(System.err);
-		}
-	}
-
 	protected void writeParserTestFile(String parserName,
 									   String lexerName,
 									   String parserStartRuleName,

runtime-testsuite/test/org/antlr/v4/test/runtime/swift/TestVisitors.java

@@ -1,26 +0,0 @@
-/*
- * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-
-package org.antlr.v4.test.runtime.swift;
-
-import org.antlr.v4.test.runtime.BaseRuntimeTest;
-import org.antlr.v4.test.runtime.RuntimeTestDescriptor;
-import org.antlr.v4.test.runtime.descriptors.VisitorsDescriptors;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class TestVisitors extends BaseRuntimeTest {
-	public TestVisitors(RuntimeTestDescriptor descriptor) {
-		super(descriptor,new BaseSwiftTest());
-	}
-
-	@Parameterized.Parameters(name="{0}")
-	public static RuntimeTestDescriptor[] getAllTestDescriptors() {
-		return BaseRuntimeTest.getRuntimeTestDescriptors(VisitorsDescriptors.class, "Swift");
-	}
-}
-

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.xproj

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+
+  <Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>12962409-846e-4b80-933a-bc484d264132</ProjectGuid>
+    <RootNamespace>Antlr4.Runtime.dotnet</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">.\obj</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+    <TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/AntlrInputStream.cs

@@ -4,36 +4,19 @@
  */
 using System;
 using System.IO;
+using System.Text;
 using Antlr4.Runtime;
 using Antlr4.Runtime.Misc;
 using Antlr4.Runtime.Sharpen;
 
 namespace Antlr4.Runtime
 {
-    /// <summary>
+    public abstract class BaseInputCharStream : ICharStream
-    /// Vacuum all input from a
-    /// <see cref="System.IO.TextReader"/>
-    /// /
-    /// <see cref="System.IO.Stream"/>
-    /// and then treat it
-    /// like a
-    /// <c>char[]</c>
-    /// buffer. Can also pass in a
-    /// <see cref="string"/>
-    /// or
-    /// <c>char[]</c>
-    /// to use.
-    /// <p>If you need encoding, pass in stream/reader with correct encoding.</p>
-    /// </summary>
-    public class AntlrInputStream : ICharStream
     {
         public const int ReadBufferSize = 1024;
 
         public const int InitialBufferSize = 1024;
 
-        /// <summary>The data being scanned</summary>
-        protected internal char[] data;
-
         /// <summary>How many characters are actually in the buffer</summary>
         protected internal int n;
 
@@ -43,72 +26,6 @@ namespace Antlr4.Runtime
         /// <summary>What is name or source of this char stream?</summary>
         public string name;
 
-        public AntlrInputStream()
-        {
-        }
-
-        /// <summary>Copy data in string to a local char array</summary>
-        public AntlrInputStream(string input)
-        {
-            this.data = input.ToCharArray();
-            this.n = input.Length;
-        }
-
-        /// <summary>This is the preferred constructor for strings as no data is copied</summary>
-        public AntlrInputStream(char[] data, int numberOfActualCharsInArray)
-        {
-            this.data = data;
-            this.n = numberOfActualCharsInArray;
-        }
-
-        /// <exception cref="System.IO.IOException"/>
-        public AntlrInputStream(TextReader r)
-            : this(r, InitialBufferSize, ReadBufferSize)
-        {
-        }
-
-        /// <exception cref="System.IO.IOException"/>
-        public AntlrInputStream(TextReader r, int initialSize)
-            : this(r, initialSize, ReadBufferSize)
-        {
-        }
-
-        /// <exception cref="System.IO.IOException"/>
-        public AntlrInputStream(TextReader r, int initialSize, int readChunkSize)
-        {
-            Load(r, initialSize, readChunkSize);
-        }
-
-        /// <exception cref="System.IO.IOException"/>
-        public AntlrInputStream(Stream input)
-            : this(new StreamReader(input), InitialBufferSize)
-        {
-        }
-
-        /// <exception cref="System.IO.IOException"/>
-        public AntlrInputStream(Stream input, int initialSize)
-            : this(new StreamReader(input), initialSize)
-        {
-        }
-
-        /// <exception cref="System.IO.IOException"/>
-        public AntlrInputStream(Stream input, int initialSize, int readChunkSize)
-            : this(new StreamReader(input), initialSize, readChunkSize)
-        {
-        }
-
-        /// <exception cref="System.IO.IOException"/>
-        public virtual void Load(TextReader r, int size, int readChunkSize)
-        {
-            if (r == null)
-            {
-                return;
-            }
-
-            data = r.ReadToEnd().ToCharArray();
-            n = data.Length;
-        }
-
         /// <summary>
         /// Reset the stream so that it's in the same state it was
         /// when the object was created *except* the data array is not
@@ -163,7 +80,7 @@ namespace Antlr4.Runtime
             }
             //System.out.println("char LA("+i+")="+(char)data[p+i-1]+"; p="+p);
             //System.out.println("LA("+i+"); p="+p+" n="+n+" data.length="+data.length);
-            return data[p + i - 1];
+            return ValueAt(p + i - 1);
         }
 
         public virtual int Lt(int i)
@@ -243,10 +160,16 @@ namespace Antlr4.Runtime
             {
                 return string.Empty;
             }
-            //		System.err.println("data: "+Arrays.toString(data)+", n="+n+
+            return ConvertDataToString(start, count);
-            //						   ", start="+start+
+        }
-            //						   ", stop="+stop);
+
-            return new string(data, start, count);
+        protected abstract int ValueAt(int i);
+
+        protected abstract string ConvertDataToString(int start, int count);
+
+        public override sealed string ToString()
+        {
+            return ConvertDataToString(0, n);
         }
 
         public virtual string SourceName
@@ -260,10 +183,148 @@ namespace Antlr4.Runtime
                 return name;
             }
         }
+    }
 
-        public override string ToString()
+    /// <summary>
+    /// Vacuum all input from a
+    /// <see cref="System.IO.TextReader"/>
+    /// /
+    /// <see cref="System.IO.Stream"/>
+    /// and then treat it
+    /// like a
+    /// <c>char[]</c>
+    /// buffer. Can also pass in a
+    /// <see cref="string"/>
+    /// or
+    /// <c>char[]</c>
+    /// to use.
+    /// <p>If you need encoding, pass in stream/reader with correct encoding.</p>
+    /// </summary>
+    public class AntlrInputStream : BaseInputCharStream
+    {
+        /// <summary>The data being scanned</summary>
+        protected internal char[] data;
+
+        public AntlrInputStream()
         {
-            return new string(data);
+        }
+
+        /// <summary>Copy data in string to a local char array</summary>
+        public AntlrInputStream(string input)
+        {
+            this.data = input.ToCharArray();
+            this.n = input.Length;
+        }
+
+        /// <summary>This is the preferred constructor for strings as no data is copied</summary>
+        public AntlrInputStream(char[] data, int numberOfActualCharsInArray)
+        {
+            this.data = data;
+            this.n = numberOfActualCharsInArray;
+        }
+
+        /// <exception cref="System.IO.IOException"/>
+        public AntlrInputStream(TextReader r)
+            : this(r, InitialBufferSize, ReadBufferSize)
+        {
+        }
+
+        /// <exception cref="System.IO.IOException"/>
+        public AntlrInputStream(TextReader r, int initialSize)
+            : this(r, initialSize, ReadBufferSize)
+        {
+        }
+
+        /// <exception cref="System.IO.IOException"/>
+        public AntlrInputStream(TextReader r, int initialSize, int readChunkSize)
+        {
+            Load(r, initialSize, readChunkSize);
+        }
+
+        /// <exception cref="System.IO.IOException"/>
+        public AntlrInputStream(Stream input)
+            : this(new StreamReader(input), InitialBufferSize)
+        {
+        }
+
+        /// <exception cref="System.IO.IOException"/>
+        public AntlrInputStream(Stream input, int initialSize)
+            : this(new StreamReader(input), initialSize)
+        {
+        }
+
+        /// <exception cref="System.IO.IOException"/>
+        public AntlrInputStream(Stream input, int initialSize, int readChunkSize)
+            : this(new StreamReader(input), initialSize, readChunkSize)
+        {
+        }
+
+        /// <exception cref="System.IO.IOException"/>
+        public virtual void Load(TextReader r, int size, int readChunkSize)
+        {
+            if (r == null)
+            {
+                return;
+            }
+
+            data = r.ReadToEnd().ToCharArray();
+            n = data.Length;
+        }
+
+        protected override int ValueAt(int i)
+        {
+            return data[i];
+        }
+
+        protected override string ConvertDataToString(int start, int count)
+        {
+            //		System.err.println("data: "+Arrays.toString(data)+", n="+n+
+            //                                             ", start="+start+
+            //                                             ", stop="+stop);
+            return new string(data, start, count);
+        }
+    }
+
+    /// <summary>
+    /// Alternative to
+    /// <see cref="ANTLRInputStream"/>
+    /// which treats the input as a series of Unicode code points,
+    /// instead of a series of UTF-16 code units.
+    ///
+    /// Use this if you need to parse input which potentially contains
+    /// Unicode values > U+FFFF.
+    /// </summary>
+    public class CodePointCharStream : BaseInputCharStream
+    {
+        private int[] data;
+
+        public CodePointCharStream(string input)
+        {
+            this.data = new int[input.Length];
+            int dataIdx = 0;
+            for (int i = 0; i < input.Length; ) {
+                var codePoint = Char.ConvertToUtf32(input, i);
+                data[dataIdx++] = codePoint;
+                if (dataIdx > data.Length) {
+                    Array.Resize(ref data, data.Length * 2);
+                }
+                i += codePoint <= 0xFFFF ? 1 : 2;
+            }
+            this.n = dataIdx;
+        }
+
+        protected override int ValueAt(int i)
+        {
+            return data[i];
+        }
+
+        protected override string ConvertDataToString(int start, int count)
+        {
+            var sb = new StringBuilder(count);
+            for (int i = start; i < start + count; i++) {
+                sb.Append(Char.ConvertFromUtf32(data[i]));
+            }
+            return sb.ToString();
         }
     }
 }

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/LexerATNSimulator.cs

@@ -380,7 +380,7 @@ namespace Antlr4.Runtime.Atn
 
 		protected ATNState GetReachableTarget(Transition trans, int t)
 		{
-			if (trans.Matches(t, char.MinValue, char.MaxValue))
+			if (trans.Matches(t, Lexer.MinCharValue, Lexer.MaxCharValue))
 			{
 				return trans.target;
 			}
@@ -572,7 +572,7 @@ namespace Antlr4.Runtime.Atn
 				case TransitionType.SET:
 					if (treatEofAsEpsilon)
 					{
-						if (t.Matches(IntStreamConstants.EOF, char.MinValue, char.MaxValue))
+						if (t.Matches(IntStreamConstants.EOF, Lexer.MinCharValue, Lexer.MaxCharValue))
 						{
 							c = new LexerATNConfig(config, t.target);
 							break;

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/OrderedATNConfigSet.cs

@@ -1,42 +0,0 @@
-/* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
- * Use of this file is governed by the BSD 3-clause license that
- * can be found in the LICENSE.txt file in the project root.
- */
-using Antlr4.Runtime.Atn;
-using Antlr4.Runtime.Sharpen;
-
-namespace Antlr4.Runtime.Atn
-{
-    /// <author>Sam Harwell</author>
-    public class OrderedATNConfigSet : ATNConfigSet
-    {
-        public OrderedATNConfigSet()
-        {
-        }
-
-        public OrderedATNConfigSet(ATNConfigSet set, bool @readonly)
-            : base(set, @readonly)
-        {
-        }
-
-        public override ATNConfigSet Clone(bool @readonly)
-        {
-            Antlr4.Runtime.Atn.OrderedATNConfigSet copy = new Antlr4.Runtime.Atn.OrderedATNConfigSet(this, @readonly);
-            if (!@readonly && this.IsReadOnly)
-            {
-                copy.AddAll(this);
-            }
-            return copy;
-        }
-
-        protected internal override long GetKey(ATNConfig e)
-        {
-            return e.GetHashCode();
-        }
-
-        protected internal override bool CanMerge(ATNConfig left, long leftKey, ATNConfig right)
-        {
-            return left.Equals(right);
-        }
-    }
-}

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/BaseErrorListener.cs

@@ -6,6 +6,7 @@ using Antlr4.Runtime;
 using Antlr4.Runtime.Atn;
 using Antlr4.Runtime.Dfa;
 using Antlr4.Runtime.Sharpen;
+using System.IO;
 
 namespace Antlr4.Runtime
 {
@@ -19,7 +20,7 @@ namespace Antlr4.Runtime
     /// <author>Sam Harwell</author>
     public class BaseErrorListener : IParserErrorListener
     {
-        public virtual void SyntaxError(IRecognizer recognizer, IToken offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e)
+        public virtual void SyntaxError(TextWriter output, IRecognizer recognizer, IToken offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e)
         {
         }
 

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/BufferedTokenStream.cs

@@ -371,6 +371,7 @@ namespace Antlr4.Runtime
             this._tokenSource = tokenSource;
             tokens.Clear();
             p = -1;
+			this.fetchedEOF = false;
         }
 
         public virtual IList<IToken> GetTokens()

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/ConsoleErrorListener.cs

@@ -7,6 +7,7 @@
 
 using Antlr4.Runtime;
 using Antlr4.Runtime.Sharpen;
+using System.IO;
 
 namespace Antlr4.Runtime
 {
@@ -38,9 +39,9 @@ namespace Antlr4.Runtime
         /// line <em>line</em>:<em>charPositionInLine</em> <em>msg</em>
         /// </pre>
         /// </summary>
-        public virtual void SyntaxError(IRecognizer recognizer, Symbol offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e)
+        public virtual void SyntaxError(TextWriter output, IRecognizer recognizer, Symbol offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e)
         {
-            System.Console.Error.WriteLine("line " + line + ":" + charPositionInLine + " " + msg);
+            output.WriteLine("line " + line + ":" + charPositionInLine + " " + msg);
         }
     }
 }

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/DefaultErrorStrategy.cs

@@ -265,12 +265,8 @@ namespace Antlr4.Runtime
             ITokenStream tokens = ((ITokenStream)recognizer.InputStream);
             int la = tokens.LA(1);
             // try cheaper subset first; might get lucky. seems to shave a wee bit off
-            if (recognizer.Atn.NextTokens(s).Contains(la) || la == TokenConstants.EOF)
+            var nextTokens = recognizer.Atn.NextTokens(s);
-            {
+            if (nextTokens.Contains(TokenConstants.EPSILON) || nextTokens.Contains(la))
-                return;
-            }
-            // Return but don't end recovery. only do that upon valid token match
-            if (recognizer.IsExpectedToken(la))
             {
                 return;
             }

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Dfa/SparseEdgeMap.cs

@@ -4,6 +4,7 @@
  */
 using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using Antlr4.Runtime.Sharpen;
 
 namespace Antlr4.Runtime.Dfa

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/IAntlrErrorListener.cs

@@ -4,6 +4,7 @@
  */
 using Antlr4.Runtime;
 using Antlr4.Runtime.Sharpen;
+using System.IO;
 
 namespace Antlr4.Runtime
 {
@@ -30,6 +31,9 @@ namespace Antlr4.Runtime
         /// in-line, without returning from the surrounding rule (via the single
         /// token insertion and deletion mechanism).</p>
         /// </remarks>
+        /// <param name="output">
+        /// Where the error should be written.
+        /// </param>
         /// <param name="recognizer">
         /// What parser got the error. From this
         /// object, you can access the context as well
@@ -52,6 +56,6 @@ namespace Antlr4.Runtime
         /// the parser was able to recover in line without exiting the
         /// surrounding rule.
         /// </param>
-        void SyntaxError(IRecognizer recognizer, TSymbol offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e);
+        void SyntaxError(TextWriter output, IRecognizer recognizer, TSymbol offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e);
     }
 }

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/IAntlrErrorStrategy.cs

@@ -5,6 +5,7 @@
 using Antlr4.Runtime;
 using Antlr4.Runtime.Misc;
 using Antlr4.Runtime.Sharpen;
+using System.IO;
 
 namespace Antlr4.Runtime
 {

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Lexer.cs

@@ -3,6 +3,7 @@
  * can be found in the LICENSE.txt file in the project root.
  */
 using System;
+using System.IO;
 using System.Collections.Generic;
 using System.Text;
 using Antlr4.Runtime;
@@ -27,12 +28,16 @@ namespace Antlr4.Runtime
 
         public const int Hidden = TokenConstants.HiddenChannel;
 
-        public const int MinCharValue = '\u0000';
+        public const int MinCharValue = 0x0000;
 
-        public const int MaxCharValue = '\uFFFE';
+        public const int MaxCharValue = 0x10FFFF;
 
         private ICharStream _input;
 
+        protected readonly TextWriter Output;
+
+        protected readonly TextWriter ErrorOutput;
+
 		private Tuple<ITokenSource, ICharStream> _tokenFactorySourcePair;
 
         /// <summary>How to create token objects</summary>
@@ -94,9 +99,13 @@ namespace Antlr4.Runtime
         /// </remarks>
 		private string _text;
 
-        public Lexer(ICharStream input)
+        public Lexer(ICharStream input) : this(input, Console.Out, Console.Error) { }
+
+        public Lexer(ICharStream input, TextWriter output, TextWriter errorOutput)
         {
             this._input = input;
+            this.Output = output;
+            this.ErrorOutput = errorOutput;
             this._tokenFactorySourcePair = Tuple.Create((ITokenSource)this, input);
         }
 
@@ -502,7 +511,7 @@ outer_continue: ;
             }
         }
 
-        public virtual string[] ModeNames
+        public virtual string[] ChannelNames
         {
             get
             {
@@ -510,6 +519,13 @@ outer_continue: ;
             }
         }
 
+        public virtual string[] ModeNames
+        {
+            get
+            {
+                return null;
+            }
+        }
 
         /// <summary>Return a list of all Token objects in input char stream.</summary>
         /// <remarks>
@@ -542,22 +558,23 @@ outer_continue: ;
             string text = _input.GetText(Interval.Of(_tokenStartCharIndex, _input.Index));
             string msg = "token recognition error at: '" + GetErrorDisplay(text) + "'";
             IAntlrErrorListener<int> listener = ErrorListenerDispatch;
-            listener.SyntaxError(this, 0, _tokenStartLine, _tokenStartColumn, msg, e);
+            listener.SyntaxError(ErrorOutput, this, 0, _tokenStartLine, _tokenStartColumn, msg, e);
         }
 
         public virtual string GetErrorDisplay(string s)
         {
             StringBuilder buf = new StringBuilder();
-            foreach (char c in s.ToCharArray())
+            for (var i = 0; i < s.Length; ) {
-            {
+                var codePoint = Char.ConvertToUtf32(s, i);
-                buf.Append(GetErrorDisplay(c));
+                buf.Append(GetErrorDisplay(codePoint));
+                i += (codePoint > 0xFFFF) ? 2 : 1;
             }
             return buf.ToString();
         }
 
         public virtual string GetErrorDisplay(int c)
         {
-            string s = ((char)c).ToString();
+            string s;
             switch (c)
             {
                 case TokenConstants.EOF:
@@ -583,6 +600,12 @@ outer_continue: ;
                     s = "\\r";
                     break;
                 }
+
+                default:
+                {
+                    s = Char.ConvertFromUtf32(c);
+                    break;
+                }
             }
             return s;
         }

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/LexerInterpreter.cs

@@ -13,23 +13,31 @@ using Antlr4.Runtime.Sharpen;
 
 namespace Antlr4.Runtime
 {
-    public class LexerInterpreter : Lexer
+    public class LexerInterpreter: Lexer
     {
-		private readonly string grammarFileName;
+        private readonly string grammarFileName;
 
-		private readonly ATN atn;
+        private readonly ATN atn;
 
-		private readonly string[] ruleNames;
+        private readonly string[] ruleNames;
 
-		private readonly string[] modeNames;
+        private readonly string[] channelNames;
+
+        private readonly string[] modeNames;
 
         [NotNull]
         private readonly IVocabulary vocabulary;
 
-		protected DFA[] decisionToDFA;
+        protected DFA[] decisionToDFA;
-		protected PredictionContextCache sharedContextCache = new PredictionContextCache();
+        protected PredictionContextCache sharedContextCache = new PredictionContextCache();
 
+        [Obsolete("Use constructor with channelNames argument")]
         public LexerInterpreter(string grammarFileName, IVocabulary vocabulary, IEnumerable<string> ruleNames, IEnumerable<string> modeNames, ATN atn, ICharStream input)
+            : this(grammarFileName, vocabulary, ruleNames, new string[0], modeNames, atn, input)
+        {
+        }
+
+        public LexerInterpreter(string grammarFileName, IVocabulary vocabulary, IEnumerable<string> ruleNames, IEnumerable<string> channelNames, IEnumerable<string> modeNames, ATN atn, ICharStream input)
             : base(input)
         {
             if (atn.grammarType != ATNType.Lexer)
@@ -39,14 +47,15 @@ namespace Antlr4.Runtime
             this.grammarFileName = grammarFileName;
             this.atn = atn;
             this.ruleNames = ruleNames.ToArray();
+            this.channelNames = channelNames.ToArray();
             this.modeNames = modeNames.ToArray();
             this.vocabulary = vocabulary;
-			this.decisionToDFA = new DFA[atn.NumberOfDecisions];
+            this.decisionToDFA = new DFA[atn.NumberOfDecisions];
-			for (int i = 0; i < decisionToDFA.Length; i++)
+            for (int i = 0; i < decisionToDFA.Length; i++)
-			{
+            {
-				decisionToDFA[i] = new DFA(atn.GetDecisionState(i), i);
+                decisionToDFA[i] = new DFA(atn.GetDecisionState(i), i);
-			}
+            }
-			this.Interpreter = new LexerATNSimulator(this, atn, decisionToDFA, sharedContextCache);
+            this.Interpreter = new LexerATNSimulator(this, atn, decisionToDFA, sharedContextCache);
         }
 
         public override ATN Atn
@@ -73,6 +82,14 @@ namespace Antlr4.Runtime
             }
         }
 
+        public override string[] ChannelNames
+        {
+            get
+            {
+                return channelNames;
+            }
+        }
+
         public override string[] ModeNames
         {
             get

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Misc/DoubleKeyMap.cs

@@ -1,58 +0,0 @@
-using System;
-using System.Collections.Generic;
-
-namespace Antlr4.Runtime.Misc
-{
-	public class DoubleKeyMap<K1, K2, V>
-	{
-		Dictionary<K1, Dictionary<K2, V>> data = new Dictionary<K1, Dictionary<K2, V>>();
-
-		public V put(K1 k1, K2 k2, V v)
-		{
-			Dictionary<K2, V> data2 = data.get(k1);
-			V prev = null;
-			if (data2 == null)
-			{
-				data2 = new Dict<K2, V>();
-				data.put(k1, data2);
-			}
-			else {
-				prev = data2.get(k2);
-			}
-			data2.put(k2, v);
-			return prev;
-		}
-
-		public V get(K1 k1, K2 k2)
-		{
-			Dictionary<K2, V> data2 = data.get(k1);
-			if (data2 == null) return null;
-			return data2.get(k2);
-		}
-
-		public Dictionary<K2, V> get(K1 k1) { return data.get(k1); }
-
-		/** Get all values associated with primary key */
-		public ICollection<V> values(K1 k1)
-		{
-			Dictionary<K2, V> data2 = data.get(k1);
-			if (data2 == null) return null;
-			return data2.values();
-		}
-
-		/** get all primary keys */
-		public HashSet<K1> keySet()
-		{
-			return data.keySet();
-		}
-
-		/** get all secondary keys associated with a primary key */
-		public HashSet<K2> keySet(K1 k1)
-		{
-			Dictionary<K2, V> data2 = data.get(k1);
-			if (data2 == null) return null;
-			return data2.keySet();
-		}
-	}
-
-}

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Misc/MultiMap.cs

@@ -2,6 +2,7 @@
  * Use of this file is governed by the BSD 3-clause license that
  * can be found in the LICENSE.txt file in the project root.
  */
+using System;
 using System.Collections.Generic;
 using Antlr4.Runtime.Sharpen;
 

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Misc/RuleDependencyChecker.cs

@@ -354,7 +354,7 @@ namespace Antlr4.Runtime.Misc
                 }
                 foreach (Transition transition in state.transitions)
                 {
-                    if (transition.TransitionType != TransitionType.Rule)
+                    if (transition.TransitionType != TransitionType.RULE)
                     {
                         continue;
                     }
@@ -459,7 +459,7 @@ namespace Antlr4.Runtime.Misc
         {
         }
 
-#if PORTABLE
+#if PORTABLE || DOTNETCORE
         public interface ICustomAttributeProvider
         {
             object[] GetCustomAttributes(Type attributeType, bool inherit);
@@ -646,7 +646,11 @@ namespace Antlr4.Runtime.Misc
             StringBuilder errors = new StringBuilder();
             foreach (Tuple<RuleDependencyAttribute, ICustomAttributeProvider> dependency in dependencies)
             {
-                if (!dependency.Item1.Recognizer.IsAssignableFrom(recognizerType))
+#if DOTNETCORE
+                if (!dependency.Item1.Recognizer.GetTypeInfo().IsAssignableFrom(recognizerType))
+#else
+                    if (!dependency.Item1.Recognizer.IsAssignableFrom(recognizerType))
+#endif
                 {
                     continue;
                 }
@@ -773,7 +777,11 @@ namespace Antlr4.Runtime.Misc
         private static int[] GetRuleVersions(Type recognizerClass, string[] ruleNames)
         {
             int[] versions = new int[ruleNames.Length];
+#if DOTNETCORE
+            FieldInfo[] fields = recognizerClass.GetTypeInfo().GetFields();
+#else
             FieldInfo[] fields = recognizerClass.GetFields();
+#endif
             foreach (FieldInfo field in fields)
             {
                 bool isStatic = field.IsStatic;
@@ -805,7 +813,11 @@ namespace Antlr4.Runtime.Misc
 #endif
                             continue;
                         }
+#if DOTNETCORE
+                        RuleVersionAttribute ruleVersion = ruleMethod.GetCustomAttribute<RuleVersionAttribute>();
+#else
                         RuleVersionAttribute ruleVersion = (RuleVersionAttribute)Attribute.GetCustomAttribute(ruleMethod, typeof(RuleVersionAttribute));
+#endif
                         int version = ruleVersion != null ? ruleVersion.Version : 0;
                         versions[index] = version;
                     }
@@ -832,10 +844,18 @@ namespace Antlr4.Runtime.Misc
 
         private static MethodInfo GetRuleMethod(Type recognizerClass, string name)
         {
+#if DOTNETCORE
+            MethodInfo[] declaredMethods = recognizerClass.GetTypeInfo().GetMethods();
+#else
             MethodInfo[] declaredMethods = recognizerClass.GetMethods();
+#endif
             foreach (MethodInfo method in declaredMethods)
             {
-                if (method.Name.Equals(name) && Attribute.IsDefined(method, typeof(RuleVersionAttribute)))
+#if DOTNETCORE
+                if (method.Name.Equals(name) && method.IsDefined(typeof(RuleVersionAttribute)))
+#else
+                    if (method.Name.Equals(name) && Attribute.IsDefined(method, typeof(RuleVersionAttribute)))
+#endif
                 {
                     return method;
                 }
@@ -845,7 +865,11 @@ namespace Antlr4.Runtime.Misc
 
         private static string[] GetRuleNames(Type recognizerClass)
         {
+#if DOTNETCORE
+            FieldInfo ruleNames = recognizerClass.GetTypeInfo().GetField("ruleNames");
+#else
             FieldInfo ruleNames = recognizerClass.GetField("ruleNames");
+#endif
             return (string[])ruleNames.GetValue(null);
         }
 
@@ -853,20 +877,36 @@ namespace Antlr4.Runtime.Misc
         {
             IList<Tuple<RuleDependencyAttribute, ICustomAttributeProvider>> result = new ArrayList<Tuple<RuleDependencyAttribute, ICustomAttributeProvider>>();
 
+#if DOTNETCORE
+            GetElementDependencies(AsCustomAttributeProvider(clazz.GetTypeInfo()), result);
+#else
             GetElementDependencies(AsCustomAttributeProvider(clazz), result);
+#endif
+#if DOTNETCORE
+            foreach (ConstructorInfo ctor in clazz.GetTypeInfo().GetConstructors(AllDeclaredMembers))
+#else
             foreach (ConstructorInfo ctor in clazz.GetConstructors(AllDeclaredMembers))
+#endif
             {
                 GetElementDependencies(AsCustomAttributeProvider(ctor), result);
                 foreach (ParameterInfo parameter in ctor.GetParameters())
                     GetElementDependencies(AsCustomAttributeProvider(parameter), result);
             }
 
+#if DOTNETCORE
+            foreach (FieldInfo field in clazz.GetTypeInfo().GetFields(AllDeclaredMembers))
+#else
             foreach (FieldInfo field in clazz.GetFields(AllDeclaredMembers))
+#endif
             {
                 GetElementDependencies(AsCustomAttributeProvider(field), result);
             }
 
+#if DOTNETCORE
+            foreach (MethodInfo method in clazz.GetTypeInfo().GetMethods(AllDeclaredMembers))
+#else
             foreach (MethodInfo method in clazz.GetMethods(AllDeclaredMembers))
+#endif
             {
                 GetElementDependencies(AsCustomAttributeProvider(method), result);
 #if COMPACT
@@ -922,12 +962,21 @@ namespace Antlr4.Runtime.Misc
 
         private static string GetSerializedATN(Type recognizerClass)
         {
+#if DOTNETCORE
+            FieldInfo serializedAtnField = recognizerClass.GetTypeInfo().GetField("_serializedATN", AllDeclaredStaticMembers);
+#else
             FieldInfo serializedAtnField = recognizerClass.GetField("_serializedATN", AllDeclaredStaticMembers);
+#endif
             if (serializedAtnField != null)
                 return (string)serializedAtnField.GetValue(null);
 
+#if DOTNETCORE
+            if (recognizerClass.GetTypeInfo().BaseType != null)
+                return GetSerializedATN(recognizerClass.GetTypeInfo().BaseType);
+#else
             if (recognizerClass.BaseType != null)
                 return GetSerializedATN(recognizerClass.BaseType);
+#endif
 
             return null;
         }

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Misc/Utils.cs

@@ -131,19 +131,5 @@ namespace Antlr4.Runtime.Misc
             }
             return m;
         }
-
-        public static char[] ToCharArray(ArrayList<int> data)
-        {
-            if (data == null)
-            {
-                return null;
-            }
-            char[] cdata = new char[data.Count];
-            for (int i = 0; i < data.Count; i++)
-            {
-                cdata[i] = (char)data[i];
-            }
-            return cdata;
-        }
     }
 }

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Parser.cs

@@ -3,6 +3,7 @@
  * can be found in the LICENSE.txt file in the project root.
  */
 using System;
+using System.IO;
 using System.Text;
 using System.Collections.Generic;
 using Antlr4.Runtime.Atn;
@@ -21,14 +22,20 @@ namespace Antlr4.Runtime
 #if !PORTABLE
         public class TraceListener : IParseTreeListener
         {
+            private readonly TextWriter Output;
+
+            public TraceListener(TextWriter output) {
+                Output = output;
+            }
+
             public virtual void EnterEveryRule(ParserRuleContext ctx)
             {
-                System.Console.Out.WriteLine("enter   " + this._enclosing.RuleNames[ctx.RuleIndex] + ", LT(1)=" + this._enclosing._input.LT(1).Text);
+                Output.WriteLine("enter   " + this._enclosing.RuleNames[ctx.RuleIndex] + ", LT(1)=" + this._enclosing._input.LT(1).Text);
             }
 
             public virtual void ExitEveryRule(ParserRuleContext ctx)
             {
-                System.Console.Out.WriteLine("exit    " + this._enclosing.RuleNames[ctx.RuleIndex] + ", LT(1)=" + this._enclosing._input.LT(1).Text);
+                Output.WriteLine("exit    " + this._enclosing.RuleNames[ctx.RuleIndex] + ", LT(1)=" + this._enclosing._input.LT(1).Text);
             }
 
             public virtual void VisitErrorNode(IErrorNode node)
@@ -39,7 +46,7 @@ namespace Antlr4.Runtime
             {
                 ParserRuleContext parent = (ParserRuleContext)((IRuleNode)node.Parent).RuleContext;
                 IToken token = node.Symbol;
-                System.Console.Out.WriteLine("consume " + token + " rule " + this._enclosing.RuleNames[parent.RuleIndex]);
+                Output.WriteLine("consume " + token + " rule " + this._enclosing.RuleNames[parent.RuleIndex]);
             }
 
             internal TraceListener(Parser _enclosing)
@@ -161,9 +168,16 @@ namespace Antlr4.Runtime
         /// </remarks>
         private int _syntaxErrors;
 
-        public Parser(ITokenStream input)
+        protected readonly TextWriter Output;
+        protected readonly TextWriter ErrorOutput;
+
+        public Parser(ITokenStream input) : this(input, Console.Out, Console.Error) { }
+
+        public Parser(ITokenStream input, TextWriter output, TextWriter errorOutput)
         {
             TokenStream = input;
+            Output = output;
+            ErrorOutput = errorOutput;
         }
 
         /// <summary>reset the parser's state</summary>
@@ -676,7 +690,7 @@ namespace Antlr4.Runtime
                 charPositionInLine = offendingToken.Column;
             }
             IAntlrErrorListener<IToken> listener = ((IParserErrorListener)ErrorListenerDispatch);
-            listener.SyntaxError(this, offendingToken, line, charPositionInLine, msg, e);
+            listener.SyntaxError(ErrorOutput, this, offendingToken, line, charPositionInLine, msg, e);
         }
 
         /// <summary>
@@ -1143,10 +1157,10 @@ namespace Antlr4.Runtime
                 {
                     if (seenOne)
                     {
-                        System.Console.Out.WriteLine();
+                        Output.WriteLine();
                     }
-                    System.Console.Out.WriteLine("Decision " + dfa.decision + ":");
+                    Output.WriteLine("Decision " + dfa.decision + ":");
-                    System.Console.Out.Write(dfa.ToString(Vocabulary));
+                    Output.Write(dfa.ToString(Vocabulary));
                     seenOne = true;
                 }
             }

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/ParserRuleContext.cs

@@ -116,7 +116,18 @@ namespace Antlr4.Runtime
             }
         }
 
-        /// <summary>COPY a ctx (I'm deliberately not using copy constructor)</summary>
+        /// <summary>
+        /// COPY a ctx (I'm deliberately not using copy constructor) to avoid
+        /// confusion with creating node with parent. Does not copy children.
+        /// 
+        /// This is used in the generated parser code to flip a generic XContext
+        /// node for rule X to a YContext for alt label Y. In that sense, it is
+        /// not really a generic copy function.
+        /// 
+        /// If we do an error sync() at start of a rule, we might add error nodes
+        /// to the generic XContext so this function must copy those nodes to
+        /// the YContext as well else they are lost!
+        /// </summary>
         public virtual void CopyFrom(Antlr4.Runtime.ParserRuleContext ctx)
         {
             // from RuleContext
@@ -124,6 +135,22 @@ namespace Antlr4.Runtime
             this.invokingState = ctx.invokingState;
             this._start = ctx._start;
             this._stop = ctx._stop;
+
+            // copy any error nodes to alt label node
+            if (ctx.children != null)
+            {
+                children = new List<IParseTree>();
+                // reset parent pointer for any error nodes
+                foreach (var child in ctx.children)
+                {
+                    var errorChildNode = child as ErrorNodeImpl;
+                    if (errorChildNode != null)
+                    {
+                        children.Add(errorChildNode);
+                        errorChildNode.Parent = this;
+                    }
+                }
+            }
         }
 
         public ParserRuleContext(Antlr4.Runtime.ParserRuleContext parent, int invokingStateNumber)
@@ -252,7 +279,7 @@ namespace Antlr4.Runtime
             return null;
         }
 
-#if NET45PLUS
+#if (NET45PLUS && !DOTNETCORE)
         public virtual IReadOnlyList<ITerminalNode> GetTokens(int ttype)
 #else
         public virtual ITerminalNode[] GetTokens(int ttype)
@@ -283,7 +310,7 @@ namespace Antlr4.Runtime
             {
                 return Collections.EmptyList<ITerminalNode>();
             }
-#if NET45PLUS
+#if (NET45PLUS && !DOTNETCORE)
             return tokens;
 #else
             return tokens.ToArray();
@@ -296,7 +323,7 @@ namespace Antlr4.Runtime
             return GetChild<T>(i);
         }
 
-#if NET45PLUS
+#if (NET45PLUS && !DOTNETCORE)
         public virtual IReadOnlyList<T> GetRuleContexts<T>()
             where T : Antlr4.Runtime.ParserRuleContext
 #else
@@ -324,7 +351,7 @@ namespace Antlr4.Runtime
             {
                 return Collections.EmptyList<T>();
             }
-#if NET45PLUS
+#if (NET45PLUS && !DOTNETCORE)
             return contexts;
 #else
             return contexts.ToArray();

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs

@@ -42,8 +42,8 @@ using System.Runtime.InteropServices;
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("4.6.0")]
+[assembly: AssemblyVersion("4.6.1")]
 #if !COMPACT
-[assembly: AssemblyFileVersion("4.6.0")]
+[assembly: AssemblyFileVersion("4.6.1")]
-[assembly: AssemblyInformationalVersion("4.6.0")]
+[assembly: AssemblyInformationalVersion("4.6.1")]
 #endif

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/ProxyErrorListener.cs

@@ -4,6 +4,7 @@
  */
 using System;
 using System.Collections.Generic;
+using System.IO;
 namespace Antlr4.Runtime
 {
     /// <summary>
@@ -35,11 +36,11 @@ namespace Antlr4.Runtime
             }
         }
 
-        public virtual void SyntaxError(IRecognizer recognizer, Symbol offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e)
+        public virtual void SyntaxError(TextWriter output, IRecognizer recognizer, Symbol offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e)
         {
             foreach (IAntlrErrorListener<Symbol> listener in delegates)
             {
-                listener.SyntaxError(recognizer, offendingSymbol, line, charPositionInLine, msg, e);
+                listener.SyntaxError(output, recognizer, offendingSymbol, line, charPositionInLine, msg, e);
             }
         }
     }

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Sharpen/Compat/SerializableAttribute.cs

@@ -1,9 +1,9 @@
-/* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
+/* Copyright (c) 2012-2016 The ANTLR Project. All rights reserved.
  * Use of this file is governed by the BSD 3-clause license that
  * can be found in the LICENSE.txt file in the project root.
  */
 
-#if PORTABLE
+#if PORTABLE || DOTNETCORE
 
 namespace System
 {
@@ -14,3 +14,4 @@ namespace System
 }
 
 #endif
+

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Sharpen/Compat/SplitOrderedList.cs

@@ -28,7 +28,6 @@ using System;
 using System.Threading;
 using System.Collections;
 using System.Collections.Generic;
-using System.Runtime.Serialization;
 
 namespace Antlr4.Runtime.Sharpen
 {

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Tree/ParseTreeProperty.cs

@@ -2,7 +2,11 @@
  * Use of this file is governed by the BSD 3-clause license that
  * can be found in the LICENSE.txt file in the project root.
  */
+#if NET40PLUS
+using System.Collections.Concurrent;
+#else
 using Antlr4.Runtime.Sharpen;
+#endif
 
 namespace Antlr4.Runtime.Tree
 {

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Tree/Pattern/ParseTreePatternMatcher.cs

@@ -138,7 +138,7 @@ namespace Antlr4.Runtime.Tree.Pattern
             {
             }
         }
-
+        
         [System.Serializable]
         public class StartRuleDoesNotConsumeFullPattern : Exception
         {

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathLexer.cs

@@ -1,115 +1,151 @@
 //------------------------------------------------------------------------------
 // <auto-generated>
 //     This code was generated by a tool.
-//     ANTLR Version: 4.4.1-dev
+//     ANTLR Version: 4.6.1
 //
 //     Changes to this file may cause incorrect behavior and will be lost if
 //     the code is regenerated.
 // </auto-generated>
 //------------------------------------------------------------------------------
 
-// Generated from /Users/ericvergnaud/Development/antlr4/antlr/antlr4-csharp/runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathLexer.g4 by ANTLR 4.4.1-dev
-
 // Unreachable code detected
 #pragma warning disable 0162
 // The variable '...' is assigned but its value is never used
 #pragma warning disable 0219
 // Missing XML comment for publicly visible type or member '...'
 #pragma warning disable 1591
+// Ambiguous reference in cref attribute
+#pragma warning disable 419
 
 
-using System; 
+using System;
 
+using System;
+using System.Text;
 using Antlr4.Runtime;
 using Antlr4.Runtime.Atn;
 using Antlr4.Runtime.Misc;
 using DFA = Antlr4.Runtime.Dfa.DFA;
 
+[System.CodeDom.Compiler.GeneratedCode("ANTLR", "4.6.1")]
 [System.CLSCompliant(false)]
-public partial class XPathLexer : Lexer {
+public partial class XPathLexer : Lexer
-	public const int
+{
-		TokenRef=1, RuleRef=2, Anywhere=3, Root=4, Wildcard=5, Bang=6, ID=7, String=8;
+    protected static DFA[] decisionToDFA;
-	public static string[] modeNames = {
+    protected static PredictionContextCache sharedContextCache = new PredictionContextCache();
+    public const int
+        TokenRef = 1, RuleRef = 2, Anywhere = 3, Root = 4, Wildcard = 5, Bang = 6, ID = 7, String = 8;
+    public static string[] channelNames = {
+		"DEFAULT_TOKEN_CHANNEL", "HIDDEN"
+	};
+
+    public static string[] modeNames = {
 		"DEFAULT_MODE"
 	};
 
-	public static readonly string[] ruleNames = {
+    public static readonly string[] ruleNames = {
 		"Anywhere", "Root", "Wildcard", "Bang", "ID", "NameChar", "NameStartChar", 
 		"String"
 	};
 
 
-	public XPathLexer(ICharStream input)
+    public XPathLexer(ICharStream input)
-		: base(input)
+        : base(input)
-	{
+    {
-		Interpreter = new LexerATNSimulator(this, _ATN, null, null);
+        Interpreter = new LexerATNSimulator(this, _ATN, decisionToDFA, sharedContextCache);
-	}
+    }
 
-	private static readonly string[] _LiteralNames = {
+    private static readonly string[] _LiteralNames = {
 		null, null, null, "'//'", "'/'", "'*'", "'!'"
 	};
-	private static readonly string[] _SymbolicNames = {
+    private static readonly string[] _SymbolicNames = {
 		null, "TokenRef", "RuleRef", "Anywhere", "Root", "Wildcard", "Bang", "ID", 
 		"String"
 	};
-	public static readonly IVocabulary DefaultVocabulary = new Vocabulary(_LiteralNames, _SymbolicNames);
+    public static readonly IVocabulary DefaultVocabulary = new Vocabulary(_LiteralNames, _SymbolicNames);
 
-	[NotNull]
+    [NotNull]
-	public override IVocabulary Vocabulary
+    public override IVocabulary Vocabulary
-	{
+    {
-		get
+        get
-		{
+        {
-			return DefaultVocabulary;
+            return DefaultVocabulary;
-		}
+        }
-	}
+    }
 
-	public override string GrammarFileName { get { return "XPathLexer.g4"; } }
+    public override string GrammarFileName { get { return "XPathLexer.g4"; } }
 
-	public override string[] RuleNames { get { return ruleNames; } }
+    public override string[] RuleNames { get { return ruleNames; } }
 
-	public override string[] ModeNames { get { return modeNames; } }
+    public override string[] ChannelNames { get { return channelNames; } }
 
-	public override string SerializedAtn { get { return _serializedATN; } }
+    public override string[] ModeNames { get { return modeNames; } }
+
+    public override string SerializedAtn { get { return _serializedATN; } }
+
+    static XPathLexer()
+    {
+        decisionToDFA = new DFA[_ATN.NumberOfDecisions];
+        for (int i = 0; i < _ATN.NumberOfDecisions; i++)
+        {
+            decisionToDFA[i] = new DFA(_ATN.GetDecisionState(i), i);
+        }
+    }
+    public override void Action(RuleContext _localctx, int ruleIndex, int actionIndex)
+    {
+        switch (ruleIndex)
+        {
+            case 4: ID_action(_localctx, actionIndex); break;
+        }
+    }
+    private void ID_action(RuleContext _localctx, int actionIndex)
+    {
+        switch (actionIndex)
+        {
+            case 0:
+                String text = Text;
+                if (Char.IsUpper(text[0]))
+                    Type = TokenRef;
+                else
+                    Type = RuleRef;
+                break;
+        }
+    }
+
+    private static string _serializedATN = _serializeATN();
+    private static string _serializeATN()
+    {
+        StringBuilder sb = new StringBuilder();
+        sb.Append("\x3\x430\xD6D1\x8206\xAD2D\x4417\xAEF1\x8D80\xAADD\x2\n\x34");
+        sb.Append("\b\x1\x4\x2\t\x2\x4\x3\t\x3\x4\x4\t\x4\x4\x5\t\x5\x4\x6\t\x6");
+        sb.Append("\x4\a\t\a\x4\b\t\b\x4\t\t\t\x3\x2\x3\x2\x3\x2\x3\x3\x3\x3\x3");
+        sb.Append("\x4\x3\x4\x3\x5\x3\x5\x3\x6\x3\x6\a\x6\x1F\n\x6\f\x6\xE\x6\"");
+        sb.Append("\v\x6\x3\x6\x3\x6\x3\a\x3\a\x5\a(\n\a\x3\b\x3\b\x3\t\x3\t\a");
+        sb.Append("\t.\n\t\f\t\xE\t\x31\v\t\x3\t\x3\t\x3/\x2\n\x3\x5\x5\x6\a\a");
+        sb.Append("\t\b\v\t\r\x2\xF\x2\x11\n\x3\x2\x4\a\x2\x32;\x61\x61\xB9\xB9");
+        sb.Append("\x302\x371\x2041\x2042\xF\x2\x43\\\x63|\xC2\xD8\xDA\xF8\xFA");
+        sb.Append("\x301\x372\x37F\x381\x2001\x200E\x200F\x2072\x2191\x2C02\x2FF1");
+        sb.Append("\x3003\xD801\xF902\xFDD1\xFDF2\xFFFF\x34\x2\x3\x3\x2\x2\x2\x2");
+        sb.Append("\x5\x3\x2\x2\x2\x2\a\x3\x2\x2\x2\x2\t\x3\x2\x2\x2\x2\v\x3\x2");
+        sb.Append("\x2\x2\x2\x11\x3\x2\x2\x2\x3\x13\x3\x2\x2\x2\x5\x16\x3\x2\x2");
+        sb.Append("\x2\a\x18\x3\x2\x2\x2\t\x1A\x3\x2\x2\x2\v\x1C\x3\x2\x2\x2\r");
+        sb.Append("\'\x3\x2\x2\x2\xF)\x3\x2\x2\x2\x11+\x3\x2\x2\x2\x13\x14\a\x31");
+        sb.Append("\x2\x2\x14\x15\a\x31\x2\x2\x15\x4\x3\x2\x2\x2\x16\x17\a\x31");
+        sb.Append("\x2\x2\x17\x6\x3\x2\x2\x2\x18\x19\a,\x2\x2\x19\b\x3\x2\x2\x2");
+        sb.Append("\x1A\x1B\a#\x2\x2\x1B\n\x3\x2\x2\x2\x1C \x5\xF\b\x2\x1D\x1F");
+        sb.Append("\x5\r\a\x2\x1E\x1D\x3\x2\x2\x2\x1F\"\x3\x2\x2\x2 \x1E\x3\x2");
+        sb.Append("\x2\x2 !\x3\x2\x2\x2!#\x3\x2\x2\x2\" \x3\x2\x2\x2#$\b\x6\x2");
+        sb.Append("\x2$\f\x3\x2\x2\x2%(\x5\xF\b\x2&(\t\x2\x2\x2\'%\x3\x2\x2\x2");
+        sb.Append("\'&\x3\x2\x2\x2(\xE\x3\x2\x2\x2)*\t\x3\x2\x2*\x10\x3\x2\x2\x2");
+        sb.Append("+/\a)\x2\x2,.\v\x2\x2\x2-,\x3\x2\x2\x2.\x31\x3\x2\x2\x2/\x30");
+        sb.Append("\x3\x2\x2\x2/-\x3\x2\x2\x2\x30\x32\x3\x2\x2\x2\x31/\x3\x2\x2");
+        sb.Append("\x2\x32\x33\a)\x2\x2\x33\x12\x3\x2\x2\x2\x6\x2 \'/\x3\x3\x6");
+        sb.Append("\x2");
+        return sb.ToString();
+    }
+
+    public static readonly ATN _ATN =
+        new ATNDeserializer().Deserialize(_serializedATN.ToCharArray());
 
-	public override void Action(RuleContext _localctx, int ruleIndex, int actionIndex) {
-		switch (ruleIndex) {
-		case 4 : ID_action(_localctx, actionIndex); break;
-		}
-	}
-	private void ID_action(RuleContext _localctx, int actionIndex) {
-		switch (actionIndex) {
-		case 0: 
-						String text = Text;
-						if ( Char.IsUpper(text[0]) ) 
-							Type = TokenRef;
-						else 
-							Type = RuleRef;
-						 break;
-		}
-	}
 
-	public static readonly string _serializedATN =
-		"\x3\x430\xD6D1\x8206\xAD2D\x4417\xAEF1\x8D80\xAADD\x2\n\x34\b\x1\x4\x2"+
-		"\t\x2\x4\x3\t\x3\x4\x4\t\x4\x4\x5\t\x5\x4\x6\t\x6\x4\a\t\a\x4\b\t\b\x4"+
-		"\t\t\t\x3\x2\x3\x2\x3\x2\x3\x3\x3\x3\x3\x4\x3\x4\x3\x5\x3\x5\x3\x6\x3"+
-		"\x6\a\x6\x1F\n\x6\f\x6\xE\x6\"\v\x6\x3\x6\x3\x6\x3\a\x3\a\x5\a(\n\a\x3"+
-		"\b\x3\b\x3\t\x3\t\a\t.\n\t\f\t\xE\t\x31\v\t\x3\t\x3\t\x3/\x2\n\x3\x5\x5"+
-		"\x6\a\a\t\b\v\t\r\x2\xF\x2\x11\n\x3\x2\x4\a\x2\x32;\x61\x61\xB9\xB9\x302"+
-		"\x371\x2041\x2042\xF\x2\x43\\\x63|\xC2\xD8\xDA\xF8\xFA\x301\x372\x37F"+
-		"\x381\x2001\x200E\x200F\x2072\x2191\x2C02\x2FF1\x3003\xD801\xF902\xFDD1"+
-		"\xFDF2\xFFFF\x34\x2\x3\x3\x2\x2\x2\x2\x5\x3\x2\x2\x2\x2\a\x3\x2\x2\x2"+
-		"\x2\t\x3\x2\x2\x2\x2\v\x3\x2\x2\x2\x2\x11\x3\x2\x2\x2\x3\x13\x3\x2\x2"+
-		"\x2\x5\x16\x3\x2\x2\x2\a\x18\x3\x2\x2\x2\t\x1A\x3\x2\x2\x2\v\x1C\x3\x2"+
-		"\x2\x2\r\'\x3\x2\x2\x2\xF)\x3\x2\x2\x2\x11+\x3\x2\x2\x2\x13\x14\a\x31"+
-		"\x2\x2\x14\x15\a\x31\x2\x2\x15\x4\x3\x2\x2\x2\x16\x17\a\x31\x2\x2\x17"+
-		"\x6\x3\x2\x2\x2\x18\x19\a,\x2\x2\x19\b\x3\x2\x2\x2\x1A\x1B\a#\x2\x2\x1B"+
-		"\n\x3\x2\x2\x2\x1C \x5\xF\b\x2\x1D\x1F\x5\r\a\x2\x1E\x1D\x3\x2\x2\x2\x1F"+
-		"\"\x3\x2\x2\x2 \x1E\x3\x2\x2\x2 !\x3\x2\x2\x2!#\x3\x2\x2\x2\" \x3\x2\x2"+
-		"\x2#$\b\x6\x2\x2$\f\x3\x2\x2\x2%(\x5\xF\b\x2&(\t\x2\x2\x2\'%\x3\x2\x2"+
-		"\x2\'&\x3\x2\x2\x2(\xE\x3\x2\x2\x2)*\t\x3\x2\x2*\x10\x3\x2\x2\x2+/\a)"+
-		"\x2\x2,.\v\x2\x2\x2-,\x3\x2\x2\x2.\x31\x3\x2\x2\x2/\x30\x3\x2\x2\x2/-"+
-		"\x3\x2\x2\x2\x30\x32\x3\x2\x2\x2\x31/\x3\x2\x2\x2\x32\x33\a)\x2\x2\x33"+
-		"\x12\x3\x2\x2\x2\x6\x2 \'/\x3\x3\x6\x2";
-	public static readonly ATN _ATN =
-		new ATNDeserializer().Deserialize(_serializedATN.ToCharArray());
 }