Merge branch 'master' into patch-1

.gitignore

@@ -3,7 +3,7 @@ target/
 # ... but not code generation targets
 !tool/src/org/antlr/v4/codegen/target/
 
-# Node.js (npm and typings) cached dependencies 
+# Node.js (npm and typings) cached dependencies
 node_modules/
 typings/
 
@@ -98,3 +98,6 @@ xcuserdata
 javac-services.0.log
 javac-services.0.log.lck
 test/
+
+# Don't ignore python tests
+!runtime/Python3/test/

.travis.yml

@@ -203,6 +203,9 @@ matrix:
       jdk: openjdk8
       env: TARGET=javascript
       stage: main-test
+      before_install:
+        - nvm install 14 # otherwise it runs by default on node 8
+        - f="./.travis/before-install-linux-javascript.sh"; ! [ -x "$f" ] || "$f"
     - os: linux
       dist: trusty
       jdk: openjdk8
@@ -215,7 +218,7 @@ before_install:
 script:
   - |
     cd runtime-testsuite;
-    travis_wait 40 ../.travis/run-tests-$TARGET.sh;
+    travis_wait 40 ../.travis/run-tests-$TARGET.sh
-    rc=$?;
+    rc=$?
-    cat target/surefire-reports/*.dumpstream || true;
+    cat target/surefire-reports/*.dumpstream || true
     exit $rc

.travis/before-install-linux-javascript.sh

@@ -2,7 +2,6 @@
 
 set -euo pipefail
 
-sudo apt-get update -qq
+# use v14 and check
-curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash -
+echo node version: $(node --version)
-sudo apt-get install -qq nodejs
+
-node --version

.travis/run-tests-javascript.sh

@@ -2,4 +2,8 @@
 
 set -euo pipefail
 
-mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=javascript.* test
+cd ../runtime/JavaScript
+npm install
+npm link
+cd ../../runtime-testsuite
+mvn -q -Dparallel=methods -DthreadCount=1 -Dtest=javascript.* test

appveyor.yml

@@ -16,7 +16,7 @@ build_script:
 after_build: 
   - msbuild /target:pack /property:Configuration=Release /verbosity:detailed runtime/CSharp/runtime/CSharp/Antlr4.dotnet.sln
 test_script:
-  - mvn install -Dantlr-php-php="C:\tools\php\php.exe" -Dantlr-dart-dart="C:\tools\dart-sdk\bin\dart.exe" -Dantlr-dart-pub="C:\tools\dart-sdk\bin\pub.bat" -Dantlr-dart-dart2native="C:\tools\dart-sdk\bin\dart2native.bat" -Dantlr-python2-python="C:\Python27\python.exe" -Dantlr-python3-python="C:\Python35\python.exe" -Dantlr-javascript-nodejs="C:\Program Files (x86)\nodejs\node.exe" --batch-mode
+  - mvn install -Dantlr-php-php="C:\tools\php\php.exe" -Dantlr-dart-dart="C:\tools\dart-sdk\bin\dart.exe" -Dantlr-dart-pub="C:\tools\dart-sdk\bin\pub.bat" -Dantlr-dart-dart2native="C:\tools\dart-sdk\bin\dart2native.bat" -Dantlr-python2-python="C:\Python27\python.exe" -Dantlr-python3-python="C:\Python35\python.exe" --batch-mode
 artifacts:
 - path: 'runtime\**\*.nupkg'  
   name: NuGet

contributors.txt

@@ -208,6 +208,7 @@ YYYY/MM/DD, github id, Full name, email
 2018/07/03, jgoppert, James Goppert, james.goppert@gmail.com
 2018/07/27, Maksim Novikov, mnovikov.work@gmail.com
 2018/08/03, ENDOH takanao, djmchl@gmail.com
+2018/10/08, xsIceman, Andreas Skaar, andreas.skaar@gmail.com
 2018/10/18, edirgarcia, Edir García Lazo, edirgl@hotmail.com
 2018/07/31, Lucas Henrqiue, lucashenrique580@gmail.com
 2018/08/03, ENDOH takanao, djmchl@gmail.com
@@ -218,6 +219,7 @@ YYYY/MM/DD, github id, Full name, email
 2018/11/29, hannemann-tamas, Ralf Hannemann-Tamas, ralf.ht@gmail.com
 2018/12/20, WalterCouto, Walter Couto, WalterCouto@users.noreply.github.com
 2018/12/23, youkaichao, Kaichao You, youkaichao@gmail.com
+2019/01/02, wkhemir, Wail Khemir, khemirwail@gmail.com
 2019/01/16, kuegi, Markus Zancolo, markus.zancolo@roomle.com
 2019/02/06, ralucado, Cristina Raluca Vijulie, ralucris.v[at]gmail[dot]com
 2019/02/23, gedimitr, Gerasimos Dimitriadis, gedimitr@gmail.com
@@ -247,7 +249,11 @@ YYYY/MM/DD, github id, Full name, email
 2020/02/21, StochasticTinkr, Daniel Pitts, github@coloraura.com
 2020/03/17, XsongyangX, Song Yang, songyang1218@gmail.com
 2020/04/07, deniskyashif, Denis Kyashif, denis.kyashif@gmail.com
+2020/04/23, martinvw, Martin van Wingerden, martin@martinvw.nl
 2020/04/30, TristonianJones, Tristan Swadell, tswadell@google.com
+2020/05/06, iammosespaulr, Moses Paul R, iammosespaulr@gmail.com
+2020/05/10, gomerser, Erik Gomersbach, gomerser@gomersba.ch
+2020/06/04, sigmasoldi3r, Pablo Blanco, pablobc.1995@gmail.com
 2020/05/25, graknol, Sindre van der Linden, graknol@gmail.com
 2020/05/31, d-markey, David Markey, dmarkey@free.fr
 2020/06/02, cohomology, Kilian Kilger, kkilger AT gmail.com
@@ -256,3 +262,11 @@ YYYY/MM/DD, github id, Full name, email
 2020/08/22, stevenjohnstone, Steven Johnstone, steven.james.johnstone@gmail.com
 2020/09/06, ArthurSonzogni, Sonzogni Arthur, arthursonzogni@gmail.com
 2020/09/10, Khailian, Arunav Sanyal, arunav.sanyal91@gmail.com
+2020/09/12, Clcanny, Charles Ruan, a837940593@gmail.com
+2020/09/15, rmcgregor1990, Robert McGregor, rmcgregor1990@gmail.com
+2020/09/16, trenki2, Markus Trenkwalder, trenki2[at]gmx[dot]net
+2020/10/08, Marti2203, Martin Mirchev, mirchevmartin2203@gmail.com
+2020/10/16, adarshbhat, Adarsh Bhat, adarshbhat@users.noreply.github.com
+2020/10/20, adamwojs, Adam Wójs, adam[at]wojs.pl
+2020/10/24, cliid, Jiwu Jang, jiwujang@naver.com
+2020/11/05, MichelHartmann, Michel Hartmann, MichelHartmann@users.noreply.github.com

doc/cpp-target.md

@@ -1,6 +1,6 @@
 # C++
 
-The C++ target supports all platforms that can either run MS Visual Studio 2013 (or newer), XCode 7 (or newer) or CMake (C++11 required). All build tools can either create static or dynamic libraries, both as 64bit or 32bit arch. Additionally, XCode can create an iOS library. Also see [Antlr4 for C++ with CMake: A practical example](http://blorente.me//Antlr-,-C++-and-CMake-Wait-what.html).
+The C++ target supports all platforms that can either run MS Visual Studio 2013 (or newer), XCode 7 (or newer) or CMake (C++11 required). All build tools can either create static or dynamic libraries, both as 64bit or 32bit arch. Additionally, XCode can create an iOS library. Also see [Antlr4 for C++ with CMake: A practical example](https://beyondtheloop.dev/Antlr-cpp-cmake/).
 
 ## How to create a C++ lexer or parser?
 This is pretty much the same as creating a Java lexer or parser, except you need to specify the language target, for example:

doc/csharp-target.md

@@ -86,7 +86,7 @@ In order to execute this listener, you would simply add the following lines to t
 ...
 IParseTree tree = parser.StartRule() - only repeated here for reference
 KeyPrinter printer = new KeyPrinter();
-ParseTreeWalker.DEFAULT.walk(printer, tree);
+ParseTreeWalker.Default.Walk(printer, tree);
 ```
         
 Further information can be found from The Definitive ANTLR Reference book.

doc/faq/index.md

@@ -38,8 +38,8 @@ This is the main landing page for the ANTLR 4 FAQ. The links below will take you
 
 ## Translation
 
-* [ASTs vs parse trees](parse-trees.md)
+* [ASTs vs parse trees](translation.md)
-* [Decoupling input walking from output generation](parse-trees.md)
+* [Decoupling input walking from output generation](translation.md)
 
 ## Actions and semantic predicates
 

doc/faq/parse-trees.md

@@ -50,7 +50,7 @@ For writing a compiler, either generate [LLVM-type static-single-assignment](htt
 
 ### XPath
 
-XPath works great when you need to find specific nodes, possibly in certain contexts. The context is limited to the parents on the way to the root of the tree. For example, if you want to find all ID nodes, use path `//ID`. If you want all variable declarations, you might use path `//vardecl`.  If you only want fields declarations, then you can use some context information via path `/classdef/vardecl`, which would only find vardecls that our children of class definitions. You can merge the results of multiple XPath `findAll()`s simulating a set union for XPath. The only caveat is that the order from the original tree is not preserved when you union multiple `findAll()` sets.
+XPath works great when you need to find specific nodes, possibly in certain contexts. The context is limited to the parents on the way to the root of the tree. For example, if you want to find all ID nodes, use path `//ID`. If you want all variable declarations, you might use path `//vardecl`.  If you only want fields declarations, then you can use some context information via path `/classdef/vardecl`, which would only find vardecls that are children of class definitions. You can merge the results of multiple XPath `findAll()`s simulating a set union for XPath. The only caveat is that the order from the original tree is not preserved when you union multiple `findAll()` sets.
 
 ### Tree pattern matching
 
@@ -70,4 +70,4 @@ scopeStack.peek().define(new VariableSymbol("foo"))
 
 That way each listener function does not have to compute its appropriate scope.
 
-Examples: [DefScopesAndSymbols.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/DefScopesAndSymbols.java) and [SetScopeListener.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/SetScopeListener.java) and [VerifyListener.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/VerifyListener.java)
+Examples: [DefScopesAndSymbols.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/DefScopesAndSymbols.java) and [SetScopeListener.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/SetScopeListener.java) and [VerifyListener.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/VerifyListener.java)

doc/getting-started.md

@@ -8,7 +8,7 @@ ANTLR is really two things: a tool that translates your grammar to a parser/lexe
 
 The first thing you should do is probably download and install a development tool plug-in. Even if you only use such tools for editing, they are great. Then, follow the instructions below to get the runtime environment available to your system to run generated parsers/lexers.  In what follows, I talk about antlr-4.7.1-complete.jar, which has the tool and the runtime and any other support libraries (e.g., ANTLR v4 is written in v3).
 
-If you are going to integrate ANTLR into your existing build system using mvn, ant, or want to get ANTLR into your IDE such as eclipse or intellij, see Integrating ANTLR into Development Systems.
+If you are going to integrate ANTLR into your existing build system using mvn, ant, or want to get ANTLR into your IDE such as eclipse or intellij, see [Integrating ANTLR into Development Systems](https://github.com/antlr/antlr4/blob/master/doc/IDEs.md).
 
 ### UNIX
 

doc/javascript-target.md

@@ -11,11 +11,18 @@ In practice, this target has been extensively tested against:
 * Chrome 39.0.2171
 * Explorer 11.0.3
  
-The tests were conducted using Selenium. No issue was found, so you should find that the runtime works pretty much against any recent JavaScript engine.
+The above tests were conducted using Selenium. No issue was found, so you should find that the runtime works pretty much against any recent JavaScript engine.
 
 ## Is NodeJS supported?
 
-The runtime has also been extensively tested against Node.js 10 LTS. No issue was found.
+The runtime has also been extensively tested against Node.js 14 LTS. No issue was found.
+NodeJS together with a packaging tool is now the preferred development path, developers are encouraged to follow it.
+
+## What about modules?
+
+Starting with version 8.1, Antlr4 JavaScript runtime follows esm semantics (see https://tc39.es/ecma262/#sec-modules for details)
+Generated lexers, parsers, listeners and visitors also follow this new standard.
+If you have used previous versions of the runtime, you will need to migrate and make your parser a module.
 
 ## How to create a JavaScript lexer or parser?
 
@@ -61,7 +68,10 @@ The steps to create your parsing code are the following:
  
 You are now ready to bundle your parsing code as follows:
  - following webpack specs, create a webpack.config file
- - in the `webpack.config` file, exclude node.js only modules using: `node: { module: "empty", net: "empty", fs: "empty" }`
+ - For Webpack version 5,
+   - in the `webpack.config` file, exclude node.js only modules using: `resolve: { fallback: { fs: false } }`
+ - For older versions of Webpack,
+   - in the `webpack.config` file, exclude node.js only modules using: `node: { module: "empty", net: "empty", fs: "empty" }`
  - from the cmd line, navigate to the directory containing webpack.config and type: webpack
  
 This will produce a single js file containing all your parsing code. Easy to include in your web pages!
@@ -80,18 +90,18 @@ Let's suppose that your grammar is named, as above, "MyGrammar". Let's suppose t
 Now a fully functioning script might look like the following:
 
 ```javascript
-   var antlr4 = require('antlr4');
+   import antlr4 from 'antlr4';
-   var MyGrammarLexer = require('./MyGrammarLexer').MyGrammarLexer;
+   import MyGrammarLexer from './MyGrammarLexer.js');
-   var MyGrammarParser = require('./MyGrammarParser').MyGrammarParser;
+   import MyGrammarParser from './MyGrammarParser.js';
-   var MyGrammarListener = require('./MyGrammarListener').MyGrammarListener;
+   import MyGrammarListener from './MyGrammarListener.js';
 
-   var input = "your text to parse here"
+   const input = "your text to parse here"
-   var chars = new antlr4.InputStream(input);
+   const chars = new antlr4.InputStream(input);
-   var lexer = new MyGrammarLexer(chars);
+   const lexer = new MyGrammarLexer(chars);
-   var tokens  = new antlr4.CommonTokenStream(lexer);
+   const tokens  = new antlr4.CommonTokenStream(lexer);
-   var parser = new MyGrammarParser(tokens);
+   const parser = new MyGrammarParser(tokens);
    parser.buildParseTrees = true;
-   var tree = parser.MyStartRule();
+   const tree = parser.MyStartRule();
 ```
 
 This program will work. But it won't be useful unless you do one of the following:
@@ -105,19 +115,19 @@ This program will work. But it won't be useful unless you do one of the followin
 ## How do I create and run a visitor?
 ```javascript
 // test.js
-var antlr4 = require('antlr4');
+import antlr4 from 'antlr4';
-var MyGrammarLexer = require('./QueryLexer').QueryLexer;
+import MyGrammarLexer from './QueryLexer.js';
-var MyGrammarParser = require('./QueryParser').QueryParser;
+import MyGrammarParser from './QueryParser.js';
-var MyGrammarListener = require('./QueryListener').QueryListener;
+import MyGrammarListener from './QueryListener.js';
 
 
-var input = "field = 123 AND items in (1,2,3)"
+const input = "field = 123 AND items in (1,2,3)"
-var chars = new antlr4.InputStream(input);
+const chars = new antlr4.InputStream(input);
-var lexer = new MyGrammarLexer(chars);
+const lexer = new MyGrammarLexer(chars);
-var tokens = new antlr4.CommonTokenStream(lexer);
+const tokens = new antlr4.CommonTokenStream(lexer);
-var parser = new MyGrammarParser(tokens);
+const parser = new MyGrammarParser(tokens);
 parser.buildParseTrees = true;
-var tree = parser.query();
+const tree = parser.query();
 
 class Visitor {
   visitChildren(ctx) {
@@ -145,40 +155,37 @@ tree.accept(new Visitor());
 Let's suppose your MyGrammar grammar comprises 2 rules: "key" and "value". The antlr4 tool will have generated the following listener: 
 
 ```javascript
-   MyGrammarListener = function(ParseTreeListener) {
+class MyGrammarListener extends ParseTreeListener {
-       // some code here
+       
-   }
+    constructor() {
-   // some code here
+        super();
-   MyGrammarListener.prototype.enterKey = function(ctx) {};
+    }
-   MyGrammarListener.prototype.exitKey = function(ctx) {};
+   
-   MyGrammarListener.prototype.enterValue = function(ctx) {};
+   enterKey(ctx) {}
-   MyGrammarListener.prototype.exitValue = function(ctx) {};
+   exitKey(ctx) {}
+   enterValue(ctx) {}
+   exitValue(ctx) {}
+}
 ```
 
 In order to provide custom behavior, you might want to create the following class:
 
 ```javascript
-var KeyPrinter = function() {
+class KeyPrinter extends MyGrammarListener {
-    MyGrammarListener.call(this); // inherit default listener
-    return this;
-};
 
-// continue inheriting default listener
+    // override default listener behavior
-KeyPrinter.prototype = Object.create(MyGrammarListener.prototype);
+    exitKey(ctx) {
-KeyPrinter.prototype.constructor = KeyPrinter;
+        console.log("Oh, a key!");
-
+    }
-// override default listener behavior
+}
-KeyPrinter.prototype.exitKey = function(ctx) {
-    console.log("Oh, a key!");
-};
 ```
 
 In order to execute this listener, you would simply add the following lines to the above code:
 
 ```javascript
-    ...
+...
-    tree = parser.StartRule() // only repeated here for reference
+tree = parser.StartRule() // only repeated here for reference
-var printer = new KeyPrinter();
+const printer = new KeyPrinter();
 antlr4.tree.ParseTreeWalker.DEFAULT.walk(printer, tree);
 ```
 

doc/lexicon.md

@@ -96,7 +96,7 @@ The recognizers that ANTLR generates assume a character vocabulary containing al
 
 ## Actions
 
-Actions are code blocks written in the target language. You can use actions in a number of places within a grammar, but the syntax is always the same: arbitrary text surrounded by curly braces. You don’t need to escape a closing curly character if it’s in a string or comment: `"}"` or `/*}*/`. If the curlies are balanced, you also don’t need to escape }: `{...}`. Otherwise, escape extra curlies with a backslash: `\{` or `\}`. The action text should conform to the target language as specified with thelanguage option.
+Actions are code blocks written in the target language. You can use actions in a number of places within a grammar, but the syntax is always the same: arbitrary text surrounded by curly braces. You don’t need to escape a closing curly character if it’s in a string or comment: `"}"` or `/*}*/`. If the curlies are balanced, you also don’t need to escape }: `{...}`. Otherwise, escape extra curlies with a backslash: `\{` or `\}`. The action text should conform to the target language as specified with the language option.
 
 Embedded code can appear in: `@header` and `@members` named actions, parser and lexer rules, exception catching specifications, attribute sections for parser rules (return values, arguments, and locals), and some rule element options (currently predicates).
 

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Chrome.test.stg

@@ -74,7 +74,7 @@ LANotEquals(i, v) ::= <%this._input.LA(<i>)!=<v>%>
 
 TokenStartColumnEquals(i) ::= <%this._tokenStartColumn===<i>%>
 
-ImportListener(X) ::= <<var <X>Listener = require('./<X>Listener').<X>Listener;>>
+ImportListener(X) ::= ""
 
 GetExpectedTokenNames() ::= "this.getExpectedTokens().toString(this.literalNames)"
 

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Firefox.test.stg

@@ -76,11 +76,7 @@ LANotEquals(i, v) ::= <%this._input.LA(<i>)!=<v>%>
 
 TokenStartColumnEquals(i) ::= <%this._tokenStartColumn===<i>%>
 
-ImportListener(X) ::= <<
+ImportListener(X) ::= ""
-@parser::header {
-var <X>Listener = require('./<X>Listener').<X>Listener;
-}
->>
 
 GetExpectedTokenNames() ::= "this.getExpectedTokens().toString(this.literalNames)"
 

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Node.test.stg

@@ -74,11 +74,7 @@ LANotEquals(i, v) ::= <%this._input.LA(<i>)!=<v>%>
 
 TokenStartColumnEquals(i) ::= <%this._tokenStartColumn===<i>%>
 
-ImportListener(X) ::= <<
+ImportListener(X) ::= ""
-@parser::header {
-var <X>Listener = require('./<X>Listener').<X>Listener;
-}
->>
 
 GetExpectedTokenNames() ::= "this.getExpectedTokens().toString(this.literalNames)"
 

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Safari.test.stg

@@ -74,11 +74,7 @@ LANotEquals(i, v) ::= <%this._input.LA(<i>)!=<v>%>
 
 TokenStartColumnEquals(i) ::= <%this._tokenStartColumn===<i>%>
 
-ImportListener(X) ::= <<
+ImportListener(X) ::= ""
-@parser::header {
-var <X>Listener = require('./<X>Listener').<X>Listener;
-}
->>
 
 GetExpectedTokenNames() ::= "this.getExpectedTokens().toString(this.literalNames)"
 

runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java

@@ -46,13 +46,14 @@ import static org.junit.Assume.assumeFalse;
 public abstract class BaseRuntimeTest {
 	public final static String[] Targets = {
 		"Cpp",
-		"Java",
-		"Go",
 		"CSharp",
-		"Python2", "Python3",
+		"Dart",
-		"PHP",
+		"Go",
+		"Java",
 		"Node",
-		"Dart"
+		"PHP",
+		"Python2", "Python3",
+		"Swift"
 	};
 
 	static {
@@ -95,10 +96,17 @@ public abstract class BaseRuntimeTest {
 	public void setUp() throws Exception {
 		// From http://junit.sourceforge.net/javadoc/org/junit/Assume.html
 		// "The default JUnit runner treats tests with failing assumptions as ignored"
-		assumeFalse(descriptor.ignore(descriptor.getTarget()));
+		assumeFalse(checkIgnored());
 		delegate.testSetUp();
 	}
 
+	public boolean checkIgnored() {
+		boolean ignored = !TestContext.isSupportedTarget(descriptor.getTarget()) || descriptor.ignore(descriptor.getTarget());
+		if(ignored)
+			System.out.println("Ignore " + descriptor);
+		return ignored;
+	}
+
 	@Rule
 	public final TestRule testWatcher = new TestWatcher() {
 		@Override
@@ -112,7 +120,7 @@ public abstract class BaseRuntimeTest {
 	public void testOne() throws Exception {
 		// System.out.println(delegate.getTmpDir());
 		if ( descriptor.ignore(descriptor.getTarget()) ) {
-			System.out.printf("Ignore "+descriptor);
+			System.out.println("Ignore " + descriptor);
 			return;
 		}
 
@@ -273,6 +281,8 @@ public abstract class BaseRuntimeTest {
 	// ---- support ----
 
 	public static RuntimeTestDescriptor[] getRuntimeTestDescriptors(Class<?> clazz, String targetName) {
+		if(!TestContext.isSupportedTarget(targetName))
+			return new RuntimeTestDescriptor[0];
 		Class<?>[] nestedClasses = clazz.getClasses();
 		List<RuntimeTestDescriptor> descriptors = new ArrayList<RuntimeTestDescriptor>();
 		for (Class<?> nestedClass : nestedClasses) {
@@ -280,8 +290,10 @@ public abstract class BaseRuntimeTest {
 			if ( RuntimeTestDescriptor.class.isAssignableFrom(nestedClass) && !Modifier.isAbstract(modifiers) ) {
 				try {
 					RuntimeTestDescriptor d = (RuntimeTestDescriptor) nestedClass.newInstance();
-					d.setTarget(targetName);
+					if(!d.ignore(targetName)) {
-					descriptors.add(d);
+						d.setTarget(targetName);
+						descriptors.add(d);
+					}
 				} catch (Exception e) {
 					e.printStackTrace(System.err);
 				}

runtime-testsuite/test/org/antlr/v4/test/runtime/TestContext.java

@@ -0,0 +1,19 @@
+package org.antlr.v4.test.runtime;
+
+public abstract class TestContext {
+
+	public static boolean isTravisCI() {
+		return "true".equals(String.valueOf(System.getenv("TRAVIS")).toLowerCase());
+	}
+
+	public static boolean isAppVeyorCI() {
+		return "true".equals(String.valueOf(System.getenv("APPVEYOR")).toLowerCase());
+	}
+
+	public static boolean isSupportedTarget(String target) {
+		if(isAppVeyorCI())
+			return !target.matches("Swift|Node");
+		else
+			return true;
+	}
+}

runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LexerExecDescriptors.java

@@ -1057,4 +1057,33 @@ public class LexerExecDescriptors {
 			return new Pair<>(grammarName, grammar);
 		}
 	}
+
+	/**
+	 * This is a regression test for antlr/antlr4#2709 "PHP target generates
+	 * invalid output when $ is used as part of the literal in lexer rule"
+	 * https://github.com/antlr/antlr4/issues/2709
+	 */
+	public static class EscapeTargetStringLiteral extends BaseLexerTestDescriptor {
+		/**
+		[@0,0:-1='<EOF>',<-1>,1:0]
+		 */
+		@CommentHasStringValue
+		public String output;
+
+		public String errors = null;
+		public String startRule = "";
+		public String grammarName = "L";
+
+		/**
+		 lexer grammar L;
+		 ACTION_WITH_DOLLAR: '$ACTION';
+		 */
+		@CommentHasStringValue
+		public String grammar;
+
+		@Override
+		public boolean ignore(String targetName) {
+			return !targetName.equals("PHP");
+		}
+	}
 }

runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java

@@ -34,6 +34,7 @@ import org.antlr.v4.semantics.SemanticPipeline;
 import org.antlr.v4.test.runtime.ErrorQueue;
 import org.antlr.v4.test.runtime.RuntimeTestSupport;
 import org.antlr.v4.test.runtime.StreamVacuum;
+import org.antlr.v4.test.runtime.TestContext;
 import org.antlr.v4.tool.ANTLRMessage;
 import org.antlr.v4.tool.DOTGenerator;
 import org.antlr.v4.tool.Grammar;
@@ -45,6 +46,7 @@ import org.stringtemplate.v4.STGroup;
 import org.stringtemplate.v4.STGroupString;
 
 import java.io.File;
+import java.io.IOException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -55,6 +57,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
 
 import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
 import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
@@ -128,16 +131,6 @@ public class BaseNodeTest implements RuntimeTestSupport {
 		return antlrToolErrors.toString();
 	}
 
-	protected org.antlr.v4.Tool newTool(String[] args) {
-		Tool tool = new Tool(args);
-		return tool;
-	}
-
-	protected Tool newTool() {
-		org.antlr.v4.Tool tool = new Tool(new String[] { "-o", tmpdir });
-		return tool;
-	}
-
 	protected ATN createATN(Grammar g, boolean useSerializer) {
 		if (g.atn == null) {
 			semanticProcess(g);
@@ -179,57 +172,6 @@ public class BaseNodeTest implements RuntimeTestSupport {
 		}
 	}
 
-	IntegerList getTypesFromString(Grammar g, String expecting) {
-		IntegerList expectingTokenTypes = new IntegerList();
-		if (expecting != null && !expecting.trim().isEmpty()) {
-			for (String tname : expecting.replace(" ", "").split(",")) {
-				int ttype = g.getTokenType(tname);
-				expectingTokenTypes.add(ttype);
-			}
-		}
-		return expectingTokenTypes;
-	}
-
-	public IntegerList getTokenTypesViaATN(String input,
-			LexerATNSimulator lexerATN) {
-		ANTLRInputStream in = new ANTLRInputStream(input);
-		IntegerList tokenTypes = new IntegerList();
-		int ttype;
-		do {
-			ttype = lexerATN.match(in, Lexer.DEFAULT_MODE);
-			tokenTypes.add(ttype);
-		} while (ttype != Token.EOF);
-		return tokenTypes;
-	}
-
-	public List<String> getTokenTypes(LexerGrammar lg, ATN atn, CharStream input) {
-		LexerATNSimulator interp = new LexerATNSimulator(atn,
-				new DFA[] { new DFA(
-						atn.modeToStartState.get(Lexer.DEFAULT_MODE)) }, null);
-		List<String> tokenTypes = new ArrayList<String>();
-		int ttype;
-		boolean hitEOF = false;
-		do {
-			if (hitEOF) {
-				tokenTypes.add("EOF");
-				break;
-			}
-			int t = input.LA(1);
-			ttype = interp.match(input, Lexer.DEFAULT_MODE);
-			if (ttype == Token.EOF) {
-				tokenTypes.add("EOF");
-			}
-			else {
-				tokenTypes.add(lg.typeToTokenList.get(ttype));
-			}
-
-			if (t == IntStream.EOF) {
-				hitEOF = true;
-			}
-		} while (ttype != Token.EOF);
-		return tokenTypes;
-	}
-
 	protected String execLexer(String grammarFileName, String grammarStr,
 	                           String lexerName, String input) {
 		return execLexer(grammarFileName, grammarStr, lexerName, input, false);
@@ -243,8 +185,9 @@ public class BaseNodeTest implements RuntimeTestSupport {
 		assertTrue(success);
 		writeFile(tmpdir, "input", input);
 		writeLexerTestFile(lexerName, showDFA);
+		writeFile(tmpdir, "package.json", "{\"type\": \"module\"}");
 		String output = execModule("Test.js");
-		if ( output.length()==0 ) {
+		if ( output!=null && output.length()==0 ) {
 			output = null;
 		}
 		return output;
@@ -262,6 +205,7 @@ public class BaseNodeTest implements RuntimeTestSupport {
 		writeFile(tmpdir, "input", input);
 		rawBuildRecognizerTestFile(parserName, lexerName, listenerName,
 		                           visitorName, startRuleName, showDiagnosticErrors);
+		writeFile(tmpdir, "package.json", "{\"type\": \"module\"}");
 		return execRecognizer();
 	}
 
@@ -323,17 +267,21 @@ public class BaseNodeTest implements RuntimeTestSupport {
 	}
 
 	public String execModule(String fileName) {
-		String nodejsPath = locateNodeJS();
-		String runtimePath = locateRuntime();
-		String modulePath = new File(new File(tmpdir), fileName)
-				.getAbsolutePath();
-		String inputPath = new File(new File(tmpdir), "input")
-				.getAbsolutePath();
 		try {
+			String npmPath = locateNpm();
+			if(!TestContext.isTravisCI()) {
+				installRuntime(npmPath);
+				registerRuntime(npmPath);
+			}
+			String modulePath = new File(new File(tmpdir), fileName)
+					.getAbsolutePath();
+			linkRuntime(npmPath);
+			String nodejsPath = locateNodeJS();
+			String inputPath = new File(new File(tmpdir), "input")
+					.getAbsolutePath();
 			ProcessBuilder builder = new ProcessBuilder(nodejsPath, modulePath,
 					inputPath);
-			builder.environment().put("NODE_PATH",
+			builder.environment().put("NODE_PATH", tmpdir);
-					runtimePath + File.pathSeparator + tmpdir);
 			builder.directory(new File(tmpdir));
 			Process process = builder.start();
 			StreamVacuum stdoutVacuum = new StreamVacuum(
@@ -342,7 +290,10 @@ public class BaseNodeTest implements RuntimeTestSupport {
 					process.getErrorStream());
 			stdoutVacuum.start();
 			stderrVacuum.start();
+			// TODO switch to jdk 8
 			process.waitFor();
+			// if(!process.waitFor(1L, TimeUnit.MINUTES))
+			//	process.destroyForcibly();
 			stdoutVacuum.join();
 			stderrVacuum.join();
 			String output = stdoutVacuum.toString();
@@ -353,22 +304,59 @@ public class BaseNodeTest implements RuntimeTestSupport {
 				this.stderrDuringParse = stderrVacuum.toString();
 			}
 			return output;
-		}
+		} catch (Exception e) {
-		catch (Exception e) {
 			System.err.println("can't exec recognizer");
 			e.printStackTrace(System.err);
+			System.err.println();
+			return null;
 		}
-		return null;
 	}
 
-	private String locateTool(String tool) {
+	private void installRuntime(String npmPath) throws IOException, InterruptedException {
-		String[] roots = { "/usr/bin/", "/usr/local/bin/" };
+		String runtimePath = locateRuntime();
-		for (String root : roots) {
+		ProcessBuilder builder = new ProcessBuilder(npmPath, "install");
-			if (new File(root + tool).exists()) {
+		builder.directory(new File(runtimePath));
-				return root + tool;
+		builder.redirectError(new File(tmpdir, "error.txt"));
-			}
+		builder.redirectOutput(new File(tmpdir, "output.txt"));
-		}
+		Process process = builder.start();
-		return null;
+		// TODO switch to jdk 8
+		process.waitFor();
+		// if(!process.waitFor(30L, TimeUnit.SECONDS))
+		// 	process.destroyForcibly();
+		int error = process.exitValue();
+		if(error!=0)
+			throw new IOException("'npm install' failed");
+	}
+
+	private void registerRuntime(String npmPath) throws IOException, InterruptedException {
+		String runtimePath = locateRuntime();
+		ProcessBuilder builder = new ProcessBuilder(npmPath, "link");
+		builder.directory(new File(runtimePath));
+		builder.redirectError(new File(tmpdir, "error.txt"));
+		builder.redirectOutput(new File(tmpdir, "output.txt"));
+		Process process = builder.start();
+		// TODO switch to jdk 8
+		process.waitFor();
+		// if(!process.waitFor(30L, TimeUnit.SECONDS))
+		//	process.destroyForcibly();
+		int error = process.exitValue();
+		if(error!=0)
+			throw new IOException("'npm link' failed");
+	}
+
+	private void linkRuntime(String npmPath) throws IOException, InterruptedException {
+		ProcessBuilder builder = new ProcessBuilder(npmPath, "link", "antlr4");
+		builder.directory(new File(tmpdir));
+		builder.redirectError(new File(tmpdir, "error.txt"));
+		builder.redirectOutput(new File(tmpdir, "output.txt"));
+		Process process = builder.start();
+		// TODO switch to jdk 8
+		process.waitFor();
+		// if(!process.waitFor(30L, TimeUnit.SECONDS))
+		//	process.destroyForcibly();
+		int error = process.exitValue();
+		if(error!=0)
+			throw new IOException("'npm link antlr4' failed");
 	}
 
 	private boolean canExecute(String tool) {
@@ -378,21 +366,29 @@ public class BaseNodeTest implements RuntimeTestSupport {
 			Process process = builder.start();
 			StreamVacuum vacuum = new StreamVacuum(process.getInputStream());
 			vacuum.start();
+			// TODO switch to jdk 8
 			process.waitFor();
+			// if(!process.waitFor(30L, TimeUnit.SECONDS))
+			//	process.destroyForcibly();
 			vacuum.join();
 			return process.exitValue() == 0;
+		} catch (Exception e) {
+			return false;
 		}
-		catch (Exception e) {
+	}
-			;
+
+	private String locateNpm() {
+		// typically /usr/local/bin/npm
+		String prop = System.getProperty("antlr-javascript-npm");
+		if ( prop!=null && prop.length()!=0 ) {
+			return prop;
 		}
-		return false;
+		return "npm"; // everywhere
 	}
 
 	private String locateNodeJS() {
 		// typically /usr/local/bin/node
-		String propName = "antlr-javascript-nodejs";
+		String prop = System.getProperty("antlr-javascript-nodejs");
-		String prop = System.getProperty(propName);
-
 		if ( prop!=null && prop.length()!=0 ) {
 			return prop;
 		}
@@ -404,7 +400,7 @@ public class BaseNodeTest implements RuntimeTestSupport {
 
 	private String locateRuntime() {
 		final ClassLoader loader = Thread.currentThread().getContextClassLoader();
-		final URL runtimeSrc = loader.getResource("JavaScript/src");
+		final URL runtimeSrc = loader.getResource("JavaScript");
 		if ( runtimeSrc==null ) {
 			throw new RuntimeException("Cannot find JavaScript runtime");
 		}
@@ -418,213 +414,16 @@ public class BaseNodeTest implements RuntimeTestSupport {
 		return System.getProperty("os.name").toLowerCase().contains("windows");
 	}
 
-	// void ambig(List<Message> msgs, int[] expectedAmbigAlts, String
-	// expectedAmbigInput)
-	// throws Exception
-	// {
-	// ambig(msgs, 0, expectedAmbigAlts, expectedAmbigInput);
-	// }
-
-	// void ambig(List<Message> msgs, int i, int[] expectedAmbigAlts, String
-	// expectedAmbigInput)
-	// throws Exception
-	// {
-	// List<Message> amsgs = getMessagesOfType(msgs, AmbiguityMessage.class);
-	// AmbiguityMessage a = (AmbiguityMessage)amsgs.get(i);
-	// if ( a==null ) assertNull(expectedAmbigAlts);
-	// else {
-	// assertEquals(a.conflictingAlts.toString(),
-	// Arrays.toString(expectedAmbigAlts));
-	// }
-	// assertEquals(expectedAmbigInput, a.input);
-	// }
-
-	// void unreachable(List<Message> msgs, int[] expectedUnreachableAlts)
-	// throws Exception
-	// {
-	// unreachable(msgs, 0, expectedUnreachableAlts);
-	// }
-
-	// void unreachable(List<Message> msgs, int i, int[]
-	// expectedUnreachableAlts)
-	// throws Exception
-	// {
-	// List<Message> amsgs = getMessagesOfType(msgs,
-	// UnreachableAltsMessage.class);
-	// UnreachableAltsMessage u = (UnreachableAltsMessage)amsgs.get(i);
-	// if ( u==null ) assertNull(expectedUnreachableAlts);
-	// else {
-	// assertEquals(u.conflictingAlts.toString(),
-	// Arrays.toString(expectedUnreachableAlts));
-	// }
-	// }
-
-	List<ANTLRMessage> getMessagesOfType(List<ANTLRMessage> msgs,
-			Class<? extends ANTLRMessage> c) {
-		List<ANTLRMessage> filtered = new ArrayList<ANTLRMessage>();
-		for (ANTLRMessage m : msgs) {
-			if (m.getClass() == c)
-				filtered.add(m);
-		}
-		return filtered;
-	}
-
-	void checkRuleATN(Grammar g, String ruleName, String expecting) {
-		ParserATNFactory f = new ParserATNFactory(g);
-		ATN atn = f.createATN();
-
-		DOTGenerator dot = new DOTGenerator(g);
-		System.out
-				.println(dot.getDOT(atn.ruleToStartState[g.getRule(ruleName).index]));
-
-		Rule r = g.getRule(ruleName);
-		ATNState startState = atn.ruleToStartState[r.index];
-		ATNPrinter serializer = new ATNPrinter(g, startState);
-		String result = serializer.asString();
-
-		// System.out.print(result);
-		assertEquals(expecting, result);
-	}
-
-	public void testActions(String templates, String actionName, String action,
-			String expected) throws org.antlr.runtime.RecognitionException {
-		int lp = templates.indexOf('(');
-		String name = templates.substring(0, lp);
-		STGroup group = new STGroupString(templates);
-		ST st = group.getInstanceOf(name);
-		st.add(actionName, action);
-		String grammar = st.render();
-		ErrorQueue equeue = new ErrorQueue();
-		Grammar g = new Grammar(grammar, equeue);
-		if (g.ast != null && !g.ast.hasErrors) {
-			SemanticPipeline sem = new SemanticPipeline(g);
-			sem.process();
-
-			ATNFactory factory = new ParserATNFactory(g);
-			if (g.isLexer())
-				factory = new LexerATNFactory((LexerGrammar) g);
-			g.atn = factory.createATN();
-
-			CodeGenerator gen = new CodeGenerator(g);
-			ST outputFileST = gen.generateParser();
-			String output = outputFileST.render();
-			// System.out.println(output);
-			String b = "#" + actionName + "#";
-			int start = output.indexOf(b);
-			String e = "#end-" + actionName + "#";
-			int end = output.indexOf(e);
-			String snippet = output.substring(start + b.length(), end);
-			assertEquals(expected, snippet);
-		}
-		if (equeue.size() > 0) {
-			System.err.println(equeue.toString());
-		}
-	}
-
-	protected void checkGrammarSemanticsError(ErrorQueue equeue,
-			GrammarSemanticsMessage expectedMessage) throws Exception {
-		ANTLRMessage foundMsg = null;
-		for (int i = 0; i < equeue.errors.size(); i++) {
-			ANTLRMessage m = equeue.errors.get(i);
-			if (m.getErrorType() == expectedMessage.getErrorType()) {
-				foundMsg = m;
-			}
-		}
-		assertNotNull("no error; " + expectedMessage.getErrorType()
-				+ " expected", foundMsg);
-		assertTrue("error is not a GrammarSemanticsMessage",
-				foundMsg instanceof GrammarSemanticsMessage);
-		assertEquals(Arrays.toString(expectedMessage.getArgs()),
-				Arrays.toString(foundMsg.getArgs()));
-		if (equeue.size() != 1) {
-			System.err.println(equeue);
-		}
-	}
-
-	protected void checkGrammarSemanticsWarning(ErrorQueue equeue,
-			GrammarSemanticsMessage expectedMessage) throws Exception {
-		ANTLRMessage foundMsg = null;
-		for (int i = 0; i < equeue.warnings.size(); i++) {
-			ANTLRMessage m = equeue.warnings.get(i);
-			if (m.getErrorType() == expectedMessage.getErrorType()) {
-				foundMsg = m;
-			}
-		}
-		assertNotNull("no error; " + expectedMessage.getErrorType()
-				+ " expected", foundMsg);
-		assertTrue("error is not a GrammarSemanticsMessage",
-				foundMsg instanceof GrammarSemanticsMessage);
-		assertEquals(Arrays.toString(expectedMessage.getArgs()),
-				Arrays.toString(foundMsg.getArgs()));
-		if (equeue.size() != 1) {
-			System.err.println(equeue);
-		}
-	}
-
-	protected void checkError(ErrorQueue equeue, ANTLRMessage expectedMessage)
-			throws Exception {
-		// System.out.println("errors="+equeue);
-		ANTLRMessage foundMsg = null;
-		for (int i = 0; i < equeue.errors.size(); i++) {
-			ANTLRMessage m = equeue.errors.get(i);
-			if (m.getErrorType() == expectedMessage.getErrorType()) {
-				foundMsg = m;
-			}
-		}
-		assertTrue("no error; " + expectedMessage.getErrorType() + " expected",
-				!equeue.errors.isEmpty());
-		assertTrue("too many errors; " + equeue.errors,
-				equeue.errors.size() <= 1);
-		assertNotNull(
-				"couldn't find expected error: "
-						+ expectedMessage.getErrorType(), foundMsg);
-		/*
-		 * assertTrue("error is not a GrammarSemanticsMessage", foundMsg
-		 * instanceof GrammarSemanticsMessage);
-		 */
-		assertArrayEquals(expectedMessage.getArgs(), foundMsg.getArgs());
-	}
-
-	public static class FilteringTokenStream extends CommonTokenStream {
-		public FilteringTokenStream(TokenSource src) {
-			super(src);
-		}
-
-		Set<Integer> hide = new HashSet<Integer>();
-
-		@Override
-		protected boolean sync(int i) {
-			if (!super.sync(i)) {
-				return false;
-			}
-
-			Token t = get(i);
-			if (hide.contains(t.getType())) {
-				((WritableToken) t).setChannel(Token.HIDDEN_CHANNEL);
-			}
-
-			return true;
-		}
-
-		public void setTokenTypeChannel(int ttype, int channel) {
-			hide.add(ttype);
-		}
-	}
-
-	protected void mkdir(String dir) {
-		File f = new File(dir);
-		f.mkdirs();
-	}
 
 	protected void writeParserTestFile(String parserName, String lexerName,
 			String listenerName, String visitorName,
 			String parserStartRuleName, boolean debug) {
 		ST outputFileST = new ST(
-				"var antlr4 = require('antlr4');\n"
+				"import antlr4 from 'antlr4';\n"
-						+ "var <lexerName> = require('./<lexerName>');\n"
+						+ "import <lexerName> from './<lexerName>.js';\n"
-						+ "var <parserName> = require('./<parserName>');\n"
+						+ "import <parserName> from './<parserName>.js';\n"
-						+ "var <listenerName> = require('./<listenerName>').<listenerName>;\n"
+						+ "import <listenerName> from './<listenerName>.js';\n"
-						+ "var <visitorName> = require('./<visitorName>').<visitorName>;\n"
+						+ "import <visitorName> from './<visitorName>.js';\n"
 						+ "\n"
 						+ "class TreeShapeListener extends antlr4.tree.ParseTreeListener {\n" +
 						"    enterEveryRule(ctx) {\n" +
@@ -640,11 +439,11 @@ public class BaseNodeTest implements RuntimeTestSupport {
 						+ "\n"
 						+ "function main(argv) {\n"
 						+ "    var input = new antlr4.FileStream(argv[2], true);\n"
-						+ "    var lexer = new <lexerName>.<lexerName>(input);\n"
+						+ "    var lexer = new <lexerName>(input);\n"
 						+ "    var stream = new antlr4.CommonTokenStream(lexer);\n"
 						+ "<createParser>"
 						+ "    parser.buildParseTrees = true;\n"
-						+ "	 printer = function() {\n"
+						+ "	   const printer = function() {\n"
 						+ "		this.println = function(s) { console.log(s); }\n"
 						+ "		this.print = function(s) { process.stdout.write(s); }\n"
 						+ "		return this;\n"
@@ -654,10 +453,10 @@ public class BaseNodeTest implements RuntimeTestSupport {
 						+ "    antlr4.tree.ParseTreeWalker.DEFAULT.walk(new TreeShapeListener(), tree);\n"
 						+ "}\n" + "\n" + "main(process.argv);\n" + "\n");
 		ST createParserST = new ST(
-				"	var parser = new <parserName>.<parserName>(stream);\n");
+				"	var parser = new <parserName>(stream);\n");
 		if (debug) {
 			createParserST = new ST(
-					"	var parser = new <parserName>.<parserName>(stream);\n"
+					"	var parser = new <parserName>(stream);\n"
 							+ "	parser.addErrorListener(new antlr4.error.DiagnosticErrorListener());\n");
 		}
 		outputFileST.add("createParser", createParserST);
@@ -671,12 +470,12 @@ public class BaseNodeTest implements RuntimeTestSupport {
 
 	protected void writeLexerTestFile(String lexerName, boolean showDFA) {
 		ST outputFileST = new ST(
-				"var antlr4 = require('antlr4');\n"
+				"import antlr4 from 'antlr4';\n"
-						+ "var <lexerName> = require('./<lexerName>');\n"
+						+ "import <lexerName> from './<lexerName>.js';\n"
 						+ "\n"
 						+ "function main(argv) {\n"
 						+ "    var input = new antlr4.FileStream(argv[2], true);\n"
-						+ "    var lexer = new <lexerName>.<lexerName>(input);\n"
+						+ "    var lexer = new <lexerName>(input);\n"
 						+ "    var stream = new antlr4.CommonTokenStream(lexer);\n"
 						+ "    stream.fill();\n"
 						+ "    for(var i=0; i\\<stream.tokens.length; i++) {\n"
@@ -689,28 +488,6 @@ public class BaseNodeTest implements RuntimeTestSupport {
 		writeFile(tmpdir, "Test.js", outputFileST.render());
 	}
 
-	public void writeRecognizer(String parserName, String lexerName,
-			String listenerName, String visitorName,
-			String parserStartRuleName, boolean debug) {
-		if (parserName == null) {
-			writeLexerTestFile(lexerName, debug);
-		}
-		else {
-			writeParserTestFile(parserName, lexerName, listenerName,
-					visitorName, parserStartRuleName, debug);
-		}
-	}
-
-	protected void eraseFiles(final String filesEndingWith) {
-		File tmpdirF = new File(tmpdir);
-		String[] files = tmpdirF.list();
-		for (int i = 0; files != null && i < files.length; i++) {
-			if (files[i].endsWith(filesEndingWith)) {
-				new File(tmpdir + "/" + files[i]).delete();
-			}
-		}
-	}
-
 	protected void eraseFiles(File dir) {
 		String[] files = dir.list();
 		for (int i = 0; files != null && i < files.length; i++) {
@@ -734,153 +511,10 @@ public class BaseNodeTest implements RuntimeTestSupport {
 		}
 	}
 
-	public String getFirstLineOfException() {
-		if (this.stderrDuringParse == null) {
-			return null;
-		}
-		String[] lines = this.stderrDuringParse.split("\n");
-		String prefix = "Exception in thread \"main\" ";
-		return lines[0].substring(prefix.length(), lines[0].length());
-	}
-
-	/**
-	 * When looking at a result set that consists of a Map/HashTable we cannot
-	 * rely on the output order, as the hashing algorithm or other aspects of
-	 * the implementation may be different on differnt JDKs or platforms. Hence
-	 * we take the Map, convert the keys to a List, sort them and Stringify the
-	 * Map, which is a bit of a hack, but guarantees that we get the same order
-	 * on all systems. We assume that the keys are strings.
-	 *
-	 * @param m
-	 *            The Map that contains keys we wish to return in sorted order
-	 * @return A string that represents all the keys in sorted order.
-	 */
-	public <K, V> String sortMapToString(Map<K, V> m) {
-		// Pass in crap, and get nothing back
-		//
-		if (m == null) {
-			return null;
-		}
-
-		System.out.println("Map toString looks like: " + m.toString());
-
-		// Sort the keys in the Map
-		//
-		TreeMap<K, V> nset = new TreeMap<K, V>(m);
-
-		System.out.println("Tree map looks like: " + nset.toString());
-		return nset.toString();
-	}
-
-	public List<String> realElements(List<String> elements) {
-		return elements.subList(Token.MIN_USER_TOKEN_TYPE, elements.size());
-	}
-
-	public void assertNotNullOrEmpty(String message, String text) {
-		assertNotNull(message, text);
-		assertFalse(message, text.isEmpty());
-	}
-
-	public void assertNotNullOrEmpty(String text) {
-		assertNotNull(text);
-		assertFalse(text.isEmpty());
-	}
-
-	public static class IntTokenStream implements TokenStream {
-		IntegerList types;
-		int p = 0;
-
-		public IntTokenStream(IntegerList types) {
-			this.types = types;
-		}
-
-		@Override
-		public void consume() {
-			p++;
-		}
-
-		@Override
-		public int LA(int i) {
-			return LT(i).getType();
-		}
-
-		@Override
-		public int mark() {
-			return index();
-		}
-
-		@Override
-		public int index() {
-			return p;
-		}
-
-		@Override
-		public void release(int marker) {
-			seek(marker);
-		}
-
-		@Override
-		public void seek(int index) {
-			p = index;
-		}
-
-		@Override
-		public int size() {
-			return types.size();
-		}
-
-		@Override
-		public String getSourceName() {
-			return null;
-		}
-
-		@Override
-		public Token LT(int i) {
-			CommonToken t;
-			int rawIndex = p + i - 1;
-			if (rawIndex >= types.size())
-				t = new CommonToken(Token.EOF);
-			else
-				t = new CommonToken(types.get(rawIndex));
-			t.setTokenIndex(rawIndex);
-			return t;
-		}
-
-		@Override
-		public Token get(int i) {
-			return new org.antlr.v4.runtime.CommonToken(types.get(i));
-		}
-
-		@Override
-		public TokenSource getTokenSource() {
-			return null;
-		}
-
-		@Override
-		public String getText() {
-			throw new UnsupportedOperationException("can't give strings");
-		}
-
-		@Override
-		public String getText(Interval interval) {
-			throw new UnsupportedOperationException("can't give strings");
-		}
-
-		@Override
-		public String getText(RuleContext ctx) {
-			throw new UnsupportedOperationException("can't give strings");
-		}
-
-		@Override
-		public String getText(Token start, Token stop) {
-			throw new UnsupportedOperationException("can't give strings");
-		}
-	}
 
 	/** Sort a list */
 	public <T extends Comparable<? super T>> List<T> sort(List<T> data) {
-		List<T> dup = new ArrayList<T>();
+		List<T> dup = new ArrayList<T>(data);
-		dup.addAll(data);
 		Collections.sort(dup);
 		return dup;
 	}
@@ -889,8 +523,7 @@ public class BaseNodeTest implements RuntimeTestSupport {
 	public <K extends Comparable<? super K>, V> LinkedHashMap<K, V> sort(
 			Map<K, V> data) {
 		LinkedHashMap<K, V> dup = new LinkedHashMap<K, V>();
-		List<K> keys = new ArrayList<K>();
+		List<K> keys = new ArrayList<K>(data.keySet());
-		keys.addAll(data.keySet());
 		Collections.sort(keys);
 		for (K k : keys) {
 			dup.put(k, data.get(k));

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/SingletonPredictionContext.cs

@@ -79,7 +79,7 @@ namespace Antlr4.Runtime.Atn
                 return false;
             }
 			Antlr4.Runtime.Atn.SingletonPredictionContext other = (Antlr4.Runtime.Atn.SingletonPredictionContext)o;
-            return returnState == other.returnState && parent.Equals(other.parent);
+            return returnState == other.returnState && (parent != null && parent.Equals(other.parent));
         }
 
 		public override string ToString()

runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Parser.cs

@@ -22,20 +22,20 @@ namespace Antlr4.Runtime
 #if !PORTABLE
         public class TraceListener : IParseTreeListener
         {
-            private readonly TextWriter Output;
 
-            public TraceListener(TextWriter output) {
+            public TraceListener(TextWriter output,Parser enclosing) {
-                Output = output;
+                _output = output;
+                _enclosing = enclosing;
             }
 
             public virtual void EnterEveryRule(ParserRuleContext ctx)
             {
-                Output.WriteLine("enter   " + this._enclosing.RuleNames[ctx.RuleIndex] + ", LT(1)=" + this._enclosing._input.LT(1).Text);
+                _output.WriteLine("enter   " + this._enclosing.RuleNames[ctx.RuleIndex] + ", LT(1)=" + this._enclosing._input.LT(1).Text);
             }
 
             public virtual void ExitEveryRule(ParserRuleContext ctx)
             {
-                Output.WriteLine("exit    " + this._enclosing.RuleNames[ctx.RuleIndex] + ", LT(1)=" + this._enclosing._input.LT(1).Text);
+                _output.WriteLine("exit    " + this._enclosing.RuleNames[ctx.RuleIndex] + ", LT(1)=" + this._enclosing._input.LT(1).Text);
             }
 
             public virtual void VisitErrorNode(IErrorNode node)
@@ -46,15 +46,17 @@ namespace Antlr4.Runtime
             {
                 ParserRuleContext parent = (ParserRuleContext)((IRuleNode)node.Parent).RuleContext;
                 IToken token = node.Symbol;
-                Output.WriteLine("consume " + token + " rule " + this._enclosing.RuleNames[parent.RuleIndex]);
+                _output.WriteLine("consume " + token + " rule " + this._enclosing.RuleNames[parent.RuleIndex]);
             }
 
             internal TraceListener(Parser _enclosing)
             {
                 this._enclosing = _enclosing;
+                _output = Console.Out;
             }
 
             private readonly Parser _enclosing;
+            private readonly TextWriter _output;
         }
 #endif
 

runtime/Cpp/CMakeLists.txt

@@ -71,6 +71,9 @@ else()
   set(MY_CXX_WARNING_FLAGS "  -Wall -pedantic -W")
 endif()
 
+# Define USE_UTF8_INSTEAD_OF_CODECVT macro.
+# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_UTF8_INSTEAD_OF_CODECVT")
+
 # Initialize CXXFLAGS.
 if("${CMAKE_VERSION}" VERSION_GREATER 3.1.0)
   set(CMAKE_CXX_STANDARD 11)

runtime/Cpp/deploy-macos.sh

@@ -4,9 +4,27 @@
 rm -f -R antlr4-runtime build lib 2> /dev/null
 rm antlr4-cpp-runtime-macos.zip 2> /dev/null
 
+# Get utf8 dependency.
+mkdir -p runtime/thirdparty 2> /dev/null
+pushd runtime/thirdparty
+if [ ! -d utfcpp ]
+then
+    git clone https://github.com/nemtrif/utfcpp.git utfcpp
+    pushd utfcpp
+    git checkout tags/v3.1.1
+    popd
+fi
+popd
+
 # Binaries
-xcodebuild -project runtime/antlrcpp.xcodeproj -target antlr4 -configuration Release
+xcodebuild -project runtime/antlrcpp.xcodeproj \
-xcodebuild -project runtime/antlrcpp.xcodeproj -target antlr4_static -configuration Release
+           -target antlr4                      \
+           # GCC_PREPROCESSOR_DEFINITIONS='$GCC_PREPROCESSOR_DEFINITIONS USE_UTF8_INSTEAD_OF_CODECVT' \
+           -configuration Release
+xcodebuild -project runtime/antlrcpp.xcodeproj \
+           -target antlr4_static               \
+           # GCC_PREPROCESSOR_DEFINITIONS='$GCC_PREPROCESSOR_DEFINITIONS USE_UTF8_INSTEAD_OF_CODECVT' \
+           -configuration Release
 rm -f -R lib
 mkdir lib
 mv runtime/build/Release/libantlr4-runtime.a lib/
@@ -17,6 +35,9 @@ rm -f -R antlr4-runtime
 pushd runtime/src
 find . -name '*.h' | cpio -pdm ../../antlr4-runtime
 popd
+pushd runtime/thirdparty/utfcpp/source
+find . -name '*.h' | cpio -pdm ../../../../antlr4-runtime
+popd
 
 # Zip up and clean up
 zip -r antlr4-cpp-runtime-macos.zip antlr4-runtime lib

runtime/Cpp/runtime/CMakeLists.txt

@@ -1,4 +1,19 @@
 
+include(${CMAKE_ROOT}/Modules/ExternalProject.cmake)
+
+set(THIRDPARTY_DIR ${CMAKE_BINARY_DIR}/runtime/thirdparty)
+set(UTFCPP_DIR ${THIRDPARTY_DIR}/utfcpp)
+ExternalProject_Add(
+  utfcpp
+  GIT_REPOSITORY        "git://github.com/nemtrif/utfcpp"
+  GIT_TAG               "v3.1.1"
+  SOURCE_DIR            ${UTFCPP_DIR}
+  UPDATE_DISCONNECTED   1
+  CMAKE_ARGS            -DCMAKE_INSTALL_PREFIX=${UTFCPP_DIR}/install -Dgtest_force_shared_crt=ON
+  TEST_AFTER_INSTALL    1
+  STEP_TARGETS          build)
+
+
 include_directories(
   ${PROJECT_SOURCE_DIR}/runtime/src
   ${PROJECT_SOURCE_DIR}/runtime/src/atn
@@ -8,6 +23,8 @@ include_directories(
   ${PROJECT_SOURCE_DIR}/runtime/src/tree
   ${PROJECT_SOURCE_DIR}/runtime/src/tree/pattern
   ${PROJECT_SOURCE_DIR}/runtime/src/tree/xpath
+  ${UTFCPP_DIR}/install/include/utf8cpp
+  ${UTFCPP_DIR}/install/include/utf8cpp/utf8
 )
 
 
@@ -33,8 +50,8 @@ add_custom_target(make_lib_output_dir ALL
     COMMAND ${CMAKE_COMMAND} -E make_directory ${LIB_OUTPUT_DIR}
     )
 
-add_dependencies(antlr4_shared make_lib_output_dir)
+add_dependencies(antlr4_shared make_lib_output_dir utfcpp)
-add_dependencies(antlr4_static make_lib_output_dir)
+add_dependencies(antlr4_static make_lib_output_dir utfcpp)
 
 if(CMAKE_SYSTEM_NAME MATCHES "Linux")
   target_link_libraries(antlr4_shared ${UUID_LIBRARIES})
@@ -102,15 +119,23 @@ set_target_properties(antlr4_static
                                  COMPILE_FLAGS "${disabled_compile_warnings} ${extra_static_compile_flags}")
 
 install(TARGETS antlr4_shared
-        DESTINATION lib 
+        DESTINATION lib
         EXPORT antlr4-targets)
 install(TARGETS antlr4_static
         DESTINATION lib
         EXPORT antlr4-targets)
 
-install(DIRECTORY "${PROJECT_SOURCE_DIR}/runtime/src/" 
+install(DIRECTORY "${PROJECT_SOURCE_DIR}/runtime/src/"
         DESTINATION "include/antlr4-runtime"
-        COMPONENT dev 
+        COMPONENT dev
+        FILES_MATCHING PATTERN "*.h"
+        )
+
+install(FILES "${UTFCPP_DIR}/source/utf8.h"
+        DESTINATION "include/antlr4-runtime")
+install(DIRECTORY "${UTFCPP_DIR}/source/utf8"
+        DESTINATION "include/antlr4-runtime"
+        COMPONENT dev
         FILES_MATCHING PATTERN "*.h"
         )
 

runtime/Cpp/runtime/antlrcpp.xcodeproj/project.pbxproj

@@ -2889,7 +2889,7 @@
 				GCC_WARN_UNUSED_LABEL = YES;
 				GCC_WARN_UNUSED_PARAMETER = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				HEADER_SEARCH_PATHS = src/;
+				HEADER_SEARCH_PATHS = "src/ thirdparty/utfcpp/source/ thirdparty/utfcpp/source/utf8/";
 				MACOSX_DEPLOYMENT_TARGET = 10.9;
 				ONLY_ACTIVE_ARCH = YES;
 				SDKROOT = macosx;
@@ -2945,7 +2945,7 @@
 				GCC_WARN_UNUSED_LABEL = YES;
 				GCC_WARN_UNUSED_PARAMETER = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				HEADER_SEARCH_PATHS = src/;
+				HEADER_SEARCH_PATHS = "src/ thirdparty/utfcpp/source/ thirdparty/utfcpp/source/utf8/";
 				MACOSX_DEPLOYMENT_TARGET = 10.9;
 				SDKROOT = macosx;
 			};

runtime/Cpp/runtime/src/IntStream.h

@@ -27,7 +27,13 @@ namespace antlr4 {
   /// </summary>
   class ANTLR4CPP_PUBLIC IntStream {
   public:
-    static const size_t EOF = static_cast<size_t>(-1); // std::numeric_limits<size_t>::max(); doesn't work in VS 2013
+#if __cplusplus >= 201703L
+    static constexpr size_t EOF = std::numeric_limits<size_t>::max();
+#else
+    enum : size_t {
+      EOF = static_cast<size_t>(-1), // std::numeric_limits<size_t>::max(); doesn't work in VS 2013
+    };
+#endif
 
     /// The value returned by <seealso cref="#LA LA()"/> when the end of the stream is
     /// reached.

runtime/Cpp/runtime/src/Lexer.h

@@ -18,14 +18,27 @@ namespace antlr4 {
   /// of speed.
   class ANTLR4CPP_PUBLIC Lexer : public Recognizer, public TokenSource {
   public:
-    static const size_t DEFAULT_MODE = 0;
+#if __cplusplus >= 201703L
-    static const size_t MORE = static_cast<size_t>(-2);
+    static constexpr size_t DEFAULT_MODE = 0;
-    static const size_t SKIP = static_cast<size_t>(-3);
+    static constexpr size_t MORE = std::numeric_limits<size_t>::max() - 1;
+    static constexpr size_t SKIP = std::numeric_limits<size_t>::max() - 2;
 
-    static const size_t DEFAULT_TOKEN_CHANNEL = Token::DEFAULT_CHANNEL;
+    static constexpr size_t DEFAULT_TOKEN_CHANNEL = Token::DEFAULT_CHANNEL;
-    static const size_t HIDDEN = Token::HIDDEN_CHANNEL;
+    static constexpr size_t HIDDEN = Token::HIDDEN_CHANNEL;
-    static const size_t MIN_CHAR_VALUE = 0;
+    static constexpr size_t MIN_CHAR_VALUE = 0;
-    static const size_t MAX_CHAR_VALUE = 0x10FFFF;
+    static constexpr size_t MAX_CHAR_VALUE = 0x10FFFF;
+#else
+    enum : size_t {
+      DEFAULT_MODE = 0,
+      MORE = static_cast<size_t>(-2), // std::numeric_limits<size_t>::max() - 1; doesn't work in VS 2013
+      SKIP = static_cast<size_t>(-3), // std::numeric_limits<size_t>::max() - 2; doesn't work in VS 2013
+
+      DEFAULT_TOKEN_CHANNEL = Token::DEFAULT_CHANNEL,
+      HIDDEN = Token::HIDDEN_CHANNEL,
+      MIN_CHAR_VALUE = 0,
+      MAX_CHAR_VALUE = 0x10FFFF,
+    };
+#endif
 
     CharStream *_input; // Pure reference, usually from statically allocated instance.
 

runtime/Cpp/runtime/src/Recognizer.h

@@ -11,7 +11,13 @@ namespace antlr4 {
 
   class ANTLR4CPP_PUBLIC Recognizer {
   public:
-    static const size_t EOF = static_cast<size_t>(-1); // std::numeric_limits<size_t>::max(); doesn't work in VS 2013.
+#if __cplusplus >= 201703L
+    static constexpr size_t EOF = std::numeric_limits<size_t>::max();
+#else
+    enum : size_t {
+      EOF = static_cast<size_t>(-1), // std::numeric_limits<size_t>::max(); doesn't work in VS 2013.
+    };
+#endif
 
     Recognizer();
     Recognizer(Recognizer const&) = delete;

runtime/Cpp/runtime/src/Token.h

@@ -14,24 +14,50 @@ namespace antlr4 {
   /// we obtained this token.
   class ANTLR4CPP_PUBLIC Token {
   public:
-    static const size_t INVALID_TYPE = 0;
+#if __cplusplus >= 201703L
+    static constexpr size_t INVALID_TYPE = 0;
+#else
+    enum : size_t {
+      INVALID_TYPE = 0,
+    };
+#endif
 
     /// During lookahead operations, this "token" signifies we hit rule end ATN state
     /// and did not follow it despite needing to.
-    static const size_t EPSILON = static_cast<size_t>(-2);
+#if __cplusplus >= 201703L
-    static const size_t MIN_USER_TOKEN_TYPE = 1;
+    static constexpr size_t EPSILON = std::numeric_limits<size_t>::max() - 1;
-    static const size_t EOF = IntStream::EOF;
+    static constexpr size_t MIN_USER_TOKEN_TYPE = 1;
+    static constexpr size_t EOF = IntStream::EOF;
+#else
+    enum : size_t {
+      EPSILON = static_cast<size_t>(-2), // std::numeric_limits<size_t>::max() - 1; doesn't work in VS 2013
+      MIN_USER_TOKEN_TYPE = 1,
+      EOF = IntStream::EOF,
+    };
+#endif
 
     virtual ~Token();
 
     /// All tokens go to the parser (unless skip() is called in that rule)
     /// on a particular "channel".  The parser tunes to a particular channel
     /// so that whitespace etc... can go to the parser on a "hidden" channel.
-    static const size_t DEFAULT_CHANNEL = 0;
+#if __cplusplus >= 201703L
+    static constexpr size_t DEFAULT_CHANNEL = 0;
+#else
+    enum : size_t {
+      DEFAULT_CHANNEL = 0,
+    };
+#endif
 
     /// Anything on different channel than DEFAULT_CHANNEL is not parsed
     /// by parser.
-    static const size_t HIDDEN_CHANNEL = 1;
+#if __cplusplus >= 201703L
+    static constexpr size_t HIDDEN_CHANNEL = 1;
+#else
+    enum : size_t {
+      HIDDEN_CHANNEL = 1,
+    };
+#endif
 
     /**
      * This is the minimum constant value which can be assigned to a
@@ -44,7 +70,13 @@ namespace antlr4 {
      *
      * @see Token#getChannel()
      */
-    static const size_t MIN_USER_CHANNEL_VALUE = 2;
+#if __cplusplus >= 201703L
+    static constexpr size_t MIN_USER_CHANNEL_VALUE = 2;
+#else
+    enum : size_t {
+      MIN_USER_CHANNEL_VALUE = 2,
+    };
+#endif
 
     /// Get the text of the token.
     virtual std::string getText() const = 0;

runtime/Cpp/runtime/src/TokenStreamRewriter.h

@@ -86,8 +86,15 @@ namespace antlr4 {
   class ANTLR4CPP_PUBLIC TokenStreamRewriter {
   public:
     static const std::string DEFAULT_PROGRAM_NAME;
-    static const size_t PROGRAM_INIT_SIZE = 100;
+#if __cplusplus >= 201703L
-    static const size_t MIN_TOKEN_INDEX = 0;
+    static constexpr size_t PROGRAM_INIT_SIZE = 100;
+    static constexpr size_t MIN_TOKEN_INDEX = 0;
+#else
+    enum : size_t {
+      PROGRAM_INIT_SIZE = 100,
+      MIN_TOKEN_INDEX = 0,
+    };
+#endif
 
     TokenStreamRewriter(TokenStream *tokens);
     virtual ~TokenStreamRewriter();

runtime/Cpp/runtime/src/antlr4-common.h

@@ -8,7 +8,6 @@
 #include <algorithm>
 #include <assert.h>
 #include <atomic>
-#include <codecvt>
 #include <chrono>
 #include <fstream>
 #include <iostream>
@@ -37,6 +36,10 @@
 #include <condition_variable>
 #include <functional>
 
+#ifndef USE_UTF8_INSTEAD_OF_CODECVT
+  #include <codecvt>
+#endif
+
 // Defines for the Guid class and other platform dependent stuff.
 #ifdef _WIN32
   #ifdef _MSC_VER

runtime/Cpp/runtime/src/atn/ATN.h

@@ -12,7 +12,13 @@ namespace atn {
 
   class ANTLR4CPP_PUBLIC ATN {
   public:
-    static const size_t INVALID_ALT_NUMBER = 0;
+#if __cplusplus >= 201703L
+    static constexpr size_t INVALID_ALT_NUMBER = 0;
+#else
+    enum : size_t {
+      INVALID_ALT_NUMBER = 0,
+    };
+#endif
 
     /// Used for runtime deserialization of ATNs from strings.
     ATN();

runtime/Cpp/runtime/src/atn/ATNConfig.cpp

@@ -11,8 +11,6 @@
 
 using namespace antlr4::atn;
 
-const size_t ATNConfig::SUPPRESS_PRECEDENCE_FILTER = 0x40000000;
-
 ATNConfig::ATNConfig(ATNState *state_, size_t alt_, Ref<PredictionContext> const& context_)
   : ATNConfig(state_, alt_, context_, SemanticContext::NONE) {
 }

runtime/Cpp/runtime/src/atn/ATNConfig.h

@@ -114,7 +114,13 @@ namespace atn {
      * {@link #isPrecedenceFilterSuppressed} property as a bit within the
      * existing {@link #reachesIntoOuterContext} field.
      */
-    static const size_t SUPPRESS_PRECEDENCE_FILTER;
+#if __cplusplus >= 201703L
+    static constexpr size_t SUPPRESS_PRECEDENCE_FILTER = 0x40000000;
+#else
+    enum : size_t {
+      SUPPRESS_PRECEDENCE_FILTER = 0x40000000,
+    };
+#endif
   };
 
 } // namespace atn

runtime/Cpp/runtime/src/atn/ATNDeserializer.cpp

@@ -57,8 +57,6 @@ using namespace antlr4;
 using namespace antlr4::atn;
 using namespace antlrcpp;
 
-const size_t ATNDeserializer::SERIALIZED_VERSION = 3;
-
 namespace {
 
 uint32_t deserializeInt32(const std::vector<uint16_t>& data, size_t offset) {

runtime/Cpp/runtime/src/atn/ATNDeserializer.h

@@ -13,7 +13,13 @@ namespace atn {
 
   class ANTLR4CPP_PUBLIC ATNDeserializer {
   public:
-    static const size_t SERIALIZED_VERSION;
+#if __cplusplus >= 201703L
+    static constexpr size_t SERIALIZED_VERSION = 3;
+#else
+    enum : size_t {
+      SERIALIZED_VERSION = 3,
+    };
+#endif
 
     /// This is the current serialized UUID.
     // ml: defined as function to avoid the “static initialization order fiasco”.

runtime/Cpp/runtime/src/atn/ATNState.h

@@ -81,8 +81,15 @@ namespace atn {
 
     ATNState& operator=(ATNState const&) = delete;
 
-    static const size_t INITIAL_NUM_TRANSITIONS = 4;
+#if __cplusplus >= 201703L
-    static const size_t INVALID_STATE_NUMBER = static_cast<size_t>(-1); // std::numeric_limits<size_t>::max();
+    static constexpr size_t INITIAL_NUM_TRANSITIONS = 4;
+    static constexpr size_t INVALID_STATE_NUMBER = std::numeric_limits<size_t>::max();
+#else
+    enum : size_t {
+      INITIAL_NUM_TRANSITIONS = 4,
+      INVALID_STATE_NUMBER = static_cast<size_t>(-1), // std::numeric_limits<size_t>::max(); doesn't work in VS 2013
+    };
+#endif
 
     enum {
       ATN_INVALID_TYPE = 0,

runtime/Cpp/runtime/src/atn/LL1Analyzer.h

@@ -17,7 +17,13 @@ namespace atn {
   public:
     /// Special value added to the lookahead sets to indicate that we hit
     ///  a predicate during analysis if {@code seeThruPreds==false}.
-    static const size_t HIT_PRED = Token::INVALID_TYPE;
+#if __cplusplus >= 201703L
+    static constexpr size_t HIT_PRED = Token::INVALID_TYPE;
+#else
+    enum : size_t {
+      HIT_PRED = Token::INVALID_TYPE,
+    };
+#endif
 
     const atn::ATN &_atn;
 

runtime/Cpp/runtime/src/atn/LexerATNSimulator.h

@@ -38,8 +38,15 @@ namespace atn {
 
 
   public:
-    static const size_t MIN_DFA_EDGE = 0;
+#if __cplusplus >= 201703L
-    static const size_t MAX_DFA_EDGE = 127; // forces unicode to stay in ATN
+    static constexpr size_t MIN_DFA_EDGE = 0;
+    static constexpr size_t MAX_DFA_EDGE = 127; // forces unicode to stay in ATN
+#else
+    enum : size_t {
+      MIN_DFA_EDGE = 0,
+      MAX_DFA_EDGE = 127, // forces unicode to stay in ATN
+    };
+#endif
 
   protected:
     /// <summary>

runtime/Cpp/runtime/src/atn/PredictionContext.h

@@ -30,10 +30,22 @@ namespace atn {
     // ml: originally Integer.MAX_VALUE, which would be -1 for us, but this is already used in places where
     //     -1 is converted to unsigned, so we use a different value here. Any value does the job provided it doesn't
     //     conflict with real return states.
-    static const size_t EMPTY_RETURN_STATE = static_cast<size_t>(-10); // std::numeric_limits<size_t>::max() - 9;
+#if __cplusplus >= 201703L
+    static constexpr size_t EMPTY_RETURN_STATE = std::numeric_limits<size_t>::max() - 9;
+#else
+    enum : size_t {
+      EMPTY_RETURN_STATE = static_cast<size_t>(-10), // std::numeric_limits<size_t>::max() - 9; doesn't work in VS 2013
+    };
+#endif
 
   private:
-    static const size_t INITIAL_HASH = 1;
+#if __cplusplus >= 201703L
+    static constexpr size_t INITIAL_HASH = 1;
+#else
+    enum : size_t {
+      INITIAL_HASH = 1,
+    };
+#endif
 
   public:
     static size_t globalNodeCount;

runtime/Cpp/runtime/src/misc/MurmurHash.h

@@ -13,7 +13,13 @@ namespace misc {
   class ANTLR4CPP_PUBLIC MurmurHash {
 
   private:
-    static const size_t DEFAULT_SEED = 0;
+#if __cplusplus >= 201703L
+    static constexpr size_t DEFAULT_SEED = 0;
+#else
+    enum : size_t {
+      DEFAULT_SEED = 0,
+    };
+#endif
 
     /// Initialize the hash using the default seed value.
     /// Returns the intermediate hash value.

runtime/Cpp/runtime/src/support/StringUtils.cpp

@@ -20,16 +20,26 @@ void replaceAll(std::string& str, std::string const& from, std::string const& to
 }
 
 std::string ws2s(std::wstring const& wstr) {
+#ifndef USE_UTF8_INSTEAD_OF_CODECVT
   std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
   std::string narrow = converter.to_bytes(wstr);
+#else
+  std::string narrow;
+  utf8::utf32to8(wstr.begin(), wstr.end(), std::back_inserter(narrow));
+#endif
 
   return narrow;
 }
 
 std::wstring s2ws(const std::string &str) {
+#ifndef USE_UTF8_INSTEAD_OF_CODECVT
   std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
   std::wstring wide = converter.from_bytes(str);
-  
+#else
+  std::wstring wide;
+  utf8::utf8to32(str.begin(), str.end(), std::back_inserter(wide));
+#endif
+
   return wide;
 }
 

runtime/Cpp/runtime/src/support/StringUtils.h

@@ -7,43 +7,65 @@
 
 #include "antlr4-common.h"
 
+#ifdef USE_UTF8_INSTEAD_OF_CODECVT
+#include "utf8.h"
+#endif
+
 namespace antlrcpp {
 
   // For all conversions utf8 <-> utf32.
+  // I wouldn't prefer wstring_convert because: according to
+  // https://en.cppreference.com/w/cpp/locale/wstring_convert,
+  // wstring_convert is deprecated in C++17.
+  // utfcpp (https://github.com/nemtrif/utfcpp) is a substitution.
+#ifndef USE_UTF8_INSTEAD_OF_CODECVT
   // VS 2015 and VS 2017 have different bugs in std::codecvt_utf8<char32_t> (VS 2013 works fine).
-#if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
+  #if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
-  typedef std::wstring_convert<std::codecvt_utf8<__int32>, __int32> UTF32Converter;
+    typedef std::wstring_convert<std::codecvt_utf8<__int32>, __int32> UTF32Converter;
-#else
+  #else
-  typedef std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> UTF32Converter;
+    typedef std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> UTF32Converter;
+  #endif
 #endif
 
   // The conversion functions fails in VS2017, so we explicitly use a workaround.
   template<typename T>
   inline std::string utf32_to_utf8(T const& data)
   {
-    // Don't make the converter static or we have to serialize access to it.
+    #ifndef USE_UTF8_INSTEAD_OF_CODECVT
-    thread_local UTF32Converter converter;
+      // Don't make the converter static or we have to serialize access to it.
+      thread_local UTF32Converter converter;
 
-    #if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
+      #if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
-      const auto p = reinterpret_cast<const int32_t *>(data.data());
+        const auto p = reinterpret_cast<const int32_t *>(data.data());
-      return converter.to_bytes(p, p + data.size());
+        return converter.to_bytes(p, p + data.size());
+      #else
+        return converter.to_bytes(data);
+      #endif
     #else
-      return converter.to_bytes(data);
+      std::string narrow;
+      utf8::utf32to8(data.begin(), data.end(), std::back_inserter(narrow));
+      return narrow;
     #endif
   }
 
   inline UTF32String utf8_to_utf32(const char* first, const char* last)
   {
-    thread_local UTF32Converter converter;
+    #ifndef USE_UTF8_INSTEAD_OF_CODECVT
+      thread_local UTF32Converter converter;
 
-    #if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
+      #if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
-      auto r = converter.from_bytes(first, last);
+        auto r = converter.from_bytes(first, last);
-      i32string s = reinterpret_cast<const int32_t *>(r.data());
+        i32string s = reinterpret_cast<const int32_t *>(r.data());
+        return s;
+      #else
+        std::u32string s = converter.from_bytes(first, last);
+        return s;
+      #endif
     #else
-      std::u32string s = converter.from_bytes(first, last);
+      UTF32String wide;
+      utf8::utf8to32(first, last, std::back_inserter(wide));
+      return wide;
     #endif
-
-    return s;
   }
 
   void replaceAll(std::string &str, std::string const& from, std::string const& to);

runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java

@@ -19,7 +19,7 @@ public interface Tree {
 	Tree getParent();
 
 	/**
-	 * This method returns whatever object represents the data at this note. For
+	 * This method returns whatever object represents the data at this node. For
 	 * example, for parse trees, the payload can be a {@link Token} representing
 	 * a leaf node or a {@link RuleContext} object representing a rule
 	 * invocation. For abstract syntax trees (ASTs), this is a {@link Token}

runtime/JavaScript/package-lock.json

@@ -2468,7 +2468,8 @@
         "ansi-regex": {
           "version": "2.1.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "aproba": {
           "version": "1.2.0",
@@ -2489,12 +2490,14 @@
         "balanced-match": {
           "version": "1.0.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -2509,17 +2512,20 @@
         "code-point-at": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -2636,7 +2642,8 @@
         "inherits": {
           "version": "2.0.4",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -2648,6 +2655,7 @@
           "version": "1.0.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -2662,6 +2670,7 @@
           "version": "3.0.4",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -2669,12 +2678,14 @@
         "minimist": {
           "version": "0.0.8",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "minipass": {
           "version": "2.9.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.2",
             "yallist": "^3.0.0"
@@ -2693,6 +2704,7 @@
           "version": "0.5.1",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -2782,7 +2794,8 @@
         "number-is-nan": {
           "version": "1.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -2794,6 +2807,7 @@
           "version": "1.4.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -2879,7 +2893,8 @@
         "safe-buffer": {
           "version": "5.1.2",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "safer-buffer": {
           "version": "2.1.2",
@@ -2915,6 +2930,7 @@
           "version": "1.0.2",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",
@@ -2934,6 +2950,7 @@
           "version": "3.0.1",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "ansi-regex": "^2.0.0"
           }
@@ -2977,12 +2994,14 @@
         "wrappy": {
           "version": "1.0.2",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "yallist": {
           "version": "3.1.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         }
       }
     },

runtime/JavaScript/package.json

@@ -27,6 +27,6 @@
     "build": "webpack"
   },
   "engines": {
-    "node": ">=10"
+    "node": ">=14"
   }
 }

runtime/JavaScript/src/antlr4/PredictionContext.js

@@ -4,9 +4,10 @@
  */
 
 const RuleContext = require('./RuleContext');
-const {Hash, Map} = require('./Utils');
+const {Hash, Map, equalArrays} = require('./Utils');
 
 class PredictionContext {
+
 	constructor(cachedHashCode) {
 		this.cachedHashCode = cachedHashCode;
 	}
@@ -83,6 +84,7 @@ function calculateHashString(parent, returnState) {
  * can be used for both lexers and parsers.
  */
 class PredictionContextCache {
+
 	constructor() {
 		this.cache = new Map();
 	}
@@ -115,6 +117,7 @@ class PredictionContextCache {
 
 
 class SingletonPredictionContext extends PredictionContext {
+
 	constructor(parent, returnState) {
 		let hashCode = 0;
 		const hash = new Hash();
@@ -182,6 +185,7 @@ class SingletonPredictionContext extends PredictionContext {
 }
 
 class EmptyPredictionContext extends SingletonPredictionContext {
+
 	constructor() {
 		super(null, PredictionContext.EMPTY_RETURN_STATE);
 	}
@@ -211,6 +215,7 @@ class EmptyPredictionContext extends SingletonPredictionContext {
 PredictionContext.EMPTY = new EmptyPredictionContext();
 
 class ArrayPredictionContext extends PredictionContext {
+
 	constructor(parents, returnStates) {
 		/**
 		 * Parent can be null only if full ctx mode and we make an array
@@ -249,8 +254,8 @@ class ArrayPredictionContext extends PredictionContext {
 		} else if (this.hashCode() !== other.hashCode()) {
 			return false; // can't be same if hash is different
 		} else {
-			return this.returnStates === other.returnStates &&
+			return equalArrays(this.returnStates, other.returnStates) &&
-					this.parents === other.parents;
+				equalArrays(this.parents, other.parents);
 		}
 	}
 
@@ -557,7 +562,7 @@ function mergeArrays(a, b, rootIsWildcard, mergeCache) {
 	while (i < a.returnStates.length && j < b.returnStates.length) {
 		const a_parent = a.parents[i];
 		const b_parent = b.parents[j];
-		if (a.returnStates[i] === b.returnStates[j]) {
+		if (equalArrays(a.returnStates[i], b.returnStates[j])) {
 			// same payload (stack tops are equal), must yield merged singleton
 			const payload = a.returnStates[i];
 			// $+$ = $

runtime/JavaScript/src/antlr4/RuleContext.js

@@ -5,7 +5,6 @@
 
 const {RuleNode} = require('./tree/Tree');
 const {INVALID_INTERVAL} = require('./tree/Tree');
-const INVALID_ALT_NUMBER = require('./atn/ATN').INVALID_ALT_NUMBER || 0; // TODO: solve cyclic dependency to avoid || 0
 const Trees = require('./tree/Trees');
 
 class RuleContext extends RuleNode {
@@ -98,7 +97,10 @@ class RuleContext extends RuleNode {
 	 * option contextSuperClass.
 	 * to set it.
 	 */
-	getAltNumber() { return INVALID_ALT_NUMBER; }
+	getAltNumber() {
+	    // use constant value of ATN.INVALID_ALT_NUMBER to avoid circular dependency
+	    return 0;
+    }
 
 	/**
 	 * Set the outer alternative number for this context node. Default

runtime/JavaScript/src/antlr4/Utils.js

@@ -430,7 +430,7 @@ function equalArrays(a, b) {
     for (let i = 0; i < a.length; i++) {
         if (a[i] == b[i])
             continue;
-        if (!a[i].equals(b[i]))
+        if (!a[i].equals || !a[i].equals(b[i]))
             return false;
     }
     return true;

runtime/JavaScript/src/antlr4/atn/ATN.js

@@ -8,8 +8,8 @@ const {IntervalSet} = require('./../IntervalSet');
 const {Token} = require('./../Token');
 
 class ATN {
-    constructor(grammarType , maxTokenType) {
 
+    constructor(grammarType , maxTokenType) {
         /**
          * Used for runtime deserialization of ATNs from strings
          * The type of the ATN.

runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js

@@ -18,7 +18,7 @@ const PredictionMode = require('./PredictionMode');
 const RuleContext = require('./../RuleContext');
 const ParserRuleContext = require('./../ParserRuleContext');
 const {SemanticContext} = require('./SemanticContext');
-const PredictionContext = require('./../PredictionContext');
+const {PredictionContext} = require('./../PredictionContext');
 const {Interval} = require('./../IntervalSet');
 const {Transition, SetTransition, NotSetTransition, RuleTransition, ActionTransition} = require('./Transition');
 const {NoViableAltException} = require('./../error/Errors');

runtime/JavaScript/src/antlr4/tree/Trees.js

@@ -6,7 +6,6 @@
 const Utils = require('./../Utils');
 const {Token} = require('./../Token');
 const {ErrorNode, TerminalNode, RuleNode} = require('./Tree');
-const ATN = require('./../atn/ATN');
 
 /** A set of utility routines useful for all kinds of ANTLR trees. */
 const Trees = {
@@ -50,7 +49,8 @@ const Trees = {
             if (t instanceof RuleNode) {
                 const context = t.getRuleContext()
                 const altNumber = context.getAltNumber();
-                if ( altNumber != (ATN.INVALID_ALT_NUMBER || 0) ) { // TODO: solve cyclic dependency to avoid || 0
+                // use const value of ATN.INVALID_ALT_NUMBER to avoid circular dependency
+                if ( altNumber != 0 ) {
                     return ruleNames[t.ruleIndex]+":"+altNumber;
                 }
                 return ruleNames[t.ruleIndex];

runtime/Python3/src/antlr4/FileStream.py

@@ -10,7 +10,6 @@
 #
 
 import codecs
-import unittest
 from antlr4.InputStream import InputStream
 
 
@@ -25,10 +24,3 @@ class FileStream(InputStream):
         with open(fileName, 'rb') as file:
             bytes = file.read()
             return codecs.decode(bytes, encoding, errors)
-
-
-class TestFileStream(unittest.TestCase):
-
-    def testStream(self):
-        stream = FileStream(__file__)
-        self.assertTrue(stream.size>0)

runtime/Python3/src/antlr4/InputStream.py

@@ -3,7 +3,6 @@
 # Use of this file is governed by the BSD 3-clause license that
 # can be found in the LICENSE.txt file in the project root.
 #
-import unittest
 
 
 #
@@ -85,20 +84,3 @@ class InputStream (object):
 
     def __str__(self):
         return self.strdata
-
-
-class TestInputStream(unittest.TestCase):
-
-    def testStream(self):
-        stream = InputStream("abcde")
-        self.assertEqual(0, stream.index)
-        self.assertEqual(5, stream.size)
-        self.assertEqual(ord("a"), stream.LA(1))
-        stream.consume()
-        self.assertEqual(1, stream.index)
-        stream.seek(5)
-        self.assertEqual(Token.EOF, stream.LA(1))
-        self.assertEqual("bcd", stream.getText(1, 3))
-        stream.reset()
-        self.assertEqual(0, stream.index)
-

runtime/Python3/src/antlr4/IntervalSet.py

@@ -5,7 +5,6 @@
 #
 
 from io import StringIO
-import unittest
 from antlr4.Token import Token
 
 # need forward declarations
@@ -178,105 +177,3 @@ class IntervalSet(object):
             if a<len(symbolicNames):
                 return symbolicNames[a]
             return "<UNKNOWN>"
-
-
-class TestIntervalSet(unittest.TestCase):
-
-    def testEmpty(self):
-        s = IntervalSet()
-        self.assertIsNone(s.intervals)
-        self.assertFalse(30 in s)
-
-    def testOne(self):
-        s = IntervalSet()
-        s.addOne(30)
-        self.assertTrue(30 in s)
-        self.assertFalse(29 in s)
-        self.assertFalse(31 in s)
-
-    def testTwo(self):
-        s = IntervalSet()
-        s.addOne(30)
-        s.addOne(40)
-        self.assertTrue(30 in s)
-        self.assertTrue(40 in s)
-        self.assertFalse(35 in s)
-
-    def testRange(self):
-        s = IntervalSet()
-        s.addRange(range(30,41))
-        self.assertTrue(30 in s)
-        self.assertTrue(40 in s)
-        self.assertTrue(35 in s)
-
-    def testDistinct1(self):
-        s = IntervalSet()
-        s.addRange(range(30,32))
-        s.addRange(range(40,42))
-        self.assertEquals(2,len(s.intervals))
-        self.assertTrue(30 in s)
-        self.assertTrue(40 in s)
-        self.assertFalse(35 in s)
-
-    def testDistinct2(self):
-        s = IntervalSet()
-        s.addRange(range(40,42))
-        s.addRange(range(30,32))
-        self.assertEquals(2,len(s.intervals))
-        self.assertTrue(30 in s)
-        self.assertTrue(40 in s)
-        self.assertFalse(35 in s)
-
-    def testContiguous1(self):
-        s = IntervalSet()
-        s.addRange(range(30,36))
-        s.addRange(range(36,41))
-        self.assertEquals(1,len(s.intervals))
-        self.assertTrue(30 in s)
-        self.assertTrue(40 in s)
-        self.assertTrue(35 in s)
-
-    def testContiguous2(self):
-        s = IntervalSet()
-        s.addRange(range(36,41))
-        s.addRange(range(30,36))
-        self.assertEquals(1,len(s.intervals))
-        self.assertTrue(30 in s)
-        self.assertTrue(40 in s)
-
-    def testOverlapping1(self):
-        s = IntervalSet()
-        s.addRange(range(30,40))
-        s.addRange(range(35,45))
-        self.assertEquals(1,len(s.intervals))
-        self.assertTrue(30 in s)
-        self.assertTrue(44 in s)
-
-    def testOverlapping2(self):
-        s = IntervalSet()
-        s.addRange(range(35,45))
-        s.addRange(range(30,40))
-        self.assertEquals(1,len(s.intervals))
-        self.assertTrue(30 in s)
-        self.assertTrue(44 in s)
-
-    def testOverlapping3(self):
-        s = IntervalSet()
-        s.addRange(range(30,32))
-        s.addRange(range(40,42))
-        s.addRange(range(50,52))
-        s.addRange(range(20,61))
-        self.assertEquals(1,len(s.intervals))
-        self.assertTrue(20 in s)
-        self.assertTrue(60 in s)
-
-    def testComplement(self):
-        s = IntervalSet()
-        s.addRange(range(10,21))
-        c = s.complement(1,100)
-        self.assertTrue(1 in c)
-        self.assertTrue(100 in c)
-        self.assertTrue(10 not in c)
-        self.assertTrue(20 not in c)
-
-

runtime/Python3/src/antlr4/Lexer.py

@@ -9,8 +9,12 @@
 #  of speed.
 #/
 from io import StringIO
-from typing.io import TextIO
+
 import sys
+if sys.version_info[1] > 5:
+    from typing import TextIO
+else:
+    from typing.io import TextIO
 from antlr4.CommonTokenFactory import CommonTokenFactory
 from antlr4.atn.LexerATNSimulator import LexerATNSimulator
 from antlr4.InputStream import InputStream

runtime/Python3/src/antlr4/Parser.py

@@ -3,7 +3,10 @@
 # Use of this file is governed by the BSD 3-clause license that
 # can be found in the LICENSE.txt file in the project root.
 import sys
-from typing.io import TextIO
+if sys.version_info[1] > 5:
+    from typing import TextIO
+else:
+    from typing.io import TextIO
 from antlr4.BufferedTokenStream import TokenStream
 from antlr4.CommonTokenFactory import TokenFactory
 from antlr4.error.ErrorStrategy import DefaultErrorStrategy

runtime/Python3/src/antlr4/PredictionContext.py

@@ -158,7 +158,7 @@ class SingletonPredictionContext(PredictionContext):
 class EmptyPredictionContext(SingletonPredictionContext):
 
     def __init__(self):
-        super().__init__(None, self.EMPTY_RETURN_STATE)
+        super().__init__(None, PredictionContext.EMPTY_RETURN_STATE)
 
     def isEmpty(self):
         return True

runtime/Python3/src/antlr4/Recognizer.py

@@ -144,17 +144,3 @@ class Recognizer(object):
         self._stateNumber = atnState
 
 del RecognitionException
-
-import unittest
-class Test(unittest.TestCase):
-
-    def testVersion(self):
-        major, minor = Recognizer().extractVersion("1.2")
-        self.assertEqual("1", major)
-        self.assertEqual("2", minor)
-        major, minor = Recognizer().extractVersion("1.2.3")
-        self.assertEqual("1", major)
-        self.assertEqual("2", minor)
-        major, minor = Recognizer().extractVersion("1.2-snapshot")
-        self.assertEqual("1", major)
-        self.assertEqual("2", minor)

runtime/Python3/test/TestFileStream.py

@@ -0,0 +1,8 @@
+import unittest
+from antlr4.FileStream import FileStream
+
+
+class TestFileStream(unittest.TestCase):
+    def testStream(self):
+        stream = FileStream(__file__)
+        self.assertTrue(stream.size > 0)

runtime/Python3/test/TestInputStream.py

@@ -0,0 +1,19 @@
+import unittest
+from antlr4.Token import Token
+from antlr4.InputStream import InputStream
+
+
+class TestInputStream(unittest.TestCase):
+
+    def testStream(self):
+        stream = InputStream("abcde")
+        self.assertEqual(0, stream.index)
+        self.assertEqual(5, stream.size)
+        self.assertEqual(ord("a"), stream.LA(1))
+        stream.consume()
+        self.assertEqual(1, stream.index)
+        stream.seek(5)
+        self.assertEqual(Token.EOF, stream.LA(1))
+        self.assertEqual("bcd", stream.getText(1, 3))
+        stream.reset()
+        self.assertEqual(0, stream.index)

runtime/Python3/test/TestIntervalSet.py

@@ -0,0 +1,101 @@
+import unittest
+from antlr4.IntervalSet import IntervalSet
+
+
+class TestIntervalSet(unittest.TestCase):
+    def testEmpty(self):
+        s = IntervalSet()
+        self.assertIsNone(s.intervals)
+        self.assertFalse(30 in s)
+
+    def testOne(self):
+        s = IntervalSet()
+        s.addOne(30)
+        self.assertTrue(30 in s)
+        self.assertFalse(29 in s)
+        self.assertFalse(31 in s)
+
+    def testTwo(self):
+        s = IntervalSet()
+        s.addOne(30)
+        s.addOne(40)
+        self.assertTrue(30 in s)
+        self.assertTrue(40 in s)
+        self.assertFalse(35 in s)
+
+    def testRange(self):
+        s = IntervalSet()
+        s.addRange(range(30,41))
+        self.assertTrue(30 in s)
+        self.assertTrue(40 in s)
+        self.assertTrue(35 in s)
+
+    def testDistinct1(self):
+        s = IntervalSet()
+        s.addRange(range(30,32))
+        s.addRange(range(40,42))
+        self.assertEquals(2,len(s.intervals))
+        self.assertTrue(30 in s)
+        self.assertTrue(40 in s)
+        self.assertFalse(35 in s)
+
+    def testDistinct2(self):
+        s = IntervalSet()
+        s.addRange(range(40,42))
+        s.addRange(range(30,32))
+        self.assertEquals(2,len(s.intervals))
+        self.assertTrue(30 in s)
+        self.assertTrue(40 in s)
+        self.assertFalse(35 in s)
+
+    def testContiguous1(self):
+        s = IntervalSet()
+        s.addRange(range(30,36))
+        s.addRange(range(36,41))
+        self.assertEquals(1,len(s.intervals))
+        self.assertTrue(30 in s)
+        self.assertTrue(40 in s)
+        self.assertTrue(35 in s)
+
+    def testContiguous2(self):
+        s = IntervalSet()
+        s.addRange(range(36,41))
+        s.addRange(range(30,36))
+        self.assertEquals(1,len(s.intervals))
+        self.assertTrue(30 in s)
+        self.assertTrue(40 in s)
+
+    def testOverlapping1(self):
+        s = IntervalSet()
+        s.addRange(range(30,40))
+        s.addRange(range(35,45))
+        self.assertEquals(1,len(s.intervals))
+        self.assertTrue(30 in s)
+        self.assertTrue(44 in s)
+
+    def testOverlapping2(self):
+        s = IntervalSet()
+        s.addRange(range(35,45))
+        s.addRange(range(30,40))
+        self.assertEquals(1,len(s.intervals))
+        self.assertTrue(30 in s)
+        self.assertTrue(44 in s)
+
+    def testOverlapping3(self):
+        s = IntervalSet()
+        s.addRange(range(30,32))
+        s.addRange(range(40,42))
+        s.addRange(range(50,52))
+        s.addRange(range(20,61))
+        self.assertEquals(1,len(s.intervals))
+        self.assertTrue(20 in s)
+        self.assertTrue(60 in s)
+
+    def testComplement(self):
+        s = IntervalSet()
+        s.addRange(range(10,21))
+        c = s.complement(1,100)
+        self.assertTrue(1 in c)
+        self.assertTrue(100 in c)
+        self.assertTrue(10 not in c)
+        self.assertTrue(20 not in c)

runtime/Python3/test/TestRecognizer.py

@@ -0,0 +1,15 @@
+import unittest
+from antlr4.Recognizer import Recognizer
+
+
+class TestRecognizer(unittest.TestCase):
+    def testVersion(self):
+        major, minor = Recognizer().extractVersion("1.2")
+        self.assertEqual("1", major)
+        self.assertEqual("2", minor)
+        major, minor = Recognizer().extractVersion("1.2.3")
+        self.assertEqual("1", major)
+        self.assertEqual("2", minor)
+        major, minor = Recognizer().extractVersion("1.2-snapshot")
+        self.assertEqual("1", major)
+        self.assertEqual("2", minor)

runtime/Python3/test/run.py

@@ -4,5 +4,9 @@ src_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__
 sys.path.insert(0,src_path)
 from TestTokenStreamRewriter import TestTokenStreamRewriter
 from xpathtest import XPathTest
+from TestFileStream import TestFileStream
+from TestInputStream import TestInputStream
+from TestIntervalSet import TestIntervalSet
+from TestRecognizer import TestRecognizer
 import unittest
-unittest.main()
+unittest.main()

runtime/Swift/Sources/Antlr4/tree/ParseTreeProperty.swift

@@ -0,0 +1,15 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+import Foundation
+
+public class ParseTreeProperty<V> {
+  var annotations = Dictionary<ObjectIdentifier, V>()
+  
+  public init() {}
+  
+  open func get(_ node: ParseTree) -> V? { return annotations[ObjectIdentifier(node)] }
+  open func put(_ node: ParseTree, _ value: V) { annotations[ObjectIdentifier(node)] = value }
+  open func removeFrom(_ node: ParseTree) { annotations.removeValue(forKey: ObjectIdentifier(node)) }
+}

tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg

@@ -464,7 +464,11 @@ RuleFunction(currentRule, args, code, locals, ruleCtx, altLabelCtxs, namedAction
   <namedActions.init>
   <locals; separator = "\n">
 
+#if __cplusplus > 201703L
+  auto onExit = finally([=, this] {
+#else
   auto onExit = finally([=] {
+#endif
   <finallyAction>
     exitRule();
   });
@@ -517,7 +521,11 @@ LeftRecursiveRuleFunction(currentRule, args, code, locals, ruleCtx, altLabelCtxs
     <namedActions.init>
 <! TODO: untested !>    <locals; separator = "\n">
 
+#if __cplusplus > 201703L
+  auto onExit = finally([=, this] {
+#else
   auto onExit = finally([=] {
+#endif
   <if (finallyAction)><finallyAction><endif>
     unrollRecursionContexts(parentContext);
   });
@@ -539,7 +547,7 @@ LeftRecursiveRuleFunction(currentRule, args, code, locals, ruleCtx, altLabelCtxs
 StructDeclHeader(struct, ctorAttrs, attrs, getters, dispatchMethods, interfaces, extensionMembers) ::= <<
 class <file.exportMacro> <struct.name> : public <if (contextSuperClass)><contextSuperClass><else>antlr4::ParserRuleContext<endif><if(interfaces)>, <interfaces; separator=", "><endif> {
 public:
-  <attrs: {a | <a>;}; separator="\n">
+  <attrs: {a | <a>;}; separator = "\n">
   <if (ctorAttrs)><struct.name>(antlr4::ParserRuleContext *parent, size_t invokingState);<endif>
   <struct.name>(antlr4::ParserRuleContext *parent, size_t invokingState<ctorAttrs: {a | , <a>}>);
 <if (struct.provideCopyFrom)> <! don't need copy unless we have subclasses !>
@@ -593,7 +601,7 @@ class <file.exportMacro> <struct.name> : public <currentRule.name; format = "cap
 public:
   <struct.name>(<currentRule.name; format = "cap">Context *ctx);
 
-  <if (attrs)><attrs: {a | <a>}; separator = "\n"><endif>
+  <if (attrs)><attrs: {a | <a>;}; separator = "\n"><endif>
   <getters: {g | <g>}; separator = "\n">
   <dispatchMethods; separator = "\n">
 };

tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg

@@ -48,12 +48,12 @@ javascriptTypeInitMap ::= [
 
 ParserFile(file, parser, namedActions, contextSuperClass) ::= <<
 <fileHeader(file.grammarFileName, file.ANTLRVersion)>
-var antlr4 = require('antlr4/index');
+import antlr4 from 'antlr4';
 <if(file.genListener)>
-var <file.grammarName>Listener = require('./<file.grammarName>Listener').<file.grammarName>Listener;
+import <file.grammarName>Listener from './<file.grammarName>Listener.js';
 <endif>
 <if(file.genVisitor)>
-var <file.grammarName>Visitor = require('./<file.grammarName>Visitor').<file.grammarName>Visitor;
+import <file.grammarName>Visitor from './<file.grammarName>Visitor.js';
 <endif>
 
 <namedActions.header>
@@ -62,54 +62,43 @@ var <file.grammarName>Visitor = require('./<file.grammarName>Visitor').<file.gra
 
 ListenerFile(file, header, namedActions) ::= <<
 <fileHeader(file.grammarFileName, file.ANTLRVersion)>
-var antlr4 = require('antlr4/index');
+import antlr4 from 'antlr4';
 
 // This class defines a complete listener for a parse tree produced by <file.parserName>.
-function <file.grammarName>Listener() {
+export default class <file.grammarName>Listener extends antlr4.tree.ParseTreeListener {
-	return this;
-}
-
-<file.grammarName>Listener.prototype = Object.create(antlr4.tree.ParseTreeListener.prototype);
-<file.grammarName>Listener.prototype.constructor = <file.grammarName>Listener;
 
 <file.listenerNames:{lname |
-// Enter a parse tree produced by <file.parserName>#<lname>.
+	// Enter a parse tree produced by <file.parserName>#<lname>.
-<file.grammarName>Listener.prototype.enter<lname; format="cap"> = function(ctx) {
+	enter<lname; format="cap">(ctx) {
-\};
+	\}
 
-// Exit a parse tree produced by <file.parserName>#<lname>.
+	// Exit a parse tree produced by <file.parserName>#<lname>.
-<file.grammarName>Listener.prototype.exit<lname; format="cap"> = function(ctx) {
+	exit<lname; format="cap">(ctx) {
-\};
+	\}
 
 }; separator="\n">
 
-exports.<file.grammarName>Listener = <file.grammarName>Listener;
+}
 >>
 
 
 VisitorFile(file, header, namedActions) ::= <<
 <fileHeader(file.grammarFileName, file.ANTLRVersion)>
-var antlr4 = require('antlr4/index');
+import antlr4 from 'antlr4';
 
 // This class defines a complete generic visitor for a parse tree produced by <file.parserName>.
 
-function <file.grammarName>Visitor() {
+export default class <file.grammarName>Visitor extends antlr4.tree.ParseTreeVisitor {
-	antlr4.tree.ParseTreeVisitor.call(this);
-	return this;
-}
-
-<file.grammarName>Visitor.prototype = Object.create(antlr4.tree.ParseTreeVisitor.prototype);
-<file.grammarName>Visitor.prototype.constructor = <file.grammarName>Visitor;
 
 <file.visitorNames:{lname |
-// Visit a parse tree produced by <file.parserName>#<lname>.
+	// Visit a parse tree produced by <file.parserName>#<lname>.
-<file.grammarName>Visitor.prototype.visit<lname; format="cap"> = function(ctx) {
+	visit<lname; format="cap">(ctx) {
-  return this.visitChildren(ctx);
+	  return this.visitChildren(ctx);
-\};
+	\}
 
 }; separator="\n">
 
-exports.<file.grammarName>Visitor = <file.grammarName>Visitor;
+}
 >>
 
 
@@ -120,32 +109,30 @@ fileHeader(grammarFileName, ANTLRVersion) ::= <<
 
 Parser(parser, funcs, atn, sempredFuncs, superClass) ::= <<
 <if(superClass)>
-var <superClass> = require('./<superClass>').<superClass>;
+import <superClass> from './<superClass>.js';
 <endif>
 
-var grammarFileName = "<parser.grammarFileName; format="java-escape">";
-
 <atn>
 
-var atn = new antlr4.atn.ATNDeserializer().deserialize(serializedATN);
+const atn = new antlr4.atn.ATNDeserializer().deserialize(serializedATN);
 
-var decisionsToDFA = atn.decisionToState.map( function(ds, index) { return new antlr4.dfa.DFA(ds, index); });
+const decisionsToDFA = atn.decisionToState.map( (ds, index) => new antlr4.dfa.DFA(ds, index) );
 
-var sharedContextCache = new antlr4.PredictionContextCache();
+const sharedContextCache = new antlr4.PredictionContextCache();
 
-var literalNames = [ <parser.literalNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
+export default class <parser.name> extends <superClass; null="antlr4.Parser"> {
 
-var symbolicNames = [ <parser.symbolicNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
+    static grammarFileName = "<parser.grammarFileName; format="java-escape">";
+    static literalNames = [ <parser.literalNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
+    static symbolicNames = [ <parser.symbolicNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
+    static ruleNames = [ <parser.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor> ];
 
-var ruleNames =  [ <parser.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor> ];
-
-class <parser.name> extends <superClass; null="antlr4.Parser"> {
     constructor(input) {
         super(input);
         this._interp = new antlr4.atn.ParserATNSimulator(this, atn, decisionsToDFA, sharedContextCache);
-        this.ruleNames = ruleNames;
+        this.ruleNames = <parser.name>.ruleNames;
-        this.literalNames = literalNames;
+        this.literalNames = <parser.name>.literalNames;
-        this.symbolicNames = symbolicNames;
+        this.symbolicNames = <parser.name>.symbolicNames;
         <namedActions.members>
     }
 
@@ -161,10 +148,13 @@ class <parser.name> extends <superClass; null="antlr4.Parser"> {
         default:
             throw "No predicate with index:" + ruleIndex;
        }
-    };
+    }
 
     <sempredFuncs.values; separator="\n">
     <endif>
+
+	<funcs; separator="\n">
+
 }
 
 <parser.name>.EOF = antlr4.Token.EOF;
@@ -174,12 +164,19 @@ class <parser.name> extends <superClass; null="antlr4.Parser"> {
 
 <parser.rules:{r | <parser.name>.RULE_<r.name> = <r.index>;}; separator="\n", wrap, anchor>
 
-<funcs; separator="\n">
+<funcs:{f | <ruleContexts(f)>}; separator="\n">
 
-module.exports.<parser.name> = <parser.name>;
+<! Define fields of this parser to export the context classes !>
+<parser.funcs:{f | <parser.name>.<f.ctxType> = <f.ctxType>; }; separator="\n">
 
 >>
 
+ruleContexts(currentRule) ::= <<
+<currentRule.ruleCtx>
+
+<currentRule.altLabelCtxs:{l | <currentRule.altLabelCtxs.(l)>}; separator="\n">
+>>
+
 dumpActions(recog, argFuncs, actionFuncs, sempredFuncs) ::= <<
 <if(actionFuncs)>
 <lexer.name>.prototype.action = function(localctx, ruleIndex, actionIndex) {
@@ -244,16 +241,9 @@ RuleSempredFunction(r, actions) ::= <<
 
 RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,finallyAction,postamble,exceptions) ::= <<
 
-<ruleCtx>
 
-<altLabelCtxs:{l | <altLabelCtxs.(l)>}; separator="\n">
+<currentRule.name>(<currentRule.args:{a | <a.name>}; separator=", ">) {
-
+    let localctx = new <currentRule.ctxType>(this, this._ctx, this.state<currentRule.args:{a | , <a.name>}>);
-<! Define fields of this parser to export the context classes !>
-<parser.name>.<currentRule.ctxType> = <currentRule.ctxType>;
-
-<parser.name>.prototype.<currentRule.name> = function(<currentRule.args:{a | <a.name>}; separator=", ">) {
-
-    var localctx = new <currentRule.ctxType>(this, this._ctx, this.state<currentRule.args:{a | , <a.name>}>);
     this.enterRule(localctx, <currentRule.startState>, <parser.name>.RULE_<currentRule.name>);
     <namedActions.init>
     <locals; separator="\n">
@@ -277,7 +267,7 @@ RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,fina
         this.exitRule();
     }
     return localctx;
-};
+}
 
 >>
 
@@ -285,18 +275,15 @@ LeftRecursiveRuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,
 	namedActions,finallyAction,postamble) ::=
 <<
 
-<ruleCtx>
+<currentRule.name>(_p<if(currentRule.args)>, <args:{a | , <a>}><endif>) {
-<altLabelCtxs:{l | <altLabelCtxs.(l)>}; separator="\n">
-
-<parser.name>.prototype.<currentRule.name> = function(_p<if(currentRule.args)>, <args:{a | , <a>}><endif>) {
 	if(_p===undefined) {
 	    _p = 0;
 	}
-    var _parentctx = this._ctx;
+    const _parentctx = this._ctx;
-    var _parentState = this.state;
+    const _parentState = this.state;
-    var localctx = new <currentRule.ctxType>(this, this._ctx, _parentState<args:{a | , <a.name>}>);
+    let localctx = new <currentRule.ctxType>(this, this._ctx, _parentState<args:{a | , <a.name>}>);
-    var _prevctx = localctx;
+    let _prevctx = localctx;
-    var _startState = <currentRule.startState>;
+    const _startState = <currentRule.startState>;
     this.enterRecursionRule(localctx, <currentRule.startState>, <parser.name>.RULE_<currentRule.name>, _p);
     <namedActions.init>
     <locals; separator="\n">
@@ -317,7 +304,7 @@ LeftRecursiveRuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,
         this.unrollRecursionContexts(_parentctx)
     }
     return localctx;
-};
+}
 
 >>
 
@@ -424,7 +411,7 @@ if(la_===<i><if(!choice.ast.greedy)>+1<endif>) {
 StarBlock(choice, alts, sync, iteration) ::= <<
 this.state = <choice.stateNumber>;
 this._errHandler.sync(this);
-var _alt = this._interp.adaptivePredict(this._input,<choice.decision>,this._ctx)
+let _alt = this._interp.adaptivePredict(this._input,<choice.decision>,this._ctx)
 while(_alt!=<choice.exitAlt> && _alt!=antlr4.atn.ATN.INVALID_ALT_NUMBER) {
     if(_alt===1<if(!choice.ast.greedy)>+1<endif>) {
         <iteration>
@@ -440,7 +427,7 @@ while(_alt!=<choice.exitAlt> && _alt!=antlr4.atn.ATN.INVALID_ALT_NUMBER) {
 PlusBlock(choice, alts, error) ::= <<
 this.state = <choice.blockStartStateNumber>; <! alt block decision !>
 this._errHandler.sync(this);
-var _alt = 1<if(!choice.ast.greedy)>+1<endif>;
+let _alt = 1<if(!choice.ast.greedy)>+1<endif>;
 do {
 	switch (_alt) {
 	<alts:{alt|
@@ -592,8 +579,8 @@ ThisRulePropertyRef_text(r)	 ::= "this._input.getText(new antlr4.Interval(localc
 ThisRulePropertyRef_ctx(r)	 ::= "localctx"
 ThisRulePropertyRef_parser(r)	 ::= "this"
 
-NonLocalAttrRef(s)		 	 ::= "getInvokingContext(<s.ruleIndex>).<s.name>"
+NonLocalAttrRef(s)		 	 ::= "this.getInvokingContext(<s.ruleIndex>).<s.name>"
-SetNonLocalAttr(s, rhsChunks)  ::= "getInvokingContext(<s.ruleIndex>).<s.name> = <rhsChunks>"
+SetNonLocalAttr(s, rhsChunks)  ::= "this.getInvokingContext(<s.ruleIndex>).<s.name> = <rhsChunks>"
 
 AddToLabelList(a) ::= "<ctx(a.label)>.<a.listName>.push(<labelref(a.label)>);"
 
@@ -604,15 +591,16 @@ RuleContextDecl(r) ::= "this.<r.name> = null; // <r.ctxName>"
 RuleContextListDecl(rdecl) ::= "this.<rdecl.name> = []; // of <rdecl.ctxName>s"
 
 ContextTokenGetterDecl(t)      ::= <<
-<t.name> = function() {
+<t.name>() {
     return this.getToken(<parser.name>.<t.name>, 0);
 };
 >>
 
 // should never be called
 ContextTokenListGetterDecl(t)  ::= <<
-def <t.name>_list(self):
+<t.name>_list() {
-    return self.getTokens(<parser.name>.<t.name>)
+    return this.getTokens(<parser.name>.<t.name>);
+}
 >>
 
 ContextTokenListIndexedGetterDecl(t)  ::= <<
@@ -630,16 +618,16 @@ ContextTokenListIndexedGetterDecl(t)  ::= <<
 >>
 
 ContextRuleGetterDecl(r)       ::= <<
-<r.name> = function() {
+<r.name>() {
     return this.getTypedRuleContext(<r.ctxName>,0);
 };
 >>
 
 // should never be called
 ContextRuleListGetterDecl(r)   ::= <<
-def <r.name>_list(self):
+<r.name>_list() {
-    return self.getTypedRuleContexts(<parser.name>.<r.ctxName>)
+    return this.getTypedRuleContexts(<parser.name>.<r.ctxName>);
-
+}
 >>
 
 ContextRuleListIndexedGetterDecl(r)   ::= <<
@@ -672,6 +660,7 @@ CaptureNextTokenType(d) ::= "<d.varName> = this._input.LA(1);"
 
 StructDecl(struct,ctorAttrs,attrs,getters,dispatchMethods,interfaces,extensionMembers) ::= <<
 class <struct.name> extends <if(contextSuperClass)><contextSuperClass><else>antlr4.ParserRuleContext<endif> {
+
     constructor(parser, parent, invokingState<struct.ctorAttrs:{a | , <a.name>}>) {
         if(parent===undefined) {
             parent = null;
@@ -685,58 +674,59 @@ class <struct.name> extends <if(contextSuperClass)><contextSuperClass><else>antl
         <attrs:{a | <a>}; separator="\n">
         <struct.ctorAttrs:{a | this.<a.name> = <a.name> || null;}; separator="\n">
     }
+
+	<getters:{g | <g>}; separator="\n\n">
+
+	<if(struct.provideCopyFrom)> <! don't need copy unless we have subclasses !>
+	copyFrom(ctx) {
+		super.copyFrom(ctx);
+		<struct.attrs:{a | this.<a.name> = ctx.<a.name>;}; separator="\n">
+	}
+	<endif>
+	<dispatchMethods; separator="\n">
+	<extensionMembers; separator="\n">
+
 }
 
-<getters:{g | <struct.name>.prototype.<g>}; separator="\n\n">
-
-<if(struct.provideCopyFrom)> <! don't need copy unless we have subclasses !>
-<struct.name>.prototype.copyFrom = function(ctx) {
-    <if(contextSuperClass)><contextSuperClass><else>antlr4.ParserRuleContext<endif>.prototype.copyFrom.call(this, ctx);
-    <struct.attrs:{a | this.<a.name> = ctx.<a.name>;}; separator="\n">
-};
-<endif>
-<dispatchMethods; separator="\n">
-<extensionMembers; separator="\n">
-
 >>
 
 AltLabelStructDecl(struct,attrs,getters,dispatchMethods) ::= <<
-class <struct.name> extends <currentRule.name; format="cap">Context {
+class <struct.name> extends <struct.parentRule; format="cap">Context {
+
     constructor(parser, ctx) {
         super(parser);
         <attrs:{a | <a>;}; separator="\n">
-        <currentRule.name; format="cap">Context.prototype.copyFrom.call(this, ctx);
+        super.copyFrom(ctx);
     }
-}
 
-<struct.name>.prototype = Object.create(<currentRule.name; format="cap">Context.prototype);
+	<getters:{g | <g>}; separator="\n\n">
-<struct.name>.prototype.constructor = <struct.name>;
+
+	<dispatchMethods; separator="\n">
+
+}
 
 <! Define fields of this parser to export this struct/context class !>
 <parser.name>.<struct.name> = <struct.name>;
 
-<getters:{g | <struct.name>.prototype.<g>}; separator="\n\n">
-<dispatchMethods; separator="\n">
-
 >>
 
 ListenerDispatchMethod(method) ::= <<
-<struct.name>.prototype.<if(method.isEnter)>enter<else>exit<endif>Rule = function(listener) {
+<if(method.isEnter)>enter<else>exit<endif>Rule(listener) {
     if(listener instanceof <parser.grammarName>Listener ) {
         listener.<if(method.isEnter)>enter<else>exit<endif><struct.derivedFromName; format="cap">(this);
 	}
-};
+}
 
 >>
 
 VisitorDispatchMethod(method) ::= <<
-<struct.name>.prototype.accept = function(visitor) {
+accept(visitor) {
     if ( visitor instanceof <parser.grammarName>Visitor ) {
         return visitor.visit<struct.derivedFromName; format="cap">(this);
     } else {
         return visitor.visitChildren(this);
     }
-};
+}
 
 >>
 
@@ -787,7 +777,7 @@ _prevctx = localctx;
 
 LexerFile(lexerFile, lexer, namedActions) ::= <<
 <fileHeader(lexerFile.grammarFileName, lexerFile.ANTLRVersion)>
-var antlr4 = require('antlr4/index');
+import antlr4 from 'antlr4';
 
 <namedActions.header>
 
@@ -797,32 +787,28 @@ var antlr4 = require('antlr4/index');
 
 Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass) ::= <<
 <if(superClass)>
-var <superClass> = require('./<superClass>').<superClass>;
+import <superClass> from './<superClass>.js';
 <endif>
 
 <atn>
 
-var atn = new antlr4.atn.ATNDeserializer().deserialize(serializedATN);
+const atn = new antlr4.atn.ATNDeserializer().deserialize(serializedATN);
 
-var decisionsToDFA = atn.decisionToState.map( function(ds, index) { return new antlr4.dfa.DFA(ds, index); });
+const decisionsToDFA = atn.decisionToState.map( (ds, index) => new antlr4.dfa.DFA(ds, index) );
+
+export default class <lexer.name> extends <if(superClass)><superClass><else>antlr4.Lexer<endif> {
+
+    static grammarFileName = "<lexer.grammarFileName>";
+    static channelNames = [ "DEFAULT_TOKEN_CHANNEL", "HIDDEN"<if (lexer.channels)>, <lexer.channels:{c| "<c>"}; separator=", ", wrap, anchor><endif> ];
+	static modeNames = [ <lexer.modes:{m| "<m>"}; separator=", ", wrap, anchor> ];
+	static literalNames = [ <lexer.literalNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
+	static symbolicNames = [ <lexer.symbolicNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
+	static ruleNames = [ <lexer.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor> ];
 
-class <lexer.name> extends <if(superClass)><superClass><else>antlr4.Lexer<endif> {
     constructor(input) {
         super(input)
         this._interp = new antlr4.atn.LexerATNSimulator(this, atn, decisionsToDFA, new antlr4.PredictionContextCache());
         <namedActions.members>
-
-        this.channelNames = [ "DEFAULT_TOKEN_CHANNEL", "HIDDEN"<if (lexer.channels)>, <lexer.channels:{c| "<c>"}; separator=", ", wrap, anchor><endif> ];
-
-        this.modeNames = [ <lexer.modes:{m| "<m>"}; separator=", ", wrap, anchor> ];
-
-        this.literalNames = [ <lexer.literalNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
-
-        this.symbolicNames = [ <lexer.symbolicNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
-
-        this.ruleNames = [ <lexer.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor> ];
-
-        this.grammarFileName = "<lexer.grammarFileName>";
     }
 
     get atn() {
@@ -844,14 +830,13 @@ class <lexer.name> extends <if(superClass)><superClass><else>antlr4.Lexer<endif>
 
 <dumpActions(lexer, "", actionFuncs, sempredFuncs)>
 
-module.exports.<lexer.name> = <lexer.name>;
 
 >>
 
 SerializedATN(model) ::= <<
 <! only one segment, can be inlined !>
 
-var serializedATN = ["<model.serialized; wrap={",<\n>    "}>"].join("");
+const serializedATN = ["<model.serialized; wrap={",<\n>    "}>"].join("");
 
 >>
 

tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg

@@ -520,9 +520,8 @@ if not <chunks>:
 >>
 
 ExceptionClause(e, catchArg, catchAction) ::= <<
-catch (<catchArg>) {
+except <catchArg>:
     <catchAction>
-}
 >>
 
 // lexer actions are not associated with model objects
@@ -571,8 +570,8 @@ ThisRulePropertyRef_text(r)	 ::= "self._input.getText(localctx.start, self._inpu
 ThisRulePropertyRef_ctx(r)	 ::= "localctx"
 ThisRulePropertyRef_parser(r)	 ::= "self"
 
-NonLocalAttrRef(s)		 	 ::= "getInvokingContext(<s.ruleIndex>).<s.name>"
+NonLocalAttrRef(s)		 	 ::= "self.getInvokingContext(<s.ruleIndex>).<s.name>"
-SetNonLocalAttr(s, rhsChunks)  ::= "getInvokingContext(<s.ruleIndex>).<s.name> = <rhsChunks>"
+SetNonLocalAttr(s, rhsChunks)  ::= "self.getInvokingContext(<s.ruleIndex>).<s.name> = <rhsChunks>"
 
 AddToLabelList(a) ::= "<ctx(a.label)>.<a.listName>.append(<labelref(a.label)>)"
 

tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg

@@ -533,9 +533,8 @@ if not <chunks>:
 >>
 
 ExceptionClause(e, catchArg, catchAction) ::= <<
-catch (<catchArg>) {
+except <catchArg>:
     <catchAction>
-}
 >>
 
 // lexer actions are not associated with model objects
@@ -584,8 +583,8 @@ ThisRulePropertyRef_text(r)	 ::= "self._input.getText(localctx.start, self._inpu
 ThisRulePropertyRef_ctx(r)	 ::= "localctx"
 ThisRulePropertyRef_parser(r)	 ::= "self"
 
-NonLocalAttrRef(s)		 	 ::= "getInvokingContext(<s.ruleIndex>).<s.name>"
+NonLocalAttrRef(s)		 	 ::= "self.getInvokingContext(<s.ruleIndex>).<s.name>"
-SetNonLocalAttr(s, rhsChunks)  ::= "getInvokingContext(<s.ruleIndex>).<s.name> = <rhsChunks>"
+SetNonLocalAttr(s, rhsChunks)  ::= "self.getInvokingContext(<s.ruleIndex>).<s.name> = <rhsChunks>"
 
 AddToLabelList(a) ::= "<ctx(a.label)>.<a.listName>.append(<labelref(a.label)>)"
 

tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg

@@ -138,13 +138,13 @@ import Antlr4
 	 *
 	 * \<p>The default implementation does nothing.\</p>
 	 */
-	<accessLevelOpenOK(file)> func enterEveryRule(_ ctx: ParserRuleContext) { }
+	<accessLevelOpenOK(file)> func enterEveryRule(_ ctx: ParserRuleContext) throws { }
 	/**
 	 * {@inheritDoc\}
 	 *
 	 * \<p>The default implementation does nothing.\</p>
 	 */
-	<accessLevelOpenOK(file)> func exitEveryRule(_ ctx: ParserRuleContext) { }
+	<accessLevelOpenOK(file)> func exitEveryRule(_ ctx: ParserRuleContext) throws { }
 	/**
 	 * {@inheritDoc\}
 	 *
@@ -438,10 +438,10 @@ LeftRecursiveRuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,
 @discardableResult
 private func <currentRule.name>(_ _p<args:{a | , <a>}>: Int) throws -> <currentRule.ctxType>   {
 	let _parentctx: ParserRuleContext? = _ctx
-	var _parentState: Int = getState()
+	let _parentState: Int = getState()
 	var _localctx: <currentRule.ctxType> = <currentRule.ctxType>(_ctx, _parentState<currentRule.args:{a | , <a.name>}>)
-	var  _prevctx: <currentRule.ctxType> = _localctx
+	var _prevctx: <currentRule.ctxType> = _localctx
-	var _startState: Int = <currentRule.startState>
+	let _startState: Int = <currentRule.startState>
 	try enterRecursionRule(_localctx, <currentRule.startState>, <parser.name>.RULE_<currentRule.name>, _p)
 	<namedActions.init>
 	<locals; separator="\n">

tool/src/org/antlr/v4/codegen/target/PHPTarget.java

@@ -102,4 +102,12 @@ public class PHPTarget extends Target {
 	protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) {
 		UnicodeEscapes.appendPythonStyleEscapedCodePoint(codePoint, sb);
 	}
+
+   @Override
+   public String getTargetStringLiteralFromANTLRStringLiteral(CodeGenerator generator, String literal, boolean addQuotes) {
+	   String targetStringLiteral = super.getTargetStringLiteralFromANTLRStringLiteral(generator, literal, addQuotes);
+	   targetStringLiteral = targetStringLiteral.replace("$", "\\$");
+
+	   return targetStringLiteral;
+   }
 }