jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

Merge branch 'master' into patch-1

This commit is contained in:
Terence Parr 2020-11-24 10:14:45 -08:00 committed by GitHub
parent a39b77f904 b5812a1e79
commit f3a8a7a4e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
73 changed files with 892 additions and 928 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -3,7 +3,7 @@ target/
# ... but not code generation targets
!tool/src/org/antlr/v4/codegen/target/
# Node.js (npm and typings) cached dependencies
# Node.js (npm and typings) cached dependencies
node_modules/
typings/
@ -98,3 +98,6 @@ xcuserdata
javac-services.0.log
javac-services.0.log.lck
test/
# Don't ignore python tests
!runtime/Python3/test/

9
.travis.yml
View File

@ -203,6 +203,9 @@ matrix:
jdk: openjdk8
env: TARGET=javascript
stage: main-test
before_install:
- nvm install 14 # otherwise it runs by default on node 8
- f="./.travis/before-install-linux-javascript.sh"; ! [ -x "$f" ] || "$f"
- os: linux
dist: trusty
jdk: openjdk8
@ -215,7 +218,7 @@ before_install:
script:
- |
cd runtime-testsuite;
travis_wait 40 ../.travis/run-tests-$TARGET.sh;
rc=$?;
cat target/surefire-reports/*.dumpstream || true;
travis_wait 40 ../.travis/run-tests-$TARGET.sh
rc=$?
cat target/surefire-reports/*.dumpstream || true
exit $rc

7
.travis/before-install-linux-javascript.sh
View File

@ -2,7 +2,6 @@
set -euo pipefail
sudo apt-get update -qq
curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash -
sudo apt-get install -qq nodejs
node --version
# use v14 and check
echo node version: $(node --version)

6
.travis/run-tests-javascript.sh
View File

@ -2,4 +2,8 @@
set -euo pipefail
mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=javascript.* test
cd ../runtime/JavaScript
npm install
npm link
cd ../../runtime-testsuite
mvn -q -Dparallel=methods -DthreadCount=1 -Dtest=javascript.* test

2
appveyor.yml
View File

@ -16,7 +16,7 @@ build_script:
after_build:
- msbuild /target:pack /property:Configuration=Release /verbosity:detailed runtime/CSharp/runtime/CSharp/Antlr4.dotnet.sln
test_script:
- mvn install -Dantlr-php-php="C:\tools\php\php.exe" -Dantlr-dart-dart="C:\tools\dart-sdk\bin\dart.exe" -Dantlr-dart-pub="C:\tools\dart-sdk\bin\pub.bat" -Dantlr-dart-dart2native="C:\tools\dart-sdk\bin\dart2native.bat" -Dantlr-python2-python="C:\Python27\python.exe" -Dantlr-python3-python="C:\Python35\python.exe" -Dantlr-javascript-nodejs="C:\Program Files (x86)\nodejs\node.exe" --batch-mode
- mvn install -Dantlr-php-php="C:\tools\php\php.exe" -Dantlr-dart-dart="C:\tools\dart-sdk\bin\dart.exe" -Dantlr-dart-pub="C:\tools\dart-sdk\bin\pub.bat" -Dantlr-dart-dart2native="C:\tools\dart-sdk\bin\dart2native.bat" -Dantlr-python2-python="C:\Python27\python.exe" -Dantlr-python3-python="C:\Python35\python.exe" --batch-mode
artifacts:
- path: 'runtime\**\*.nupkg'
name: NuGet

14
contributors.txt
View File

@ -208,6 +208,7 @@ YYYY/MM/DD, github id, Full name, email
2018/07/03, jgoppert, James Goppert, james.goppert@gmail.com
2018/07/27, Maksim Novikov, mnovikov.work@gmail.com
2018/08/03, ENDOH takanao, djmchl@gmail.com
2018/10/08, xsIceman, Andreas Skaar, andreas.skaar@gmail.com
2018/10/18, edirgarcia, Edir García Lazo, edirgl@hotmail.com
2018/07/31, Lucas Henrqiue, lucashenrique580@gmail.com
2018/08/03, ENDOH takanao, djmchl@gmail.com
@ -218,6 +219,7 @@ YYYY/MM/DD, github id, Full name, email
2018/11/29, hannemann-tamas, Ralf Hannemann-Tamas, ralf.ht@gmail.com
2018/12/20, WalterCouto, Walter Couto, WalterCouto@users.noreply.github.com
2018/12/23, youkaichao, Kaichao You, youkaichao@gmail.com
2019/01/02, wkhemir, Wail Khemir, khemirwail@gmail.com
2019/01/16, kuegi, Markus Zancolo, markus.zancolo@roomle.com
2019/02/06, ralucado, Cristina Raluca Vijulie, ralucris.v[at]gmail[dot]com
2019/02/23, gedimitr, Gerasimos Dimitriadis, gedimitr@gmail.com
@ -247,7 +249,11 @@ YYYY/MM/DD, github id, Full name, email
2020/02/21, StochasticTinkr, Daniel Pitts, github@coloraura.com
2020/03/17, XsongyangX, Song Yang, songyang1218@gmail.com
2020/04/07, deniskyashif, Denis Kyashif, denis.kyashif@gmail.com
2020/04/23, martinvw, Martin van Wingerden, martin@martinvw.nl
2020/04/30, TristonianJones, Tristan Swadell, tswadell@google.com
2020/05/06, iammosespaulr, Moses Paul R, iammosespaulr@gmail.com
2020/05/10, gomerser, Erik Gomersbach, gomerser@gomersba.ch
2020/06/04, sigmasoldi3r, Pablo Blanco, pablobc.1995@gmail.com
2020/05/25, graknol, Sindre van der Linden, graknol@gmail.com
2020/05/31, d-markey, David Markey, dmarkey@free.fr
2020/06/02, cohomology, Kilian Kilger, kkilger AT gmail.com
@ -256,3 +262,11 @@ YYYY/MM/DD, github id, Full name, email
2020/08/22, stevenjohnstone, Steven Johnstone, steven.james.johnstone@gmail.com
2020/09/06, ArthurSonzogni, Sonzogni Arthur, arthursonzogni@gmail.com
2020/09/10, Khailian, Arunav Sanyal, arunav.sanyal91@gmail.com
2020/09/12, Clcanny, Charles Ruan, a837940593@gmail.com
2020/09/15, rmcgregor1990, Robert McGregor, rmcgregor1990@gmail.com
2020/09/16, trenki2, Markus Trenkwalder, trenki2[at]gmx[dot]net
2020/10/08, Marti2203, Martin Mirchev, mirchevmartin2203@gmail.com
2020/10/16, adarshbhat, Adarsh Bhat, adarshbhat@users.noreply.github.com
2020/10/20, adamwojs, Adam Wójs, adam[at]wojs.pl
2020/10/24, cliid, Jiwu Jang, jiwujang@naver.com
2020/11/05, MichelHartmann, Michel Hartmann, MichelHartmann@users.noreply.github.com

2
doc/cpp-target.md
View File

@ -1,6 +1,6 @@
# C++
The C++ target supports all platforms that can either run MS Visual Studio 2013 (or newer), XCode 7 (or newer) or CMake (C++11 required). All build tools can either create static or dynamic libraries, both as 64bit or 32bit arch. Additionally, XCode can create an iOS library. Also see [Antlr4 for C++ with CMake: A practical example](http://blorente.me//Antlr-,-C++-and-CMake-Wait-what.html).
The C++ target supports all platforms that can either run MS Visual Studio 2013 (or newer), XCode 7 (or newer) or CMake (C++11 required). All build tools can either create static or dynamic libraries, both as 64bit or 32bit arch. Additionally, XCode can create an iOS library. Also see [Antlr4 for C++ with CMake: A practical example](https://beyondtheloop.dev/Antlr-cpp-cmake/).
## How to create a C++ lexer or parser?
This is pretty much the same as creating a Java lexer or parser, except you need to specify the language target, for example:

2
doc/csharp-target.md
View File

@ -86,7 +86,7 @@ In order to execute this listener, you would simply add the following lines to t
...
IParseTree tree = parser.StartRule() - only repeated here for reference
KeyPrinter printer = new KeyPrinter();
ParseTreeWalker.DEFAULT.walk(printer, tree);
ParseTreeWalker.Default.Walk(printer, tree);
```
Further information can be found from The Definitive ANTLR Reference book.

4
doc/faq/index.md
View File

@ -38,8 +38,8 @@ This is the main landing page for the ANTLR 4 FAQ. The links below will take you
## Translation
* [ASTs vs parse trees](parse-trees.md)
* [Decoupling input walking from output generation](parse-trees.md)
* [ASTs vs parse trees](translation.md)
* [Decoupling input walking from output generation](translation.md)
## Actions and semantic predicates

4
doc/faq/parse-trees.md
View File

@ -50,7 +50,7 @@ For writing a compiler, either generate [LLVM-type static-single-assignment](htt
### XPath
XPath works great when you need to find specific nodes, possibly in certain contexts. The context is limited to the parents on the way to the root of the tree. For example, if you want to find all ID nodes, use path `//ID`. If you want all variable declarations, you might use path `//vardecl`. If you only want fields declarations, then you can use some context information via path `/classdef/vardecl`, which would only find vardecls that our children of class definitions. You can merge the results of multiple XPath `findAll()`s simulating a set union for XPath. The only caveat is that the order from the original tree is not preserved when you union multiple `findAll()` sets.
XPath works great when you need to find specific nodes, possibly in certain contexts. The context is limited to the parents on the way to the root of the tree. For example, if you want to find all ID nodes, use path `//ID`. If you want all variable declarations, you might use path `//vardecl`. If you only want fields declarations, then you can use some context information via path `/classdef/vardecl`, which would only find vardecls that are children of class definitions. You can merge the results of multiple XPath `findAll()`s simulating a set union for XPath. The only caveat is that the order from the original tree is not preserved when you union multiple `findAll()` sets.
### Tree pattern matching
@ -70,4 +70,4 @@ scopeStack.peek().define(new VariableSymbol("foo"))
That way each listener function does not have to compute its appropriate scope.
Examples: [DefScopesAndSymbols.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/DefScopesAndSymbols.java) and [SetScopeListener.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/SetScopeListener.java) and [VerifyListener.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/VerifyListener.java)
Examples: [DefScopesAndSymbols.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/DefScopesAndSymbols.java) and [SetScopeListener.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/SetScopeListener.java) and [VerifyListener.java](https://github.com/mantra/compiler/blob/master/src/java/mantra/semantics/VerifyListener.java)

2
doc/getting-started.md
View File

@ -8,7 +8,7 @@ ANTLR is really two things: a tool that translates your grammar to a parser/lexe
The first thing you should do is probably download and install a development tool plug-in. Even if you only use such tools for editing, they are great. Then, follow the instructions below to get the runtime environment available to your system to run generated parsers/lexers. In what follows, I talk about antlr-4.7.1-complete.jar, which has the tool and the runtime and any other support libraries (e.g., ANTLR v4 is written in v3).
If you are going to integrate ANTLR into your existing build system using mvn, ant, or want to get ANTLR into your IDE such as eclipse or intellij, see Integrating ANTLR into Development Systems.
If you are going to integrate ANTLR into your existing build system using mvn, ant, or want to get ANTLR into your IDE such as eclipse or intellij, see [Integrating ANTLR into Development Systems](https://github.com/antlr/antlr4/blob/master/doc/IDEs.md).
### UNIX

99
doc/javascript-target.md
View File

@ -11,11 +11,18 @@ In practice, this target has been extensively tested against:
* Chrome 39.0.2171
* Explorer 11.0.3
The tests were conducted using Selenium. No issue was found, so you should find that the runtime works pretty much against any recent JavaScript engine.
The above tests were conducted using Selenium. No issue was found, so you should find that the runtime works pretty much against any recent JavaScript engine.
## Is NodeJS supported?
The runtime has also been extensively tested against Node.js 10 LTS. No issue was found.
The runtime has also been extensively tested against Node.js 14 LTS. No issue was found.
NodeJS together with a packaging tool is now the preferred development path, developers are encouraged to follow it.
## What about modules?
Starting with version 8.1, Antlr4 JavaScript runtime follows esm semantics (see https://tc39.es/ecma262/#sec-modules for details)
Generated lexers, parsers, listeners and visitors also follow this new standard.
If you have used previous versions of the runtime, you will need to migrate and make your parser a module.
## How to create a JavaScript lexer or parser?
@ -61,7 +68,10 @@ The steps to create your parsing code are the following:
You are now ready to bundle your parsing code as follows:
- following webpack specs, create a webpack.config file
- in the `webpack.config` file, exclude node.js only modules using: `node: { module: "empty", net: "empty", fs: "empty" }`
- For Webpack version 5,
- in the `webpack.config` file, exclude node.js only modules using: `resolve: { fallback: { fs: false } }`
- For older versions of Webpack,
- in the `webpack.config` file, exclude node.js only modules using: `node: { module: "empty", net: "empty", fs: "empty" }`
- from the cmd line, navigate to the directory containing webpack.config and type: webpack
This will produce a single js file containing all your parsing code. Easy to include in your web pages!
@ -80,18 +90,18 @@ Let's suppose that your grammar is named, as above, "MyGrammar". Let's suppose t
Now a fully functioning script might look like the following:
```javascript
var antlr4 = require('antlr4');
var MyGrammarLexer = require('./MyGrammarLexer').MyGrammarLexer;
var MyGrammarParser = require('./MyGrammarParser').MyGrammarParser;
var MyGrammarListener = require('./MyGrammarListener').MyGrammarListener;
import antlr4 from 'antlr4';
import MyGrammarLexer from './MyGrammarLexer.js');
import MyGrammarParser from './MyGrammarParser.js';
import MyGrammarListener from './MyGrammarListener.js';
var input = "your text to parse here"
var chars = new antlr4.InputStream(input);
var lexer = new MyGrammarLexer(chars);
var tokens = new antlr4.CommonTokenStream(lexer);
var parser = new MyGrammarParser(tokens);
const input = "your text to parse here"
const chars = new antlr4.InputStream(input);
const lexer = new MyGrammarLexer(chars);
const tokens = new antlr4.CommonTokenStream(lexer);
const parser = new MyGrammarParser(tokens);
parser.buildParseTrees = true;
var tree = parser.MyStartRule();
const tree = parser.MyStartRule();
```
This program will work. But it won't be useful unless you do one of the following:
@ -105,19 +115,19 @@ This program will work. But it won't be useful unless you do one of the followin
## How do I create and run a visitor?
```javascript
// test.js
var antlr4 = require('antlr4');
var MyGrammarLexer = require('./QueryLexer').QueryLexer;
var MyGrammarParser = require('./QueryParser').QueryParser;
var MyGrammarListener = require('./QueryListener').QueryListener;
import antlr4 from 'antlr4';
import MyGrammarLexer from './QueryLexer.js';
import MyGrammarParser from './QueryParser.js';
import MyGrammarListener from './QueryListener.js';
var input = "field = 123 AND items in (1,2,3)"
var chars = new antlr4.InputStream(input);
var lexer = new MyGrammarLexer(chars);
var tokens = new antlr4.CommonTokenStream(lexer);
var parser = new MyGrammarParser(tokens);
const input = "field = 123 AND items in (1,2,3)"
const chars = new antlr4.InputStream(input);
const lexer = new MyGrammarLexer(chars);
const tokens = new antlr4.CommonTokenStream(lexer);
const parser = new MyGrammarParser(tokens);
parser.buildParseTrees = true;
var tree = parser.query();
const tree = parser.query();
class Visitor {
visitChildren(ctx) {
@ -145,40 +155,37 @@ tree.accept(new Visitor());
Let's suppose your MyGrammar grammar comprises 2 rules: "key" and "value". The antlr4 tool will have generated the following listener:
```javascript
MyGrammarListener = function(ParseTreeListener) {
// some code here
}
// some code here
MyGrammarListener.prototype.enterKey = function(ctx) {};
MyGrammarListener.prototype.exitKey = function(ctx) {};
MyGrammarListener.prototype.enterValue = function(ctx) {};
MyGrammarListener.prototype.exitValue = function(ctx) {};
class MyGrammarListener extends ParseTreeListener {
constructor() {
super();
}
enterKey(ctx) {}
exitKey(ctx) {}
enterValue(ctx) {}
exitValue(ctx) {}
}
```
In order to provide custom behavior, you might want to create the following class:
```javascript
var KeyPrinter = function() {
MyGrammarListener.call(this); // inherit default listener
return this;
};
class KeyPrinter extends MyGrammarListener {
// continue inheriting default listener
KeyPrinter.prototype = Object.create(MyGrammarListener.prototype);
KeyPrinter.prototype.constructor = KeyPrinter;
// override default listener behavior
KeyPrinter.prototype.exitKey = function(ctx) {
console.log("Oh, a key!");
};
// override default listener behavior
exitKey(ctx) {
console.log("Oh, a key!");
}
}
```
In order to execute this listener, you would simply add the following lines to the above code:
```javascript
...
tree = parser.StartRule() // only repeated here for reference
var printer = new KeyPrinter();
...
tree = parser.StartRule() // only repeated here for reference
const printer = new KeyPrinter();
antlr4.tree.ParseTreeWalker.DEFAULT.walk(printer, tree);
```

2
doc/lexicon.md
View File

@ -96,7 +96,7 @@ The recognizers that ANTLR generates assume a character vocabulary containing al
## Actions
Actions are code blocks written in the target language. You can use actions in a number of places within a grammar, but the syntax is always the same: arbitrary text surrounded by curly braces. You don’t need to escape a closing curly character if it’s in a string or comment: `"}"` or `/*}*/`. If the curlies are balanced, you also don’t need to escape }: `{...}`. Otherwise, escape extra curlies with a backslash: `\{` or `\}`. The action text should conform to the target language as specified with thelanguage option.
Actions are code blocks written in the target language. You can use actions in a number of places within a grammar, but the syntax is always the same: arbitrary text surrounded by curly braces. You don’t need to escape a closing curly character if it’s in a string or comment: `"}"` or `/*}*/`. If the curlies are balanced, you also don’t need to escape }: `{...}`. Otherwise, escape extra curlies with a backslash: `\{` or `\}`. The action text should conform to the target language as specified with the language option.
Embedded code can appear in: `@header` and `@members` named actions, parser and lexer rules, exception catching specifications, attribute sections for parser rules (return values, arguments, and locals), and some rule element options (currently predicates).

2
runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Chrome.test.stg
View File

@ -74,7 +74,7 @@ LANotEquals(i, v) ::= <%this._input.LA(<i>)!=<v>%>
TokenStartColumnEquals(i) ::= <%this._tokenStartColumn===<i>%>
ImportListener(X) ::= <<var <X>Listener = require('./<X>Listener').<X>Listener;>>
ImportListener(X) ::= ""
GetExpectedTokenNames() ::= "this.getExpectedTokens().toString(this.literalNames)"

6
runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Firefox.test.stg
View File

@ -76,11 +76,7 @@ LANotEquals(i, v) ::= <%this._input.LA(<i>)!=<v>%>
TokenStartColumnEquals(i) ::= <%this._tokenStartColumn===<i>%>
ImportListener(X) ::= <<
@parser::header {
var <X>Listener = require('./<X>Listener').<X>Listener;
}
>>
ImportListener(X) ::= ""
GetExpectedTokenNames() ::= "this.getExpectedTokens().toString(this.literalNames)"

6
runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Node.test.stg
View File

@ -74,11 +74,7 @@ LANotEquals(i, v) ::= <%this._input.LA(<i>)!=<v>%>
TokenStartColumnEquals(i) ::= <%this._tokenStartColumn===<i>%>
ImportListener(X) ::= <<
@parser::header {
var <X>Listener = require('./<X>Listener').<X>Listener;
}
>>
ImportListener(X) ::= ""
GetExpectedTokenNames() ::= "this.getExpectedTokens().toString(this.literalNames)"

6
runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Safari.test.stg
View File

@ -74,11 +74,7 @@ LANotEquals(i, v) ::= <%this._input.LA(<i>)!=<v>%>
TokenStartColumnEquals(i) ::= <%this._tokenStartColumn===<i>%>
ImportListener(X) ::= <<
@parser::header {
var <X>Listener = require('./<X>Listener').<X>Listener;
}
>>
ImportListener(X) ::= ""
GetExpectedTokenNames() ::= "this.getExpectedTokens().toString(this.literalNames)"

30
runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java
View File

@ -46,13 +46,14 @@ import static org.junit.Assume.assumeFalse;
public abstract class BaseRuntimeTest {
public final static String[] Targets = {
"Cpp",
"Java",
"Go",
"CSharp",
"Python2", "Python3",
"PHP",
"Dart",
"Go",
"Java",
"Node",
"Dart"
"PHP",
"Python2", "Python3",
"Swift"
};
static {
@ -95,10 +96,17 @@ public abstract class BaseRuntimeTest {
public void setUp() throws Exception {
// From http://junit.sourceforge.net/javadoc/org/junit/Assume.html
// "The default JUnit runner treats tests with failing assumptions as ignored"
assumeFalse(descriptor.ignore(descriptor.getTarget()));
assumeFalse(checkIgnored());
delegate.testSetUp();
}
public boolean checkIgnored() {
boolean ignored = !TestContext.isSupportedTarget(descriptor.getTarget()) || descriptor.ignore(descriptor.getTarget());
if(ignored)
System.out.println("Ignore " + descriptor);
return ignored;
}
@Rule
public final TestRule testWatcher = new TestWatcher() {
@Override
@ -112,7 +120,7 @@ public abstract class BaseRuntimeTest {
public void testOne() throws Exception {
// System.out.println(delegate.getTmpDir());
if ( descriptor.ignore(descriptor.getTarget()) ) {
System.out.printf("Ignore "+descriptor);
System.out.println("Ignore " + descriptor);
return;
}
@ -273,6 +281,8 @@ public abstract class BaseRuntimeTest {
// ---- support ----
public static RuntimeTestDescriptor[] getRuntimeTestDescriptors(Class<?> clazz, String targetName) {
if(!TestContext.isSupportedTarget(targetName))
return new RuntimeTestDescriptor[0];
Class<?>[] nestedClasses = clazz.getClasses();
List<RuntimeTestDescriptor> descriptors = new ArrayList<RuntimeTestDescriptor>();
for (Class<?> nestedClass : nestedClasses) {
@ -280,8 +290,10 @@ public abstract class BaseRuntimeTest {
if ( RuntimeTestDescriptor.class.isAssignableFrom(nestedClass) && !Modifier.isAbstract(modifiers) ) {
try {
RuntimeTestDescriptor d = (RuntimeTestDescriptor) nestedClass.newInstance();
d.setTarget(targetName);
descriptors.add(d);
if(!d.ignore(targetName)) {
d.setTarget(targetName);
descriptors.add(d);
}
} catch (Exception e) {
e.printStackTrace(System.err);
}

19
runtime-testsuite/test/org/antlr/v4/test/runtime/TestContext.java Normal file
View File

@ -0,0 +1,19 @@
package org.antlr.v4.test.runtime;
public abstract class TestContext {
public static boolean isTravisCI() {
return "true".equals(String.valueOf(System.getenv("TRAVIS")).toLowerCase());
}
public static boolean isAppVeyorCI() {
return "true".equals(String.valueOf(System.getenv("APPVEYOR")).toLowerCase());
}
public static boolean isSupportedTarget(String target) {
if(isAppVeyorCI())
return !target.matches("Swift|Node");
else
return true;
}
}

29
runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LexerExecDescriptors.java
View File

@ -1057,4 +1057,33 @@ public class LexerExecDescriptors {
return new Pair<>(grammarName, grammar);
}
}
/**
* This is a regression test for antlr/antlr4#2709 "PHP target generates
* invalid output when $ is used as part of the literal in lexer rule"
* https://github.com/antlr/antlr4/issues/2709
*/
public static class EscapeTargetStringLiteral extends BaseLexerTestDescriptor {
/**
[@0,0:-1='<EOF>',<-1>,1:0]
*/
@CommentHasStringValue
public String output;
public String errors = null;
public String startRule = "";
public String grammarName = "L";
/**
lexer grammar L;
ACTION_WITH_DOLLAR: '$ACTION';
*/
@CommentHasStringValue
public String grammar;
@Override
public boolean ignore(String targetName) {
return !targetName.equals("PHP");
}
}
}

563
runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/BaseNodeTest.java
View File

@ -34,6 +34,7 @@ import org.antlr.v4.semantics.SemanticPipeline;
import org.antlr.v4.test.runtime.ErrorQueue;
import org.antlr.v4.test.runtime.RuntimeTestSupport;
import org.antlr.v4.test.runtime.StreamVacuum;
import org.antlr.v4.test.runtime.TestContext;
import org.antlr.v4.tool.ANTLRMessage;
import org.antlr.v4.tool.DOTGenerator;
import org.antlr.v4.tool.Grammar;
@ -45,6 +46,7 @@ import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupString;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
@ -55,6 +57,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString;
import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile;
@ -128,16 +131,6 @@ public class BaseNodeTest implements RuntimeTestSupport {
return antlrToolErrors.toString();
}
protected org.antlr.v4.Tool newTool(String[] args) {
Tool tool = new Tool(args);
return tool;
}
protected Tool newTool() {
org.antlr.v4.Tool tool = new Tool(new String[] { "-o", tmpdir });
return tool;
}
protected ATN createATN(Grammar g, boolean useSerializer) {
if (g.atn == null) {
semanticProcess(g);
@ -179,57 +172,6 @@ public class BaseNodeTest implements RuntimeTestSupport {
}
}
IntegerList getTypesFromString(Grammar g, String expecting) {
IntegerList expectingTokenTypes = new IntegerList();
if (expecting != null && !expecting.trim().isEmpty()) {
for (String tname : expecting.replace(" ", "").split(",")) {
int ttype = g.getTokenType(tname);
expectingTokenTypes.add(ttype);
}
}
return expectingTokenTypes;
}
public IntegerList getTokenTypesViaATN(String input,
LexerATNSimulator lexerATN) {
ANTLRInputStream in = new ANTLRInputStream(input);
IntegerList tokenTypes = new IntegerList();
int ttype;
do {
ttype = lexerATN.match(in, Lexer.DEFAULT_MODE);
tokenTypes.add(ttype);
} while (ttype != Token.EOF);
return tokenTypes;
}
public List<String> getTokenTypes(LexerGrammar lg, ATN atn, CharStream input) {
LexerATNSimulator interp = new LexerATNSimulator(atn,
new DFA[] { new DFA(
atn.modeToStartState.get(Lexer.DEFAULT_MODE)) }, null);
List<String> tokenTypes = new ArrayList<String>();
int ttype;
boolean hitEOF = false;
do {
if (hitEOF) {
tokenTypes.add("EOF");
break;
}
int t = input.LA(1);
ttype = interp.match(input, Lexer.DEFAULT_MODE);
if (ttype == Token.EOF) {
tokenTypes.add("EOF");
}
else {
tokenTypes.add(lg.typeToTokenList.get(ttype));
}
if (t == IntStream.EOF) {
hitEOF = true;
}
} while (ttype != Token.EOF);
return tokenTypes;
}
protected String execLexer(String grammarFileName, String grammarStr,
String lexerName, String input) {
return execLexer(grammarFileName, grammarStr, lexerName, input, false);
@ -243,8 +185,9 @@ public class BaseNodeTest implements RuntimeTestSupport {
assertTrue(success);
writeFile(tmpdir, "input", input);
writeLexerTestFile(lexerName, showDFA);
writeFile(tmpdir, "package.json", "{\"type\": \"module\"}");
String output = execModule("Test.js");
if ( output.length()==0 ) {
if ( output!=null && output.length()==0 ) {
output = null;
}
return output;
@ -262,6 +205,7 @@ public class BaseNodeTest implements RuntimeTestSupport {
writeFile(tmpdir, "input", input);
rawBuildRecognizerTestFile(parserName, lexerName, listenerName,
visitorName, startRuleName, showDiagnosticErrors);
writeFile(tmpdir, "package.json", "{\"type\": \"module\"}");
return execRecognizer();
}
@ -323,17 +267,21 @@ public class BaseNodeTest implements RuntimeTestSupport {
}
public String execModule(String fileName) {
String nodejsPath = locateNodeJS();
String runtimePath = locateRuntime();
String modulePath = new File(new File(tmpdir), fileName)
.getAbsolutePath();
String inputPath = new File(new File(tmpdir), "input")
.getAbsolutePath();
try {
String npmPath = locateNpm();
if(!TestContext.isTravisCI()) {
installRuntime(npmPath);
registerRuntime(npmPath);
}
String modulePath = new File(new File(tmpdir), fileName)
.getAbsolutePath();
linkRuntime(npmPath);
String nodejsPath = locateNodeJS();
String inputPath = new File(new File(tmpdir), "input")
.getAbsolutePath();
ProcessBuilder builder = new ProcessBuilder(nodejsPath, modulePath,
inputPath);
builder.environment().put("NODE_PATH",
runtimePath + File.pathSeparator + tmpdir);
builder.environment().put("NODE_PATH", tmpdir);
builder.directory(new File(tmpdir));
Process process = builder.start();
StreamVacuum stdoutVacuum = new StreamVacuum(
@ -342,7 +290,10 @@ public class BaseNodeTest implements RuntimeTestSupport {
process.getErrorStream());
stdoutVacuum.start();
stderrVacuum.start();
// TODO switch to jdk 8
process.waitFor();
// if(!process.waitFor(1L, TimeUnit.MINUTES))
// process.destroyForcibly();
stdoutVacuum.join();
stderrVacuum.join();
String output = stdoutVacuum.toString();
@ -353,22 +304,59 @@ public class BaseNodeTest implements RuntimeTestSupport {
this.stderrDuringParse = stderrVacuum.toString();
}
return output;
}
catch (Exception e) {
} catch (Exception e) {
System.err.println("can't exec recognizer");
e.printStackTrace(System.err);
System.err.println();
return null;
}
return null;
}
private String locateTool(String tool) {
String[] roots = { "/usr/bin/", "/usr/local/bin/" };
for (String root : roots) {
if (new File(root + tool).exists()) {
return root + tool;
}
}
return null;
private void installRuntime(String npmPath) throws IOException, InterruptedException {
String runtimePath = locateRuntime();
ProcessBuilder builder = new ProcessBuilder(npmPath, "install");
builder.directory(new File(runtimePath));
builder.redirectError(new File(tmpdir, "error.txt"));
builder.redirectOutput(new File(tmpdir, "output.txt"));
Process process = builder.start();
// TODO switch to jdk 8
process.waitFor();
// if(!process.waitFor(30L, TimeUnit.SECONDS))
// process.destroyForcibly();
int error = process.exitValue();
if(error!=0)
throw new IOException("'npm install' failed");
}
private void registerRuntime(String npmPath) throws IOException, InterruptedException {
String runtimePath = locateRuntime();
ProcessBuilder builder = new ProcessBuilder(npmPath, "link");
builder.directory(new File(runtimePath));
builder.redirectError(new File(tmpdir, "error.txt"));
builder.redirectOutput(new File(tmpdir, "output.txt"));
Process process = builder.start();
// TODO switch to jdk 8
process.waitFor();
// if(!process.waitFor(30L, TimeUnit.SECONDS))
// process.destroyForcibly();
int error = process.exitValue();
if(error!=0)
throw new IOException("'npm link' failed");
}
private void linkRuntime(String npmPath) throws IOException, InterruptedException {
ProcessBuilder builder = new ProcessBuilder(npmPath, "link", "antlr4");
builder.directory(new File(tmpdir));
builder.redirectError(new File(tmpdir, "error.txt"));
builder.redirectOutput(new File(tmpdir, "output.txt"));
Process process = builder.start();
// TODO switch to jdk 8
process.waitFor();
// if(!process.waitFor(30L, TimeUnit.SECONDS))
// process.destroyForcibly();
int error = process.exitValue();
if(error!=0)
throw new IOException("'npm link antlr4' failed");
}
private boolean canExecute(String tool) {
@ -378,21 +366,29 @@ public class BaseNodeTest implements RuntimeTestSupport {
Process process = builder.start();
StreamVacuum vacuum = new StreamVacuum(process.getInputStream());
vacuum.start();
// TODO switch to jdk 8
process.waitFor();
// if(!process.waitFor(30L, TimeUnit.SECONDS))
// process.destroyForcibly();
vacuum.join();
return process.exitValue() == 0;
} catch (Exception e) {
return false;
}
catch (Exception e) {
;
}
private String locateNpm() {
// typically /usr/local/bin/npm
String prop = System.getProperty("antlr-javascript-npm");
if ( prop!=null && prop.length()!=0 ) {
return prop;
}
return false;
return "npm"; // everywhere
}
private String locateNodeJS() {
// typically /usr/local/bin/node
String propName = "antlr-javascript-nodejs";
String prop = System.getProperty(propName);
String prop = System.getProperty("antlr-javascript-nodejs");
if ( prop!=null && prop.length()!=0 ) {
return prop;
}
@ -404,7 +400,7 @@ public class BaseNodeTest implements RuntimeTestSupport {
private String locateRuntime() {
final ClassLoader loader = Thread.currentThread().getContextClassLoader();
final URL runtimeSrc = loader.getResource("JavaScript/src");
final URL runtimeSrc = loader.getResource("JavaScript");
if ( runtimeSrc==null ) {
throw new RuntimeException("Cannot find JavaScript runtime");
}
@ -418,213 +414,16 @@ public class BaseNodeTest implements RuntimeTestSupport {
return System.getProperty("os.name").toLowerCase().contains("windows");
}
// void ambig(List<Message> msgs, int[] expectedAmbigAlts, String
// expectedAmbigInput)
// throws Exception
// {
// ambig(msgs, 0, expectedAmbigAlts, expectedAmbigInput);
// }
// void ambig(List<Message> msgs, int i, int[] expectedAmbigAlts, String
// expectedAmbigInput)
// throws Exception
// {
// List<Message> amsgs = getMessagesOfType(msgs, AmbiguityMessage.class);
// AmbiguityMessage a = (AmbiguityMessage)amsgs.get(i);
// if ( a==null ) assertNull(expectedAmbigAlts);
// else {
// assertEquals(a.conflictingAlts.toString(),
// Arrays.toString(expectedAmbigAlts));
// }
// assertEquals(expectedAmbigInput, a.input);
// }
// void unreachable(List<Message> msgs, int[] expectedUnreachableAlts)
// throws Exception
// {
// unreachable(msgs, 0, expectedUnreachableAlts);
// }
// void unreachable(List<Message> msgs, int i, int[]
// expectedUnreachableAlts)
// throws Exception
// {
// List<Message> amsgs = getMessagesOfType(msgs,
// UnreachableAltsMessage.class);
// UnreachableAltsMessage u = (UnreachableAltsMessage)amsgs.get(i);
// if ( u==null ) assertNull(expectedUnreachableAlts);
// else {
// assertEquals(u.conflictingAlts.toString(),
// Arrays.toString(expectedUnreachableAlts));
// }
// }
List<ANTLRMessage> getMessagesOfType(List<ANTLRMessage> msgs,
Class<? extends ANTLRMessage> c) {
List<ANTLRMessage> filtered = new ArrayList<ANTLRMessage>();
for (ANTLRMessage m : msgs) {
if (m.getClass() == c)
filtered.add(m);
}
return filtered;
}
void checkRuleATN(Grammar g, String ruleName, String expecting) {
ParserATNFactory f = new ParserATNFactory(g);
ATN atn = f.createATN();
DOTGenerator dot = new DOTGenerator(g);
System.out
.println(dot.getDOT(atn.ruleToStartState[g.getRule(ruleName).index]));
Rule r = g.getRule(ruleName);
ATNState startState = atn.ruleToStartState[r.index];
ATNPrinter serializer = new ATNPrinter(g, startState);
String result = serializer.asString();
// System.out.print(result);
assertEquals(expecting, result);
}
public void testActions(String templates, String actionName, String action,
String expected) throws org.antlr.runtime.RecognitionException {
int lp = templates.indexOf('(');
String name = templates.substring(0, lp);
STGroup group = new STGroupString(templates);
ST st = group.getInstanceOf(name);
st.add(actionName, action);
String grammar = st.render();
ErrorQueue equeue = new ErrorQueue();
Grammar g = new Grammar(grammar, equeue);
if (g.ast != null && !g.ast.hasErrors) {
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();
ATNFactory factory = new ParserATNFactory(g);
if (g.isLexer())
factory = new LexerATNFactory((LexerGrammar) g);
g.atn = factory.createATN();
CodeGenerator gen = new CodeGenerator(g);
ST outputFileST = gen.generateParser();
String output = outputFileST.render();
// System.out.println(output);
String b = "#" + actionName + "#";
int start = output.indexOf(b);
String e = "#end-" + actionName + "#";
int end = output.indexOf(e);
String snippet = output.substring(start + b.length(), end);
assertEquals(expected, snippet);
}
if (equeue.size() > 0) {
System.err.println(equeue.toString());
}
}
protected void checkGrammarSemanticsError(ErrorQueue equeue,
GrammarSemanticsMessage expectedMessage) throws Exception {
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
ANTLRMessage m = equeue.errors.get(i);
if (m.getErrorType() == expectedMessage.getErrorType()) {
foundMsg = m;
}
}
assertNotNull("no error; " + expectedMessage.getErrorType()
+ " expected", foundMsg);
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
assertEquals(Arrays.toString(expectedMessage.getArgs()),
Arrays.toString(foundMsg.getArgs()));
if (equeue.size() != 1) {
System.err.println(equeue);
}
}
protected void checkGrammarSemanticsWarning(ErrorQueue equeue,
GrammarSemanticsMessage expectedMessage) throws Exception {
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.warnings.size(); i++) {
ANTLRMessage m = equeue.warnings.get(i);
if (m.getErrorType() == expectedMessage.getErrorType()) {
foundMsg = m;
}
}
assertNotNull("no error; " + expectedMessage.getErrorType()
+ " expected", foundMsg);
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
assertEquals(Arrays.toString(expectedMessage.getArgs()),
Arrays.toString(foundMsg.getArgs()));
if (equeue.size() != 1) {
System.err.println(equeue);
}
}
protected void checkError(ErrorQueue equeue, ANTLRMessage expectedMessage)
throws Exception {
// System.out.println("errors="+equeue);
ANTLRMessage foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
ANTLRMessage m = equeue.errors.get(i);
if (m.getErrorType() == expectedMessage.getErrorType()) {
foundMsg = m;
}
}
assertTrue("no error; " + expectedMessage.getErrorType() + " expected",
!equeue.errors.isEmpty());
assertTrue("too many errors; " + equeue.errors,
equeue.errors.size() <= 1);
assertNotNull(
"couldn't find expected error: "
+ expectedMessage.getErrorType(), foundMsg);
/*
* assertTrue("error is not a GrammarSemanticsMessage", foundMsg
* instanceof GrammarSemanticsMessage);
*/
assertArrayEquals(expectedMessage.getArgs(), foundMsg.getArgs());
}
public static class FilteringTokenStream extends CommonTokenStream {
public FilteringTokenStream(TokenSource src) {
super(src);
}
Set<Integer> hide = new HashSet<Integer>();
@Override
protected boolean sync(int i) {
if (!super.sync(i)) {
return false;
}
Token t = get(i);
if (hide.contains(t.getType())) {
((WritableToken) t).setChannel(Token.HIDDEN_CHANNEL);
}
return true;
}
public void setTokenTypeChannel(int ttype, int channel) {
hide.add(ttype);
}
}
protected void mkdir(String dir) {
File f = new File(dir);
f.mkdirs();
}
protected void writeParserTestFile(String parserName, String lexerName,
String listenerName, String visitorName,
String parserStartRuleName, boolean debug) {
ST outputFileST = new ST(
"var antlr4 = require('antlr4');\n"
+ "var <lexerName> = require('./<lexerName>');\n"
+ "var <parserName> = require('./<parserName>');\n"
+ "var <listenerName> = require('./<listenerName>').<listenerName>;\n"
+ "var <visitorName> = require('./<visitorName>').<visitorName>;\n"
"import antlr4 from 'antlr4';\n"
+ "import <lexerName> from './<lexerName>.js';\n"
+ "import <parserName> from './<parserName>.js';\n"
+ "import <listenerName> from './<listenerName>.js';\n"
+ "import <visitorName> from './<visitorName>.js';\n"
+ "\n"
+ "class TreeShapeListener extends antlr4.tree.ParseTreeListener {\n" +
" enterEveryRule(ctx) {\n" +
@ -640,11 +439,11 @@ public class BaseNodeTest implements RuntimeTestSupport {
+ "\n"
+ "function main(argv) {\n"
+ " var input = new antlr4.FileStream(argv[2], true);\n"
+ " var lexer = new <lexerName>.<lexerName>(input);\n"
+ " var lexer = new <lexerName>(input);\n"
+ " var stream = new antlr4.CommonTokenStream(lexer);\n"
+ "<createParser>"
+ " parser.buildParseTrees = true;\n"
+ " printer = function() {\n"
+ " const printer = function() {\n"
+ " this.println = function(s) { console.log(s); }\n"
+ " this.print = function(s) { process.stdout.write(s); }\n"
+ " return this;\n"
@ -654,10 +453,10 @@ public class BaseNodeTest implements RuntimeTestSupport {
+ " antlr4.tree.ParseTreeWalker.DEFAULT.walk(new TreeShapeListener(), tree);\n"
+ "}\n" + "\n" + "main(process.argv);\n" + "\n");
ST createParserST = new ST(
" var parser = new <parserName>.<parserName>(stream);\n");
" var parser = new <parserName>(stream);\n");
if (debug) {
createParserST = new ST(
" var parser = new <parserName>.<parserName>(stream);\n"
" var parser = new <parserName>(stream);\n"
+ " parser.addErrorListener(new antlr4.error.DiagnosticErrorListener());\n");
}
outputFileST.add("createParser", createParserST);
@ -671,12 +470,12 @@ public class BaseNodeTest implements RuntimeTestSupport {
protected void writeLexerTestFile(String lexerName, boolean showDFA) {
ST outputFileST = new ST(
"var antlr4 = require('antlr4');\n"
+ "var <lexerName> = require('./<lexerName>');\n"
"import antlr4 from 'antlr4';\n"
+ "import <lexerName> from './<lexerName>.js';\n"
+ "\n"
+ "function main(argv) {\n"
+ " var input = new antlr4.FileStream(argv[2], true);\n"
+ " var lexer = new <lexerName>.<lexerName>(input);\n"
+ " var lexer = new <lexerName>(input);\n"
+ " var stream = new antlr4.CommonTokenStream(lexer);\n"
+ " stream.fill();\n"
+ " for(var i=0; i\\<stream.tokens.length; i++) {\n"
@ -689,28 +488,6 @@ public class BaseNodeTest implements RuntimeTestSupport {
writeFile(tmpdir, "Test.js", outputFileST.render());
}
public void writeRecognizer(String parserName, String lexerName,
String listenerName, String visitorName,
String parserStartRuleName, boolean debug) {
if (parserName == null) {
writeLexerTestFile(lexerName, debug);
}
else {
writeParserTestFile(parserName, lexerName, listenerName,
visitorName, parserStartRuleName, debug);
}
}
protected void eraseFiles(final String filesEndingWith) {
File tmpdirF = new File(tmpdir);
String[] files = tmpdirF.list();
for (int i = 0; files != null && i < files.length; i++) {
if (files[i].endsWith(filesEndingWith)) {
new File(tmpdir + "/" + files[i]).delete();
}
}
}
protected void eraseFiles(File dir) {
String[] files = dir.list();
for (int i = 0; files != null && i < files.length; i++) {
@ -734,153 +511,10 @@ public class BaseNodeTest implements RuntimeTestSupport {
}
}
public String getFirstLineOfException() {
if (this.stderrDuringParse == null) {
return null;
}
String[] lines = this.stderrDuringParse.split("\n");
String prefix = "Exception in thread \"main\" ";
return lines[0].substring(prefix.length(), lines[0].length());
}
/**
* When looking at a result set that consists of a Map/HashTable we cannot
* rely on the output order, as the hashing algorithm or other aspects of
* the implementation may be different on differnt JDKs or platforms. Hence
* we take the Map, convert the keys to a List, sort them and Stringify the
* Map, which is a bit of a hack, but guarantees that we get the same order
* on all systems. We assume that the keys are strings.
*
* @param m
* The Map that contains keys we wish to return in sorted order
* @return A string that represents all the keys in sorted order.
*/
public <K, V> String sortMapToString(Map<K, V> m) {
// Pass in crap, and get nothing back
//
if (m == null) {
return null;
}
System.out.println("Map toString looks like: " + m.toString());
// Sort the keys in the Map
//
TreeMap<K, V> nset = new TreeMap<K, V>(m);
System.out.println("Tree map looks like: " + nset.toString());
return nset.toString();
}
public List<String> realElements(List<String> elements) {
return elements.subList(Token.MIN_USER_TOKEN_TYPE, elements.size());
}
public void assertNotNullOrEmpty(String message, String text) {
assertNotNull(message, text);
assertFalse(message, text.isEmpty());
}
public void assertNotNullOrEmpty(String text) {
assertNotNull(text);
assertFalse(text.isEmpty());
}
public static class IntTokenStream implements TokenStream {
IntegerList types;
int p = 0;
public IntTokenStream(IntegerList types) {
this.types = types;
}
@Override
public void consume() {
p++;
}
@Override
public int LA(int i) {
return LT(i).getType();
}
@Override
public int mark() {
return index();
}
@Override
public int index() {
return p;
}
@Override
public void release(int marker) {
seek(marker);
}
@Override
public void seek(int index) {
p = index;
}
@Override
public int size() {
return types.size();
}
@Override
public String getSourceName() {
return null;
}
@Override
public Token LT(int i) {
CommonToken t;
int rawIndex = p + i - 1;
if (rawIndex >= types.size())
t = new CommonToken(Token.EOF);
else
t = new CommonToken(types.get(rawIndex));
t.setTokenIndex(rawIndex);
return t;
}
@Override
public Token get(int i) {
return new org.antlr.v4.runtime.CommonToken(types.get(i));
}
@Override
public TokenSource getTokenSource() {
return null;
}
@Override
public String getText() {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Interval interval) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(RuleContext ctx) {
throw new UnsupportedOperationException("can't give strings");
}
@Override
public String getText(Token start, Token stop) {
throw new UnsupportedOperationException("can't give strings");
}
}
/** Sort a list */
public <T extends Comparable<? super T>> List<T> sort(List<T> data) {
List<T> dup = new ArrayList<T>();
dup.addAll(data);
List<T> dup = new ArrayList<T>(data);
Collections.sort(dup);
return dup;
}
@ -889,8 +523,7 @@ public class BaseNodeTest implements RuntimeTestSupport {
public <K extends Comparable<? super K>, V> LinkedHashMap<K, V> sort(
Map<K, V> data) {
LinkedHashMap<K, V> dup = new LinkedHashMap<K, V>();
List<K> keys = new ArrayList<K>();
keys.addAll(data.keySet());
List<K> keys = new ArrayList<K>(data.keySet());
Collections.sort(keys);
for (K k : keys) {
dup.put(k, data.get(k));

2
runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/SingletonPredictionContext.cs
View File

@ -79,7 +79,7 @@ namespace Antlr4.Runtime.Atn
return false;
}
Antlr4.Runtime.Atn.SingletonPredictionContext other = (Antlr4.Runtime.Atn.SingletonPredictionContext)o;
return returnState == other.returnState && parent.Equals(other.parent);
return returnState == other.returnState && (parent != null && parent.Equals(other.parent));
}
public override string ToString()

14
runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Parser.cs
View File

@ -22,20 +22,20 @@ namespace Antlr4.Runtime
#if !PORTABLE
public class TraceListener : IParseTreeListener
{
private readonly TextWriter Output;
public TraceListener(TextWriter output) {
Output = output;
public TraceListener(TextWriter output,Parser enclosing) {
_output = output;
_enclosing = enclosing;
}
public virtual void EnterEveryRule(ParserRuleContext ctx)
{
Output.WriteLine("enter " + this._enclosing.RuleNames[ctx.RuleIndex] + ", LT(1)=" + this._enclosing._input.LT(1).Text);
_output.WriteLine("enter " + this._enclosing.RuleNames[ctx.RuleIndex] + ", LT(1)=" + this._enclosing._input.LT(1).Text);
}
public virtual void ExitEveryRule(ParserRuleContext ctx)
{
Output.WriteLine("exit " + this._enclosing.RuleNames[ctx.RuleIndex] + ", LT(1)=" + this._enclosing._input.LT(1).Text);
_output.WriteLine("exit " + this._enclosing.RuleNames[ctx.RuleIndex] + ", LT(1)=" + this._enclosing._input.LT(1).Text);
}
public virtual void VisitErrorNode(IErrorNode node)
@ -46,15 +46,17 @@ namespace Antlr4.Runtime
{
ParserRuleContext parent = (ParserRuleContext)((IRuleNode)node.Parent).RuleContext;
IToken token = node.Symbol;
Output.WriteLine("consume " + token + " rule " + this._enclosing.RuleNames[parent.RuleIndex]);
_output.WriteLine("consume " + token + " rule " + this._enclosing.RuleNames[parent.RuleIndex]);
}
internal TraceListener(Parser _enclosing)
{
this._enclosing = _enclosing;
_output = Console.Out;
}
private readonly Parser _enclosing;
private readonly TextWriter _output;
}
#endif

3
runtime/Cpp/CMakeLists.txt
View File

@ -71,6 +71,9 @@ else()
set(MY_CXX_WARNING_FLAGS " -Wall -pedantic -W")
endif()
# Define USE_UTF8_INSTEAD_OF_CODECVT macro.
# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_UTF8_INSTEAD_OF_CODECVT")
# Initialize CXXFLAGS.
if("${CMAKE_VERSION}" VERSION_GREATER 3.1.0)
set(CMAKE_CXX_STANDARD 11)

25
runtime/Cpp/deploy-macos.sh
View File

@ -4,9 +4,27 @@
rm -f -R antlr4-runtime build lib 2> /dev/null
rm antlr4-cpp-runtime-macos.zip 2> /dev/null
# Get utf8 dependency.
mkdir -p runtime/thirdparty 2> /dev/null
pushd runtime/thirdparty
if [ ! -d utfcpp ]
then
git clone https://github.com/nemtrif/utfcpp.git utfcpp
pushd utfcpp
git checkout tags/v3.1.1
popd
fi
popd
# Binaries
xcodebuild -project runtime/antlrcpp.xcodeproj -target antlr4 -configuration Release
xcodebuild -project runtime/antlrcpp.xcodeproj -target antlr4_static -configuration Release
xcodebuild -project runtime/antlrcpp.xcodeproj \
-target antlr4 \
# GCC_PREPROCESSOR_DEFINITIONS='$GCC_PREPROCESSOR_DEFINITIONS USE_UTF8_INSTEAD_OF_CODECVT' \
-configuration Release
xcodebuild -project runtime/antlrcpp.xcodeproj \
-target antlr4_static \
# GCC_PREPROCESSOR_DEFINITIONS='$GCC_PREPROCESSOR_DEFINITIONS USE_UTF8_INSTEAD_OF_CODECVT' \
-configuration Release
rm -f -R lib
mkdir lib
mv runtime/build/Release/libantlr4-runtime.a lib/
@ -17,6 +35,9 @@ rm -f -R antlr4-runtime
pushd runtime/src
find . -name '*.h' | cpio -pdm ../../antlr4-runtime
popd
pushd runtime/thirdparty/utfcpp/source
find . -name '*.h' | cpio -pdm ../../../../antlr4-runtime
popd
# Zip up and clean up
zip -r antlr4-cpp-runtime-macos.zip antlr4-runtime lib

35
runtime/Cpp/runtime/CMakeLists.txt
View File

@ -1,4 +1,19 @@
include(${CMAKE_ROOT}/Modules/ExternalProject.cmake)
set(THIRDPARTY_DIR ${CMAKE_BINARY_DIR}/runtime/thirdparty)
set(UTFCPP_DIR ${THIRDPARTY_DIR}/utfcpp)
ExternalProject_Add(
utfcpp
GIT_REPOSITORY "git://github.com/nemtrif/utfcpp"
GIT_TAG "v3.1.1"
SOURCE_DIR ${UTFCPP_DIR}
UPDATE_DISCONNECTED 1
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${UTFCPP_DIR}/install -Dgtest_force_shared_crt=ON
TEST_AFTER_INSTALL 1
STEP_TARGETS build)
include_directories(
${PROJECT_SOURCE_DIR}/runtime/src
${PROJECT_SOURCE_DIR}/runtime/src/atn
@ -8,6 +23,8 @@ include_directories(
${PROJECT_SOURCE_DIR}/runtime/src/tree
${PROJECT_SOURCE_DIR}/runtime/src/tree/pattern
${PROJECT_SOURCE_DIR}/runtime/src/tree/xpath
${UTFCPP_DIR}/install/include/utf8cpp
${UTFCPP_DIR}/install/include/utf8cpp/utf8
)
@ -33,8 +50,8 @@ add_custom_target(make_lib_output_dir ALL
COMMAND ${CMAKE_COMMAND} -E make_directory ${LIB_OUTPUT_DIR}
)
add_dependencies(antlr4_shared make_lib_output_dir)
add_dependencies(antlr4_static make_lib_output_dir)
add_dependencies(antlr4_shared make_lib_output_dir utfcpp)
add_dependencies(antlr4_static make_lib_output_dir utfcpp)
if(CMAKE_SYSTEM_NAME MATCHES "Linux")
target_link_libraries(antlr4_shared ${UUID_LIBRARIES})
@ -102,15 +119,23 @@ set_target_properties(antlr4_static
COMPILE_FLAGS "${disabled_compile_warnings} ${extra_static_compile_flags}")
install(TARGETS antlr4_shared
DESTINATION lib
DESTINATION lib
EXPORT antlr4-targets)
install(TARGETS antlr4_static
DESTINATION lib
EXPORT antlr4-targets)
install(DIRECTORY "${PROJECT_SOURCE_DIR}/runtime/src/"
install(DIRECTORY "${PROJECT_SOURCE_DIR}/runtime/src/"
DESTINATION "include/antlr4-runtime"
COMPONENT dev
COMPONENT dev
FILES_MATCHING PATTERN "*.h"
)
install(FILES "${UTFCPP_DIR}/source/utf8.h"
DESTINATION "include/antlr4-runtime")
install(DIRECTORY "${UTFCPP_DIR}/source/utf8"
DESTINATION "include/antlr4-runtime"
COMPONENT dev
FILES_MATCHING PATTERN "*.h"
)

4
runtime/Cpp/runtime/antlrcpp.xcodeproj/project.pbxproj
View File

@ -2889,7 +2889,7 @@
GCC_WARN_UNUSED_LABEL = YES;
GCC_WARN_UNUSED_PARAMETER = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
HEADER_SEARCH_PATHS = src/;
HEADER_SEARCH_PATHS = "src/ thirdparty/utfcpp/source/ thirdparty/utfcpp/source/utf8/";
MACOSX_DEPLOYMENT_TARGET = 10.9;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = macosx;
@ -2945,7 +2945,7 @@
GCC_WARN_UNUSED_LABEL = YES;
GCC_WARN_UNUSED_PARAMETER = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
HEADER_SEARCH_PATHS = src/;
HEADER_SEARCH_PATHS = "src/ thirdparty/utfcpp/source/ thirdparty/utfcpp/source/utf8/";
MACOSX_DEPLOYMENT_TARGET = 10.9;
SDKROOT = macosx;
};

8
runtime/Cpp/runtime/src/IntStream.h
View File

@ -27,7 +27,13 @@ namespace antlr4 {
/// </summary>
class ANTLR4CPP_PUBLIC IntStream {
public:
static const size_t EOF = static_cast<size_t>(-1); // std::numeric_limits<size_t>::max(); doesn't work in VS 2013
#if __cplusplus >= 201703L
static constexpr size_t EOF = std::numeric_limits<size_t>::max();
#else
enum : size_t {
EOF = static_cast<size_t>(-1), // std::numeric_limits<size_t>::max(); doesn't work in VS 2013
};
#endif
/// The value returned by <seealso cref="#LA LA()"/> when the end of the stream is
/// reached.

27
runtime/Cpp/runtime/src/Lexer.h
View File

@ -18,14 +18,27 @@ namespace antlr4 {
/// of speed.
class ANTLR4CPP_PUBLIC Lexer : public Recognizer, public TokenSource {
public:
static const size_t DEFAULT_MODE = 0;
static const size_t MORE = static_cast<size_t>(-2);
static const size_t SKIP = static_cast<size_t>(-3);
#if __cplusplus >= 201703L
static constexpr size_t DEFAULT_MODE = 0;
static constexpr size_t MORE = std::numeric_limits<size_t>::max() - 1;
static constexpr size_t SKIP = std::numeric_limits<size_t>::max() - 2;
static const size_t DEFAULT_TOKEN_CHANNEL = Token::DEFAULT_CHANNEL;
static const size_t HIDDEN = Token::HIDDEN_CHANNEL;
static const size_t MIN_CHAR_VALUE = 0;
static const size_t MAX_CHAR_VALUE = 0x10FFFF;
static constexpr size_t DEFAULT_TOKEN_CHANNEL = Token::DEFAULT_CHANNEL;
static constexpr size_t HIDDEN = Token::HIDDEN_CHANNEL;
static constexpr size_t MIN_CHAR_VALUE = 0;
static constexpr size_t MAX_CHAR_VALUE = 0x10FFFF;
#else
enum : size_t {
DEFAULT_MODE = 0,
MORE = static_cast<size_t>(-2), // std::numeric_limits<size_t>::max() - 1; doesn't work in VS 2013
SKIP = static_cast<size_t>(-3), // std::numeric_limits<size_t>::max() - 2; doesn't work in VS 2013
DEFAULT_TOKEN_CHANNEL = Token::DEFAULT_CHANNEL,
HIDDEN = Token::HIDDEN_CHANNEL,
MIN_CHAR_VALUE = 0,
MAX_CHAR_VALUE = 0x10FFFF,
};
#endif
CharStream *_input; // Pure reference, usually from statically allocated instance.

8
runtime/Cpp/runtime/src/Recognizer.h
View File

@ -11,7 +11,13 @@ namespace antlr4 {
class ANTLR4CPP_PUBLIC Recognizer {
public:
static const size_t EOF = static_cast<size_t>(-1); // std::numeric_limits<size_t>::max(); doesn't work in VS 2013.
#if __cplusplus >= 201703L
static constexpr size_t EOF = std::numeric_limits<size_t>::max();
#else
enum : size_t {
EOF = static_cast<size_t>(-1), // std::numeric_limits<size_t>::max(); doesn't work in VS 2013.
};
#endif
Recognizer();
Recognizer(Recognizer const&) = delete;

46
runtime/Cpp/runtime/src/Token.h
View File

@ -14,24 +14,50 @@ namespace antlr4 {
/// we obtained this token.
class ANTLR4CPP_PUBLIC Token {
public:
static const size_t INVALID_TYPE = 0;
#if __cplusplus >= 201703L
static constexpr size_t INVALID_TYPE = 0;
#else
enum : size_t {
INVALID_TYPE = 0,
};
#endif
/// During lookahead operations, this "token" signifies we hit rule end ATN state
/// and did not follow it despite needing to.
static const size_t EPSILON = static_cast<size_t>(-2);
static const size_t MIN_USER_TOKEN_TYPE = 1;
static const size_t EOF = IntStream::EOF;
#if __cplusplus >= 201703L
static constexpr size_t EPSILON = std::numeric_limits<size_t>::max() - 1;
static constexpr size_t MIN_USER_TOKEN_TYPE = 1;
static constexpr size_t EOF = IntStream::EOF;
#else
enum : size_t {
EPSILON = static_cast<size_t>(-2), // std::numeric_limits<size_t>::max() - 1; doesn't work in VS 2013
MIN_USER_TOKEN_TYPE = 1,
EOF = IntStream::EOF,
};
#endif
virtual ~Token();
/// All tokens go to the parser (unless skip() is called in that rule)
/// on a particular "channel". The parser tunes to a particular channel
/// so that whitespace etc... can go to the parser on a "hidden" channel.
static const size_t DEFAULT_CHANNEL = 0;
#if __cplusplus >= 201703L
static constexpr size_t DEFAULT_CHANNEL = 0;
#else
enum : size_t {
DEFAULT_CHANNEL = 0,
};
#endif
/// Anything on different channel than DEFAULT_CHANNEL is not parsed
/// by parser.
static const size_t HIDDEN_CHANNEL = 1;
#if __cplusplus >= 201703L
static constexpr size_t HIDDEN_CHANNEL = 1;
#else
enum : size_t {
HIDDEN_CHANNEL = 1,
};
#endif
/**
* This is the minimum constant value which can be assigned to a
@ -44,7 +70,13 @@ namespace antlr4 {
*
* @see Token#getChannel()
*/
static const size_t MIN_USER_CHANNEL_VALUE = 2;
#if __cplusplus >= 201703L
static constexpr size_t MIN_USER_CHANNEL_VALUE = 2;
#else
enum : size_t {
MIN_USER_CHANNEL_VALUE = 2,
};
#endif
/// Get the text of the token.
virtual std::string getText() const = 0;

11
runtime/Cpp/runtime/src/TokenStreamRewriter.h
View File

@ -86,8 +86,15 @@ namespace antlr4 {
class ANTLR4CPP_PUBLIC TokenStreamRewriter {
public:
static const std::string DEFAULT_PROGRAM_NAME;
static const size_t PROGRAM_INIT_SIZE = 100;
static const size_t MIN_TOKEN_INDEX = 0;
#if __cplusplus >= 201703L
static constexpr size_t PROGRAM_INIT_SIZE = 100;
static constexpr size_t MIN_TOKEN_INDEX = 0;
#else
enum : size_t {
PROGRAM_INIT_SIZE = 100,
MIN_TOKEN_INDEX = 0,
};
#endif
TokenStreamRewriter(TokenStream *tokens);
virtual ~TokenStreamRewriter();

5
runtime/Cpp/runtime/src/antlr4-common.h
View File

@ -8,7 +8,6 @@
#include <algorithm>
#include <assert.h>
#include <atomic>
#include <codecvt>
#include <chrono>
#include <fstream>
#include <iostream>
@ -37,6 +36,10 @@
#include <condition_variable>
#include <functional>
#ifndef USE_UTF8_INSTEAD_OF_CODECVT
#include <codecvt>
#endif
// Defines for the Guid class and other platform dependent stuff.
#ifdef _WIN32
#ifdef _MSC_VER

8
runtime/Cpp/runtime/src/atn/ATN.h
View File

@ -12,7 +12,13 @@ namespace atn {
class ANTLR4CPP_PUBLIC ATN {
public:
static const size_t INVALID_ALT_NUMBER = 0;
#if __cplusplus >= 201703L
static constexpr size_t INVALID_ALT_NUMBER = 0;
#else
enum : size_t {
INVALID_ALT_NUMBER = 0,
};
#endif
/// Used for runtime deserialization of ATNs from strings.
ATN();

2
runtime/Cpp/runtime/src/atn/ATNConfig.cpp
View File

@ -11,8 +11,6 @@
using namespace antlr4::atn;
const size_t ATNConfig::SUPPRESS_PRECEDENCE_FILTER = 0x40000000;
ATNConfig::ATNConfig(ATNState *state_, size_t alt_, Ref<PredictionContext> const& context_)
: ATNConfig(state_, alt_, context_, SemanticContext::NONE) {
}

8
runtime/Cpp/runtime/src/atn/ATNConfig.h
View File

@ -114,7 +114,13 @@ namespace atn {
* {@link #isPrecedenceFilterSuppressed} property as a bit within the
* existing {@link #reachesIntoOuterContext} field.
*/
static const size_t SUPPRESS_PRECEDENCE_FILTER;
#if __cplusplus >= 201703L
static constexpr size_t SUPPRESS_PRECEDENCE_FILTER = 0x40000000;
#else
enum : size_t {
SUPPRESS_PRECEDENCE_FILTER = 0x40000000,
};
#endif
};
} // namespace atn

2
runtime/Cpp/runtime/src/atn/ATNDeserializer.cpp
View File

@ -57,8 +57,6 @@ using namespace antlr4;
using namespace antlr4::atn;
using namespace antlrcpp;
const size_t ATNDeserializer::SERIALIZED_VERSION = 3;
namespace {
uint32_t deserializeInt32(const std::vector<uint16_t>& data, size_t offset) {

8
runtime/Cpp/runtime/src/atn/ATNDeserializer.h
View File

@ -13,7 +13,13 @@ namespace atn {
class ANTLR4CPP_PUBLIC ATNDeserializer {
public:
static const size_t SERIALIZED_VERSION;
#if __cplusplus >= 201703L
static constexpr size_t SERIALIZED_VERSION = 3;
#else
enum : size_t {
SERIALIZED_VERSION = 3,
};
#endif
/// This is the current serialized UUID.
// ml: defined as function to avoid the “static initialization order fiasco”.

11
runtime/Cpp/runtime/src/atn/ATNState.h
View File

@ -81,8 +81,15 @@ namespace atn {
ATNState& operator=(ATNState const&) = delete;
static const size_t INITIAL_NUM_TRANSITIONS = 4;
static const size_t INVALID_STATE_NUMBER = static_cast<size_t>(-1); // std::numeric_limits<size_t>::max();
#if __cplusplus >= 201703L
static constexpr size_t INITIAL_NUM_TRANSITIONS = 4;
static constexpr size_t INVALID_STATE_NUMBER = std::numeric_limits<size_t>::max();
#else
enum : size_t {
INITIAL_NUM_TRANSITIONS = 4,
INVALID_STATE_NUMBER = static_cast<size_t>(-1), // std::numeric_limits<size_t>::max(); doesn't work in VS 2013
};
#endif
enum {
ATN_INVALID_TYPE = 0,

8
runtime/Cpp/runtime/src/atn/LL1Analyzer.h
View File

@ -17,7 +17,13 @@ namespace atn {
public:
/// Special value added to the lookahead sets to indicate that we hit
/// a predicate during analysis if {@code seeThruPreds==false}.
static const size_t HIT_PRED = Token::INVALID_TYPE;
#if __cplusplus >= 201703L
static constexpr size_t HIT_PRED = Token::INVALID_TYPE;
#else
enum : size_t {
HIT_PRED = Token::INVALID_TYPE,
};
#endif
const atn::ATN &_atn;

11
runtime/Cpp/runtime/src/atn/LexerATNSimulator.h
View File

@ -38,8 +38,15 @@ namespace atn {
public:
static const size_t MIN_DFA_EDGE = 0;
static const size_t MAX_DFA_EDGE = 127; // forces unicode to stay in ATN
#if __cplusplus >= 201703L
static constexpr size_t MIN_DFA_EDGE = 0;
static constexpr size_t MAX_DFA_EDGE = 127; // forces unicode to stay in ATN
#else
enum : size_t {
MIN_DFA_EDGE = 0,
MAX_DFA_EDGE = 127, // forces unicode to stay in ATN
};
#endif
protected:
/// <summary>

16
runtime/Cpp/runtime/src/atn/PredictionContext.h
View File

@ -30,10 +30,22 @@ namespace atn {
// ml: originally Integer.MAX_VALUE, which would be -1 for us, but this is already used in places where
// -1 is converted to unsigned, so we use a different value here. Any value does the job provided it doesn't
// conflict with real return states.
static const size_t EMPTY_RETURN_STATE = static_cast<size_t>(-10); // std::numeric_limits<size_t>::max() - 9;
#if __cplusplus >= 201703L
static constexpr size_t EMPTY_RETURN_STATE = std::numeric_limits<size_t>::max() - 9;
#else
enum : size_t {
EMPTY_RETURN_STATE = static_cast<size_t>(-10), // std::numeric_limits<size_t>::max() - 9; doesn't work in VS 2013
};
#endif
private:
static const size_t INITIAL_HASH = 1;
#if __cplusplus >= 201703L
static constexpr size_t INITIAL_HASH = 1;
#else
enum : size_t {
INITIAL_HASH = 1,
};
#endif
public:
static size_t globalNodeCount;

8
runtime/Cpp/runtime/src/misc/MurmurHash.h
View File

@ -13,7 +13,13 @@ namespace misc {
class ANTLR4CPP_PUBLIC MurmurHash {
private:
static const size_t DEFAULT_SEED = 0;
#if __cplusplus >= 201703L
static constexpr size_t DEFAULT_SEED = 0;
#else
enum : size_t {
DEFAULT_SEED = 0,
};
#endif
/// Initialize the hash using the default seed value.
/// Returns the intermediate hash value.

12
runtime/Cpp/runtime/src/support/StringUtils.cpp
View File

@ -20,16 +20,26 @@ void replaceAll(std::string& str, std::string const& from, std::string const& to
}
std::string ws2s(std::wstring const& wstr) {
#ifndef USE_UTF8_INSTEAD_OF_CODECVT
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
std::string narrow = converter.to_bytes(wstr);
#else
std::string narrow;
utf8::utf32to8(wstr.begin(), wstr.end(), std::back_inserter(narrow));
#endif
return narrow;
}
std::wstring s2ws(const std::string &str) {
#ifndef USE_UTF8_INSTEAD_OF_CODECVT
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
std::wstring wide = converter.from_bytes(str);
#else
std::wstring wide;
utf8::utf8to32(str.begin(), str.end(), std::back_inserter(wide));
#endif
return wide;
}

56
runtime/Cpp/runtime/src/support/StringUtils.h
View File

@ -7,43 +7,65 @@
#include "antlr4-common.h"
#ifdef USE_UTF8_INSTEAD_OF_CODECVT
#include "utf8.h"
#endif
namespace antlrcpp {
// For all conversions utf8 <-> utf32.
// I wouldn't prefer wstring_convert because: according to
// https://en.cppreference.com/w/cpp/locale/wstring_convert,
// wstring_convert is deprecated in C++17.
// utfcpp (https://github.com/nemtrif/utfcpp) is a substitution.
#ifndef USE_UTF8_INSTEAD_OF_CODECVT
// VS 2015 and VS 2017 have different bugs in std::codecvt_utf8<char32_t> (VS 2013 works fine).
#if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
typedef std::wstring_convert<std::codecvt_utf8<__int32>, __int32> UTF32Converter;
#else
typedef std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> UTF32Converter;
#if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
typedef std::wstring_convert<std::codecvt_utf8<__int32>, __int32> UTF32Converter;
#else
typedef std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> UTF32Converter;
#endif
#endif
// The conversion functions fails in VS2017, so we explicitly use a workaround.
template<typename T>
inline std::string utf32_to_utf8(T const& data)
{
// Don't make the converter static or we have to serialize access to it.
thread_local UTF32Converter converter;
#ifndef USE_UTF8_INSTEAD_OF_CODECVT
// Don't make the converter static or we have to serialize access to it.
thread_local UTF32Converter converter;
#if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
const auto p = reinterpret_cast<const int32_t *>(data.data());
return converter.to_bytes(p, p + data.size());
#if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
const auto p = reinterpret_cast<const int32_t *>(data.data());
return converter.to_bytes(p, p + data.size());
#else
return converter.to_bytes(data);
#endif
#else
return converter.to_bytes(data);
std::string narrow;
utf8::utf32to8(data.begin(), data.end(), std::back_inserter(narrow));
return narrow;
#endif
}
inline UTF32String utf8_to_utf32(const char* first, const char* last)
{
thread_local UTF32Converter converter;
#ifndef USE_UTF8_INSTEAD_OF_CODECVT
thread_local UTF32Converter converter;
#if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
auto r = converter.from_bytes(first, last);
i32string s = reinterpret_cast<const int32_t *>(r.data());
#if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
auto r = converter.from_bytes(first, last);
i32string s = reinterpret_cast<const int32_t *>(r.data());
return s;
#else
std::u32string s = converter.from_bytes(first, last);
return s;
#endif
#else
std::u32string s = converter.from_bytes(first, last);
UTF32String wide;
utf8::utf8to32(first, last, std::back_inserter(wide));
return wide;
#endif
return s;
}
void replaceAll(std::string &str, std::string const& from, std::string const& to);

2
runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java
View File

@ -19,7 +19,7 @@ public interface Tree {
Tree getParent();
/**
* This method returns whatever object represents the data at this note. For
* This method returns whatever object represents the data at this node. For
* example, for parse trees, the payload can be a {@link Token} representing
* a leaf node or a {@link RuleContext} object representing a rule
* invocation. For abstract syntax trees (ASTs), this is a {@link Token}

41
runtime/JavaScript/package-lock.json generated
View File

@ -2468,7 +2468,8 @@
"ansi-regex": {
"version": "2.1.1",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"aproba": {
"version": "1.2.0",
@ -2489,12 +2490,14 @@
"balanced-match": {
"version": "1.0.0",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"brace-expansion": {
"version": "1.1.11",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@ -2509,17 +2512,20 @@
"code-point-at": {
"version": "1.1.0",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"concat-map": {
"version": "0.0.1",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"console-control-strings": {
"version": "1.1.0",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"core-util-is": {
"version": "1.0.2",
@ -2636,7 +2642,8 @@
"inherits": {
"version": "2.0.4",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"ini": {
"version": "1.3.5",
@ -2648,6 +2655,7 @@
"version": "1.0.0",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"number-is-nan": "^1.0.0"
}
@ -2662,6 +2670,7 @@
"version": "3.0.4",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"brace-expansion": "^1.1.7"
}
@ -2669,12 +2678,14 @@
"minimist": {
"version": "0.0.8",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"minipass": {
"version": "2.9.0",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"safe-buffer": "^5.1.2",
"yallist": "^3.0.0"
@ -2693,6 +2704,7 @@
"version": "0.5.1",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"minimist": "0.0.8"
}
@ -2782,7 +2794,8 @@
"number-is-nan": {
"version": "1.0.1",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"object-assign": {
"version": "4.1.1",
@ -2794,6 +2807,7 @@
"version": "1.4.0",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"wrappy": "1"
}
@ -2879,7 +2893,8 @@
"safe-buffer": {
"version": "5.1.2",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"safer-buffer": {
"version": "2.1.2",
@ -2915,6 +2930,7 @@
"version": "1.0.2",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
@ -2934,6 +2950,7 @@
"version": "3.0.1",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"ansi-regex": "^2.0.0"
}
@ -2977,12 +2994,14 @@
"wrappy": {
"version": "1.0.2",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"yallist": {
"version": "3.1.1",
"bundled": true,
"dev": true
"dev": true,
"optional": true
}
}
},

2
runtime/JavaScript/package.json
View File

@ -27,6 +27,6 @@
"build": "webpack"
},
"engines": {
"node": ">=10"
"node": ">=14"
}
}

13
runtime/JavaScript/src/antlr4/PredictionContext.js
View File

@ -4,9 +4,10 @@
*/
const RuleContext = require('./RuleContext');
const {Hash, Map} = require('./Utils');
const {Hash, Map, equalArrays} = require('./Utils');
class PredictionContext {
constructor(cachedHashCode) {
this.cachedHashCode = cachedHashCode;
}
@ -83,6 +84,7 @@ function calculateHashString(parent, returnState) {
* can be used for both lexers and parsers.
*/
class PredictionContextCache {
constructor() {
this.cache = new Map();
}
@ -115,6 +117,7 @@ class PredictionContextCache {
class SingletonPredictionContext extends PredictionContext {
constructor(parent, returnState) {
let hashCode = 0;
const hash = new Hash();
@ -182,6 +185,7 @@ class SingletonPredictionContext extends PredictionContext {
}
class EmptyPredictionContext extends SingletonPredictionContext {
constructor() {
super(null, PredictionContext.EMPTY_RETURN_STATE);
}
@ -211,6 +215,7 @@ class EmptyPredictionContext extends SingletonPredictionContext {
PredictionContext.EMPTY = new EmptyPredictionContext();
class ArrayPredictionContext extends PredictionContext {
constructor(parents, returnStates) {
/**
* Parent can be null only if full ctx mode and we make an array
@ -249,8 +254,8 @@ class ArrayPredictionContext extends PredictionContext {
} else if (this.hashCode() !== other.hashCode()) {
return false; // can't be same if hash is different
} else {
return this.returnStates === other.returnStates &&
this.parents === other.parents;
return equalArrays(this.returnStates, other.returnStates) &&
equalArrays(this.parents, other.parents);
}
}
@ -557,7 +562,7 @@ function mergeArrays(a, b, rootIsWildcard, mergeCache) {
while (i < a.returnStates.length && j < b.returnStates.length) {
const a_parent = a.parents[i];
const b_parent = b.parents[j];
if (a.returnStates[i] === b.returnStates[j]) {
if (equalArrays(a.returnStates[i], b.returnStates[j])) {
// same payload (stack tops are equal), must yield merged singleton
const payload = a.returnStates[i];
// $+$ = $

6
runtime/JavaScript/src/antlr4/RuleContext.js
View File

@ -5,7 +5,6 @@
const {RuleNode} = require('./tree/Tree');
const {INVALID_INTERVAL} = require('./tree/Tree');
const INVALID_ALT_NUMBER = require('./atn/ATN').INVALID_ALT_NUMBER || 0; // TODO: solve cyclic dependency to avoid || 0
const Trees = require('./tree/Trees');
class RuleContext extends RuleNode {
@ -98,7 +97,10 @@ class RuleContext extends RuleNode {
* option contextSuperClass.
* to set it.
*/
getAltNumber() { return INVALID_ALT_NUMBER; }
getAltNumber() {
// use constant value of ATN.INVALID_ALT_NUMBER to avoid circular dependency
return 0;
}
/**
* Set the outer alternative number for this context node. Default

2
runtime/JavaScript/src/antlr4/Utils.js
View File

@ -430,7 +430,7 @@ function equalArrays(a, b) {
for (let i = 0; i < a.length; i++) {
if (a[i] == b[i])
continue;
if (!a[i].equals(b[i]))
if (!a[i].equals || !a[i].equals(b[i]))
return false;
}
return true;

2
runtime/JavaScript/src/antlr4/atn/ATN.js
View File

@ -8,8 +8,8 @@ const {IntervalSet} = require('./../IntervalSet');
const {Token} = require('./../Token');
class ATN {
constructor(grammarType , maxTokenType) {
constructor(grammarType , maxTokenType) {
/**
* Used for runtime deserialization of ATNs from strings
* The type of the ATN.

2
runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js
View File

@ -18,7 +18,7 @@ const PredictionMode = require('./PredictionMode');
const RuleContext = require('./../RuleContext');
const ParserRuleContext = require('./../ParserRuleContext');
const {SemanticContext} = require('./SemanticContext');
const PredictionContext = require('./../PredictionContext');
const {PredictionContext} = require('./../PredictionContext');
const {Interval} = require('./../IntervalSet');
const {Transition, SetTransition, NotSetTransition, RuleTransition, ActionTransition} = require('./Transition');
const {NoViableAltException} = require('./../error/Errors');

4
runtime/JavaScript/src/antlr4/tree/Trees.js
View File

@ -6,7 +6,6 @@
const Utils = require('./../Utils');
const {Token} = require('./../Token');
const {ErrorNode, TerminalNode, RuleNode} = require('./Tree');
const ATN = require('./../atn/ATN');
/** A set of utility routines useful for all kinds of ANTLR trees. */
const Trees = {
@ -50,7 +49,8 @@ const Trees = {
if (t instanceof RuleNode) {
const context = t.getRuleContext()
const altNumber = context.getAltNumber();
if ( altNumber != (ATN.INVALID_ALT_NUMBER || 0) ) { // TODO: solve cyclic dependency to avoid || 0
// use const value of ATN.INVALID_ALT_NUMBER to avoid circular dependency
if ( altNumber != 0 ) {
return ruleNames[t.ruleIndex]+":"+altNumber;
}
return ruleNames[t.ruleIndex];

8
runtime/Python3/src/antlr4/FileStream.py
View File

@ -10,7 +10,6 @@
#
import codecs
import unittest
from antlr4.InputStream import InputStream
@ -25,10 +24,3 @@ class FileStream(InputStream):
with open(fileName, 'rb') as file:
bytes = file.read()
return codecs.decode(bytes, encoding, errors)
class TestFileStream(unittest.TestCase):
def testStream(self):
stream = FileStream(__file__)
self.assertTrue(stream.size>0)

18
runtime/Python3/src/antlr4/InputStream.py
View File

@ -3,7 +3,6 @@
# Use of this file is governed by the BSD 3-clause license that
# can be found in the LICENSE.txt file in the project root.
#
import unittest
#
@ -85,20 +84,3 @@ class InputStream (object):
def __str__(self):
return self.strdata
class TestInputStream(unittest.TestCase):
def testStream(self):
stream = InputStream("abcde")
self.assertEqual(0, stream.index)
self.assertEqual(5, stream.size)
self.assertEqual(ord("a"), stream.LA(1))
stream.consume()
self.assertEqual(1, stream.index)
stream.seek(5)
self.assertEqual(Token.EOF, stream.LA(1))
self.assertEqual("bcd", stream.getText(1, 3))
stream.reset()
self.assertEqual(0, stream.index)

103
runtime/Python3/src/antlr4/IntervalSet.py
View File

@ -5,7 +5,6 @@
#
from io import StringIO
import unittest
from antlr4.Token import Token
# need forward declarations
@ -178,105 +177,3 @@ class IntervalSet(object):
if a<len(symbolicNames):
return symbolicNames[a]
return "<UNKNOWN>"
class TestIntervalSet(unittest.TestCase):
def testEmpty(self):
s = IntervalSet()
self.assertIsNone(s.intervals)
self.assertFalse(30 in s)
def testOne(self):
s = IntervalSet()
s.addOne(30)
self.assertTrue(30 in s)
self.assertFalse(29 in s)
self.assertFalse(31 in s)
def testTwo(self):
s = IntervalSet()
s.addOne(30)
s.addOne(40)
self.assertTrue(30 in s)
self.assertTrue(40 in s)
self.assertFalse(35 in s)
def testRange(self):
s = IntervalSet()
s.addRange(range(30,41))
self.assertTrue(30 in s)
self.assertTrue(40 in s)
self.assertTrue(35 in s)
def testDistinct1(self):
s = IntervalSet()
s.addRange(range(30,32))
s.addRange(range(40,42))
self.assertEquals(2,len(s.intervals))
self.assertTrue(30 in s)
self.assertTrue(40 in s)
self.assertFalse(35 in s)
def testDistinct2(self):
s = IntervalSet()
s.addRange(range(40,42))
s.addRange(range(30,32))
self.assertEquals(2,len(s.intervals))
self.assertTrue(30 in s)
self.assertTrue(40 in s)
self.assertFalse(35 in s)
def testContiguous1(self):
s = IntervalSet()
s.addRange(range(30,36))
s.addRange(range(36,41))
self.assertEquals(1,len(s.intervals))
self.assertTrue(30 in s)
self.assertTrue(40 in s)
self.assertTrue(35 in s)
def testContiguous2(self):
s = IntervalSet()
s.addRange(range(36,41))
s.addRange(range(30,36))
self.assertEquals(1,len(s.intervals))
self.assertTrue(30 in s)
self.assertTrue(40 in s)
def testOverlapping1(self):
s = IntervalSet()
s.addRange(range(30,40))
s.addRange(range(35,45))
self.assertEquals(1,len(s.intervals))
self.assertTrue(30 in s)
self.assertTrue(44 in s)
def testOverlapping2(self):
s = IntervalSet()
s.addRange(range(35,45))
s.addRange(range(30,40))
self.assertEquals(1,len(s.intervals))
self.assertTrue(30 in s)
self.assertTrue(44 in s)
def testOverlapping3(self):
s = IntervalSet()
s.addRange(range(30,32))
s.addRange(range(40,42))
s.addRange(range(50,52))
s.addRange(range(20,61))
self.assertEquals(1,len(s.intervals))
self.assertTrue(20 in s)
self.assertTrue(60 in s)
def testComplement(self):
s = IntervalSet()
s.addRange(range(10,21))
c = s.complement(1,100)
self.assertTrue(1 in c)
self.assertTrue(100 in c)
self.assertTrue(10 not in c)
self.assertTrue(20 not in c)

6
runtime/Python3/src/antlr4/Lexer.py
View File

@ -9,8 +9,12 @@
# of speed.
#/
from io import StringIO
from typing.io import TextIO
import sys
if sys.version_info[1] > 5:
from typing import TextIO
else:
from typing.io import TextIO
from antlr4.CommonTokenFactory import CommonTokenFactory
from antlr4.atn.LexerATNSimulator import LexerATNSimulator
from antlr4.InputStream import InputStream

5
runtime/Python3/src/antlr4/Parser.py
View File

@ -3,7 +3,10 @@
# Use of this file is governed by the BSD 3-clause license that
# can be found in the LICENSE.txt file in the project root.
import sys
from typing.io import TextIO
if sys.version_info[1] > 5:
from typing import TextIO
else:
from typing.io import TextIO
from antlr4.BufferedTokenStream import TokenStream
from antlr4.CommonTokenFactory import TokenFactory
from antlr4.error.ErrorStrategy import DefaultErrorStrategy

2
runtime/Python3/src/antlr4/PredictionContext.py
View File

@ -158,7 +158,7 @@ class SingletonPredictionContext(PredictionContext):
class EmptyPredictionContext(SingletonPredictionContext):
def __init__(self):
super().__init__(None, self.EMPTY_RETURN_STATE)
super().__init__(None, PredictionContext.EMPTY_RETURN_STATE)
def isEmpty(self):
return True

14
runtime/Python3/src/antlr4/Recognizer.py
View File

@ -144,17 +144,3 @@ class Recognizer(object):
self._stateNumber = atnState
del RecognitionException
import unittest
class Test(unittest.TestCase):
def testVersion(self):
major, minor = Recognizer().extractVersion("1.2")
self.assertEqual("1", major)
self.assertEqual("2", minor)
major, minor = Recognizer().extractVersion("1.2.3")
self.assertEqual("1", major)
self.assertEqual("2", minor)
major, minor = Recognizer().extractVersion("1.2-snapshot")
self.assertEqual("1", major)
self.assertEqual("2", minor)

8
runtime/Python3/test/TestFileStream.py Normal file
View File

@ -0,0 +1,8 @@
import unittest
from antlr4.FileStream import FileStream
class TestFileStream(unittest.TestCase):
def testStream(self):
stream = FileStream(__file__)
self.assertTrue(stream.size > 0)

19
runtime/Python3/test/TestInputStream.py Normal file
View File

@ -0,0 +1,19 @@
import unittest
from antlr4.Token import Token
from antlr4.InputStream import InputStream
class TestInputStream(unittest.TestCase):
def testStream(self):
stream = InputStream("abcde")
self.assertEqual(0, stream.index)
self.assertEqual(5, stream.size)
self.assertEqual(ord("a"), stream.LA(1))
stream.consume()
self.assertEqual(1, stream.index)
stream.seek(5)
self.assertEqual(Token.EOF, stream.LA(1))
self.assertEqual("bcd", stream.getText(1, 3))
stream.reset()
self.assertEqual(0, stream.index)

101
runtime/Python3/test/TestIntervalSet.py Normal file
View File

@ -0,0 +1,101 @@
import unittest
from antlr4.IntervalSet import IntervalSet
class TestIntervalSet(unittest.TestCase):
def testEmpty(self):
s = IntervalSet()
self.assertIsNone(s.intervals)
self.assertFalse(30 in s)
def testOne(self):
s = IntervalSet()
s.addOne(30)
self.assertTrue(30 in s)
self.assertFalse(29 in s)
self.assertFalse(31 in s)
def testTwo(self):
s = IntervalSet()
s.addOne(30)
s.addOne(40)
self.assertTrue(30 in s)
self.assertTrue(40 in s)
self.assertFalse(35 in s)
def testRange(self):
s = IntervalSet()
s.addRange(range(30,41))
self.assertTrue(30 in s)
self.assertTrue(40 in s)
self.assertTrue(35 in s)
def testDistinct1(self):
s = IntervalSet()
s.addRange(range(30,32))
s.addRange(range(40,42))
self.assertEquals(2,len(s.intervals))
self.assertTrue(30 in s)
self.assertTrue(40 in s)
self.assertFalse(35 in s)
def testDistinct2(self):
s = IntervalSet()
s.addRange(range(40,42))
s.addRange(range(30,32))
self.assertEquals(2,len(s.intervals))
self.assertTrue(30 in s)
self.assertTrue(40 in s)
self.assertFalse(35 in s)
def testContiguous1(self):
s = IntervalSet()
s.addRange(range(30,36))
s.addRange(range(36,41))
self.assertEquals(1,len(s.intervals))
self.assertTrue(30 in s)
self.assertTrue(40 in s)
self.assertTrue(35 in s)
def testContiguous2(self):
s = IntervalSet()
s.addRange(range(36,41))
s.addRange(range(30,36))
self.assertEquals(1,len(s.intervals))
self.assertTrue(30 in s)
self.assertTrue(40 in s)
def testOverlapping1(self):
s = IntervalSet()
s.addRange(range(30,40))
s.addRange(range(35,45))
self.assertEquals(1,len(s.intervals))
self.assertTrue(30 in s)
self.assertTrue(44 in s)
def testOverlapping2(self):
s = IntervalSet()
s.addRange(range(35,45))
s.addRange(range(30,40))
self.assertEquals(1,len(s.intervals))
self.assertTrue(30 in s)
self.assertTrue(44 in s)
def testOverlapping3(self):
s = IntervalSet()
s.addRange(range(30,32))
s.addRange(range(40,42))
s.addRange(range(50,52))
s.addRange(range(20,61))
self.assertEquals(1,len(s.intervals))
self.assertTrue(20 in s)
self.assertTrue(60 in s)
def testComplement(self):
s = IntervalSet()
s.addRange(range(10,21))
c = s.complement(1,100)
self.assertTrue(1 in c)
self.assertTrue(100 in c)
self.assertTrue(10 not in c)
self.assertTrue(20 not in c)

15
runtime/Python3/test/TestRecognizer.py Normal file
View File

@ -0,0 +1,15 @@
import unittest
from antlr4.Recognizer import Recognizer
class TestRecognizer(unittest.TestCase):
def testVersion(self):
major, minor = Recognizer().extractVersion("1.2")
self.assertEqual("1", major)
self.assertEqual("2", minor)
major, minor = Recognizer().extractVersion("1.2.3")
self.assertEqual("1", major)
self.assertEqual("2", minor)
major, minor = Recognizer().extractVersion("1.2-snapshot")
self.assertEqual("1", major)
self.assertEqual("2", minor)

6
runtime/Python3/test/run.py
View File

@ -4,5 +4,9 @@ src_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__
sys.path.insert(0,src_path)
from TestTokenStreamRewriter import TestTokenStreamRewriter
from xpathtest import XPathTest
from TestFileStream import TestFileStream
from TestInputStream import TestInputStream
from TestIntervalSet import TestIntervalSet
from TestRecognizer import TestRecognizer
import unittest
unittest.main()
unittest.main()

15
runtime/Swift/Sources/Antlr4/tree/ParseTreeProperty.swift Normal file
View File

@ -0,0 +1,15 @@
/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
* Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root.
*/
import Foundation
public class ParseTreeProperty<V> {
var annotations = Dictionary<ObjectIdentifier, V>()
public init() {}
open func get(_ node: ParseTree) -> V? { return annotations[ObjectIdentifier(node)] }
open func put(_ node: ParseTree, _ value: V) { annotations[ObjectIdentifier(node)] = value }
open func removeFrom(_ node: ParseTree) { annotations.removeValue(forKey: ObjectIdentifier(node)) }
}

12
tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg
View File

@ -464,7 +464,11 @@ RuleFunction(currentRule, args, code, locals, ruleCtx, altLabelCtxs, namedAction
<namedActions.init>
<locals; separator = "\n">
#if __cplusplus > 201703L
auto onExit = finally([=, this] {
#else
auto onExit = finally([=] {
#endif
<finallyAction>
exitRule();
});
@ -517,7 +521,11 @@ LeftRecursiveRuleFunction(currentRule, args, code, locals, ruleCtx, altLabelCtxs
<namedActions.init>
<! TODO: untested !> <locals; separator = "\n">
#if __cplusplus > 201703L
auto onExit = finally([=, this] {
#else
auto onExit = finally([=] {
#endif
<if (finallyAction)><finallyAction><endif>
unrollRecursionContexts(parentContext);
});
@ -539,7 +547,7 @@ LeftRecursiveRuleFunction(currentRule, args, code, locals, ruleCtx, altLabelCtxs
StructDeclHeader(struct, ctorAttrs, attrs, getters, dispatchMethods, interfaces, extensionMembers) ::= <<
class <file.exportMacro> <struct.name> : public <if (contextSuperClass)><contextSuperClass><else>antlr4::ParserRuleContext<endif><if(interfaces)>, <interfaces; separator=", "><endif> {
public:
<attrs: {a | <a>;}; separator="\n">
<attrs: {a | <a>;}; separator = "\n">
<if (ctorAttrs)><struct.name>(antlr4::ParserRuleContext *parent, size_t invokingState);<endif>
<struct.name>(antlr4::ParserRuleContext *parent, size_t invokingState<ctorAttrs: {a | , <a>}>);
<if (struct.provideCopyFrom)> <! don't need copy unless we have subclasses !>
@ -593,7 +601,7 @@ class <file.exportMacro> <struct.name> : public <currentRule.name; format = "cap
public:
<struct.name>(<currentRule.name; format = "cap">Context *ctx);
<if (attrs)><attrs: {a | <a>}; separator = "\n"><endif>
<if (attrs)><attrs: {a | <a>;}; separator = "\n"><endif>
<getters: {g | <g>}; separator = "\n">
<dispatchMethods; separator = "\n">
};

225
tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg
View File

@ -48,12 +48,12 @@ javascriptTypeInitMap ::= [
ParserFile(file, parser, namedActions, contextSuperClass) ::= <<
<fileHeader(file.grammarFileName, file.ANTLRVersion)>
var antlr4 = require('antlr4/index');
import antlr4 from 'antlr4';
<if(file.genListener)>
var <file.grammarName>Listener = require('./<file.grammarName>Listener').<file.grammarName>Listener;
import <file.grammarName>Listener from './<file.grammarName>Listener.js';
<endif>
<if(file.genVisitor)>
var <file.grammarName>Visitor = require('./<file.grammarName>Visitor').<file.grammarName>Visitor;
import <file.grammarName>Visitor from './<file.grammarName>Visitor.js';
<endif>
<namedActions.header>
@ -62,54 +62,43 @@ var <file.grammarName>Visitor = require('./<file.grammarName>Visitor').<file.gra
ListenerFile(file, header, namedActions) ::= <<
<fileHeader(file.grammarFileName, file.ANTLRVersion)>
var antlr4 = require('antlr4/index');
import antlr4 from 'antlr4';
// This class defines a complete listener for a parse tree produced by <file.parserName>.
function <file.grammarName>Listener() {
return this;
}
<file.grammarName>Listener.prototype = Object.create(antlr4.tree.ParseTreeListener.prototype);
<file.grammarName>Listener.prototype.constructor = <file.grammarName>Listener;
export default class <file.grammarName>Listener extends antlr4.tree.ParseTreeListener {
<file.listenerNames:{lname |
// Enter a parse tree produced by <file.parserName>#<lname>.
<file.grammarName>Listener.prototype.enter<lname; format="cap"> = function(ctx) {
\};
// Enter a parse tree produced by <file.parserName>#<lname>.
enter<lname; format="cap">(ctx) {
\}
// Exit a parse tree produced by <file.parserName>#<lname>.
<file.grammarName>Listener.prototype.exit<lname; format="cap"> = function(ctx) {
\};
// Exit a parse tree produced by <file.parserName>#<lname>.
exit<lname; format="cap">(ctx) {
\}
}; separator="\n">
exports.<file.grammarName>Listener = <file.grammarName>Listener;
}
>>
VisitorFile(file, header, namedActions) ::= <<
<fileHeader(file.grammarFileName, file.ANTLRVersion)>
var antlr4 = require('antlr4/index');
import antlr4 from 'antlr4';
// This class defines a complete generic visitor for a parse tree produced by <file.parserName>.
function <file.grammarName>Visitor() {
antlr4.tree.ParseTreeVisitor.call(this);
return this;
}
<file.grammarName>Visitor.prototype = Object.create(antlr4.tree.ParseTreeVisitor.prototype);
<file.grammarName>Visitor.prototype.constructor = <file.grammarName>Visitor;
export default class <file.grammarName>Visitor extends antlr4.tree.ParseTreeVisitor {
<file.visitorNames:{lname |
// Visit a parse tree produced by <file.parserName>#<lname>.
<file.grammarName>Visitor.prototype.visit<lname; format="cap"> = function(ctx) {
return this.visitChildren(ctx);
\};
// Visit a parse tree produced by <file.parserName>#<lname>.
visit<lname; format="cap">(ctx) {
return this.visitChildren(ctx);
\}
}; separator="\n">
exports.<file.grammarName>Visitor = <file.grammarName>Visitor;
}
>>
@ -120,32 +109,30 @@ fileHeader(grammarFileName, ANTLRVersion) ::= <<
Parser(parser, funcs, atn, sempredFuncs, superClass) ::= <<
<if(superClass)>
var <superClass> = require('./<superClass>').<superClass>;
import <superClass> from './<superClass>.js';
<endif>
var grammarFileName = "<parser.grammarFileName; format="java-escape">";
<atn>
var atn = new antlr4.atn.ATNDeserializer().deserialize(serializedATN);
const atn = new antlr4.atn.ATNDeserializer().deserialize(serializedATN);
var decisionsToDFA = atn.decisionToState.map( function(ds, index) { return new antlr4.dfa.DFA(ds, index); });
const decisionsToDFA = atn.decisionToState.map( (ds, index) => new antlr4.dfa.DFA(ds, index) );
var sharedContextCache = new antlr4.PredictionContextCache();
const sharedContextCache = new antlr4.PredictionContextCache();
var literalNames = [ <parser.literalNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
export default class <parser.name> extends <superClass; null="antlr4.Parser"> {
var symbolicNames = [ <parser.symbolicNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
static grammarFileName = "<parser.grammarFileName; format="java-escape">";
static literalNames = [ <parser.literalNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
static symbolicNames = [ <parser.symbolicNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
static ruleNames = [ <parser.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor> ];
var ruleNames = [ <parser.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor> ];
class <parser.name> extends <superClass; null="antlr4.Parser"> {
constructor(input) {
super(input);
this._interp = new antlr4.atn.ParserATNSimulator(this, atn, decisionsToDFA, sharedContextCache);
this.ruleNames = ruleNames;
this.literalNames = literalNames;
this.symbolicNames = symbolicNames;
this.ruleNames = <parser.name>.ruleNames;
this.literalNames = <parser.name>.literalNames;
this.symbolicNames = <parser.name>.symbolicNames;
<namedActions.members>
}
@ -161,10 +148,13 @@ class <parser.name> extends <superClass; null="antlr4.Parser"> {
default:
throw "No predicate with index:" + ruleIndex;
}
};
}
<sempredFuncs.values; separator="\n">
<endif>
<funcs; separator="\n">
}
<parser.name>.EOF = antlr4.Token.EOF;
@ -174,12 +164,19 @@ class <parser.name> extends <superClass; null="antlr4.Parser"> {
<parser.rules:{r | <parser.name>.RULE_<r.name> = <r.index>;}; separator="\n", wrap, anchor>
<funcs; separator="\n">
<funcs:{f | <ruleContexts(f)>}; separator="\n">
module.exports.<parser.name> = <parser.name>;
<! Define fields of this parser to export the context classes !>
<parser.funcs:{f | <parser.name>.<f.ctxType> = <f.ctxType>; }; separator="\n">
>>
ruleContexts(currentRule) ::= <<
<currentRule.ruleCtx>
<currentRule.altLabelCtxs:{l | <currentRule.altLabelCtxs.(l)>}; separator="\n">
>>
dumpActions(recog, argFuncs, actionFuncs, sempredFuncs) ::= <<
<if(actionFuncs)>
<lexer.name>.prototype.action = function(localctx, ruleIndex, actionIndex) {
@ -244,16 +241,9 @@ RuleSempredFunction(r, actions) ::= <<
RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,finallyAction,postamble,exceptions) ::= <<
<ruleCtx>
<altLabelCtxs:{l | <altLabelCtxs.(l)>}; separator="\n">
<! Define fields of this parser to export the context classes !>
<parser.name>.<currentRule.ctxType> = <currentRule.ctxType>;
<parser.name>.prototype.<currentRule.name> = function(<currentRule.args:{a | <a.name>}; separator=", ">) {
var localctx = new <currentRule.ctxType>(this, this._ctx, this.state<currentRule.args:{a | , <a.name>}>);
<currentRule.name>(<currentRule.args:{a | <a.name>}; separator=", ">) {
let localctx = new <currentRule.ctxType>(this, this._ctx, this.state<currentRule.args:{a | , <a.name>}>);
this.enterRule(localctx, <currentRule.startState>, <parser.name>.RULE_<currentRule.name>);
<namedActions.init>
<locals; separator="\n">
@ -277,7 +267,7 @@ RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,fina
this.exitRule();
}
return localctx;
};
}
>>
@ -285,18 +275,15 @@ LeftRecursiveRuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,
namedActions,finallyAction,postamble) ::=
<<
<ruleCtx>
<altLabelCtxs:{l | <altLabelCtxs.(l)>}; separator="\n">
<parser.name>.prototype.<currentRule.name> = function(_p<if(currentRule.args)>, <args:{a | , <a>}><endif>) {
<currentRule.name>(_p<if(currentRule.args)>, <args:{a | , <a>}><endif>) {
if(_p===undefined) {
_p = 0;
}
var _parentctx = this._ctx;
var _parentState = this.state;
var localctx = new <currentRule.ctxType>(this, this._ctx, _parentState<args:{a | , <a.name>}>);
var _prevctx = localctx;
var _startState = <currentRule.startState>;
const _parentctx = this._ctx;
const _parentState = this.state;
let localctx = new <currentRule.ctxType>(this, this._ctx, _parentState<args:{a | , <a.name>}>);
let _prevctx = localctx;
const _startState = <currentRule.startState>;
this.enterRecursionRule(localctx, <currentRule.startState>, <parser.name>.RULE_<currentRule.name>, _p);
<namedActions.init>
<locals; separator="\n">
@ -317,7 +304,7 @@ LeftRecursiveRuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,
this.unrollRecursionContexts(_parentctx)
}
return localctx;
};
}
>>
@ -424,7 +411,7 @@ if(la_===<i><if(!choice.ast.greedy)>+1<endif>) {
StarBlock(choice, alts, sync, iteration) ::= <<
this.state = <choice.stateNumber>;
this._errHandler.sync(this);
var _alt = this._interp.adaptivePredict(this._input,<choice.decision>,this._ctx)
let _alt = this._interp.adaptivePredict(this._input,<choice.decision>,this._ctx)
while(_alt!=<choice.exitAlt> && _alt!=antlr4.atn.ATN.INVALID_ALT_NUMBER) {
if(_alt===1<if(!choice.ast.greedy)>+1<endif>) {
<iteration>
@ -440,7 +427,7 @@ while(_alt!=<choice.exitAlt> && _alt!=antlr4.atn.ATN.INVALID_ALT_NUMBER) {
PlusBlock(choice, alts, error) ::= <<
this.state = <choice.blockStartStateNumber>; <! alt block decision !>
this._errHandler.sync(this);
var _alt = 1<if(!choice.ast.greedy)>+1<endif>;
let _alt = 1<if(!choice.ast.greedy)>+1<endif>;
do {
switch (_alt) {
<alts:{alt|
@ -592,8 +579,8 @@ ThisRulePropertyRef_text(r) ::= "this._input.getText(new antlr4.Interval(localc
ThisRulePropertyRef_ctx(r) ::= "localctx"
ThisRulePropertyRef_parser(r) ::= "this"
NonLocalAttrRef(s) ::= "getInvokingContext(<s.ruleIndex>).<s.name>"
SetNonLocalAttr(s, rhsChunks) ::= "getInvokingContext(<s.ruleIndex>).<s.name> = <rhsChunks>"
NonLocalAttrRef(s) ::= "this.getInvokingContext(<s.ruleIndex>).<s.name>"
SetNonLocalAttr(s, rhsChunks) ::= "this.getInvokingContext(<s.ruleIndex>).<s.name> = <rhsChunks>"
AddToLabelList(a) ::= "<ctx(a.label)>.<a.listName>.push(<labelref(a.label)>);"
@ -604,15 +591,16 @@ RuleContextDecl(r) ::= "this.<r.name> = null; // <r.ctxName>"
RuleContextListDecl(rdecl) ::= "this.<rdecl.name> = []; // of <rdecl.ctxName>s"
ContextTokenGetterDecl(t) ::= <<
<t.name> = function() {
<t.name>() {
return this.getToken(<parser.name>.<t.name>, 0);
};
>>
// should never be called
ContextTokenListGetterDecl(t) ::= <<
def <t.name>_list(self):
return self.getTokens(<parser.name>.<t.name>)
<t.name>_list() {
return this.getTokens(<parser.name>.<t.name>);
}
>>
ContextTokenListIndexedGetterDecl(t) ::= <<
@ -630,16 +618,16 @@ ContextTokenListIndexedGetterDecl(t) ::= <<
>>
ContextRuleGetterDecl(r) ::= <<
<r.name> = function() {
<r.name>() {
return this.getTypedRuleContext(<r.ctxName>,0);
};
>>
// should never be called
ContextRuleListGetterDecl(r) ::= <<
def <r.name>_list(self):
return self.getTypedRuleContexts(<parser.name>.<r.ctxName>)
<r.name>_list() {
return this.getTypedRuleContexts(<parser.name>.<r.ctxName>);
}
>>
ContextRuleListIndexedGetterDecl(r) ::= <<
@ -672,6 +660,7 @@ CaptureNextTokenType(d) ::= "<d.varName> = this._input.LA(1);"
StructDecl(struct,ctorAttrs,attrs,getters,dispatchMethods,interfaces,extensionMembers) ::= <<
class <struct.name> extends <if(contextSuperClass)><contextSuperClass><else>antlr4.ParserRuleContext<endif> {
constructor(parser, parent, invokingState<struct.ctorAttrs:{a | , <a.name>}>) {
if(parent===undefined) {
parent = null;
@ -685,58 +674,59 @@ class <struct.name> extends <if(contextSuperClass)><contextSuperClass><else>antl
<attrs:{a | <a>}; separator="\n">
<struct.ctorAttrs:{a | this.<a.name> = <a.name> || null;}; separator="\n">
}
<getters:{g | <g>}; separator="\n\n">
<if(struct.provideCopyFrom)> <! don't need copy unless we have subclasses !>
copyFrom(ctx) {
super.copyFrom(ctx);
<struct.attrs:{a | this.<a.name> = ctx.<a.name>;}; separator="\n">
}
<endif>
<dispatchMethods; separator="\n">
<extensionMembers; separator="\n">
}
<getters:{g | <struct.name>.prototype.<g>}; separator="\n\n">
<if(struct.provideCopyFrom)> <! don't need copy unless we have subclasses !>
<struct.name>.prototype.copyFrom = function(ctx) {
<if(contextSuperClass)><contextSuperClass><else>antlr4.ParserRuleContext<endif>.prototype.copyFrom.call(this, ctx);
<struct.attrs:{a | this.<a.name> = ctx.<a.name>;}; separator="\n">
};
<endif>
<dispatchMethods; separator="\n">
<extensionMembers; separator="\n">
>>
AltLabelStructDecl(struct,attrs,getters,dispatchMethods) ::= <<
class <struct.name> extends <currentRule.name; format="cap">Context {
class <struct.name> extends <struct.parentRule; format="cap">Context {
constructor(parser, ctx) {
super(parser);
<attrs:{a | <a>;}; separator="\n">
<currentRule.name; format="cap">Context.prototype.copyFrom.call(this, ctx);
super.copyFrom(ctx);
}
}
<struct.name>.prototype = Object.create(<currentRule.name; format="cap">Context.prototype);
<struct.name>.prototype.constructor = <struct.name>;
<getters:{g | <g>}; separator="\n\n">
<dispatchMethods; separator="\n">
}
<! Define fields of this parser to export this struct/context class !>
<parser.name>.<struct.name> = <struct.name>;
<getters:{g | <struct.name>.prototype.<g>}; separator="\n\n">
<dispatchMethods; separator="\n">
>>
ListenerDispatchMethod(method) ::= <<
<struct.name>.prototype.<if(method.isEnter)>enter<else>exit<endif>Rule = function(listener) {
<if(method.isEnter)>enter<else>exit<endif>Rule(listener) {
if(listener instanceof <parser.grammarName>Listener ) {
listener.<if(method.isEnter)>enter<else>exit<endif><struct.derivedFromName; format="cap">(this);
}
};
}
>>
VisitorDispatchMethod(method) ::= <<
<struct.name>.prototype.accept = function(visitor) {
accept(visitor) {
if ( visitor instanceof <parser.grammarName>Visitor ) {
return visitor.visit<struct.derivedFromName; format="cap">(this);
} else {
return visitor.visitChildren(this);
}
};
}
>>
@ -787,7 +777,7 @@ _prevctx = localctx;
LexerFile(lexerFile, lexer, namedActions) ::= <<
<fileHeader(lexerFile.grammarFileName, lexerFile.ANTLRVersion)>
var antlr4 = require('antlr4/index');
import antlr4 from 'antlr4';
<namedActions.header>
@ -797,32 +787,28 @@ var antlr4 = require('antlr4/index');
Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass) ::= <<
<if(superClass)>
var <superClass> = require('./<superClass>').<superClass>;
import <superClass> from './<superClass>.js';
<endif>
<atn>
var atn = new antlr4.atn.ATNDeserializer().deserialize(serializedATN);
const atn = new antlr4.atn.ATNDeserializer().deserialize(serializedATN);
var decisionsToDFA = atn.decisionToState.map( function(ds, index) { return new antlr4.dfa.DFA(ds, index); });
const decisionsToDFA = atn.decisionToState.map( (ds, index) => new antlr4.dfa.DFA(ds, index) );
export default class <lexer.name> extends <if(superClass)><superClass><else>antlr4.Lexer<endif> {
static grammarFileName = "<lexer.grammarFileName>";
static channelNames = [ "DEFAULT_TOKEN_CHANNEL", "HIDDEN"<if (lexer.channels)>, <lexer.channels:{c| "<c>"}; separator=", ", wrap, anchor><endif> ];
static modeNames = [ <lexer.modes:{m| "<m>"}; separator=", ", wrap, anchor> ];
static literalNames = [ <lexer.literalNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
static symbolicNames = [ <lexer.symbolicNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
static ruleNames = [ <lexer.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor> ];
class <lexer.name> extends <if(superClass)><superClass><else>antlr4.Lexer<endif> {
constructor(input) {
super(input)
this._interp = new antlr4.atn.LexerATNSimulator(this, atn, decisionsToDFA, new antlr4.PredictionContextCache());
<namedActions.members>
this.channelNames = [ "DEFAULT_TOKEN_CHANNEL", "HIDDEN"<if (lexer.channels)>, <lexer.channels:{c| "<c>"}; separator=", ", wrap, anchor><endif> ];
this.modeNames = [ <lexer.modes:{m| "<m>"}; separator=", ", wrap, anchor> ];
this.literalNames = [ <lexer.literalNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
this.symbolicNames = [ <lexer.symbolicNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ];
this.ruleNames = [ <lexer.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor> ];
this.grammarFileName = "<lexer.grammarFileName>";
}
get atn() {
@ -844,14 +830,13 @@ class <lexer.name> extends <if(superClass)><superClass><else>antlr4.Lexer<endif>
<dumpActions(lexer, "", actionFuncs, sempredFuncs)>
module.exports.<lexer.name> = <lexer.name>;
>>
SerializedATN(model) ::= <<
<! only one segment, can be inlined !>
var serializedATN = ["<model.serialized; wrap={",<\n> "}>"].join("");
const serializedATN = ["<model.serialized; wrap={",<\n> "}>"].join("");
>>

7
tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg
View File

@ -520,9 +520,8 @@ if not <chunks>:
>>
ExceptionClause(e, catchArg, catchAction) ::= <<
catch (<catchArg>) {
except <catchArg>:
<catchAction>
}
>>
// lexer actions are not associated with model objects
@ -571,8 +570,8 @@ ThisRulePropertyRef_text(r) ::= "self._input.getText(localctx.start, self._inpu
ThisRulePropertyRef_ctx(r) ::= "localctx"
ThisRulePropertyRef_parser(r) ::= "self"
NonLocalAttrRef(s) ::= "getInvokingContext(<s.ruleIndex>).<s.name>"
SetNonLocalAttr(s, rhsChunks) ::= "getInvokingContext(<s.ruleIndex>).<s.name> = <rhsChunks>"
NonLocalAttrRef(s) ::= "self.getInvokingContext(<s.ruleIndex>).<s.name>"
SetNonLocalAttr(s, rhsChunks) ::= "self.getInvokingContext(<s.ruleIndex>).<s.name> = <rhsChunks>"
AddToLabelList(a) ::= "<ctx(a.label)>.<a.listName>.append(<labelref(a.label)>)"

7
tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg
View File

@ -533,9 +533,8 @@ if not <chunks>:
>>
ExceptionClause(e, catchArg, catchAction) ::= <<
catch (<catchArg>) {
except <catchArg>:
<catchAction>
}
>>
// lexer actions are not associated with model objects
@ -584,8 +583,8 @@ ThisRulePropertyRef_text(r) ::= "self._input.getText(localctx.start, self._inpu
ThisRulePropertyRef_ctx(r) ::= "localctx"
ThisRulePropertyRef_parser(r) ::= "self"
NonLocalAttrRef(s) ::= "getInvokingContext(<s.ruleIndex>).<s.name>"
SetNonLocalAttr(s, rhsChunks) ::= "getInvokingContext(<s.ruleIndex>).<s.name> = <rhsChunks>"
NonLocalAttrRef(s) ::= "self.getInvokingContext(<s.ruleIndex>).<s.name>"
SetNonLocalAttr(s, rhsChunks) ::= "self.getInvokingContext(<s.ruleIndex>).<s.name> = <rhsChunks>"
AddToLabelList(a) ::= "<ctx(a.label)>.<a.listName>.append(<labelref(a.label)>)"

10
tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg
View File

@ -138,13 +138,13 @@ import Antlr4
*
* \<p>The default implementation does nothing.\</p>
*/
<accessLevelOpenOK(file)> func enterEveryRule(_ ctx: ParserRuleContext) { }
<accessLevelOpenOK(file)> func enterEveryRule(_ ctx: ParserRuleContext) throws { }
/**
* {@inheritDoc\}
*
* \<p>The default implementation does nothing.\</p>
*/
<accessLevelOpenOK(file)> func exitEveryRule(_ ctx: ParserRuleContext) { }
<accessLevelOpenOK(file)> func exitEveryRule(_ ctx: ParserRuleContext) throws { }
/**
* {@inheritDoc\}
*
@ -438,10 +438,10 @@ LeftRecursiveRuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,
@discardableResult
private func <currentRule.name>(_ _p<args:{a | , <a>}>: Int) throws -> <currentRule.ctxType> {
let _parentctx: ParserRuleContext? = _ctx
var _parentState: Int = getState()
let _parentState: Int = getState()
var _localctx: <currentRule.ctxType> = <currentRule.ctxType>(_ctx, _parentState<currentRule.args:{a | , <a.name>}>)
var _prevctx: <currentRule.ctxType> = _localctx
var _startState: Int = <currentRule.startState>
var _prevctx: <currentRule.ctxType> = _localctx
let _startState: Int = <currentRule.startState>
try enterRecursionRule(_localctx, <currentRule.startState>, <parser.name>.RULE_<currentRule.name>, _p)
<namedActions.init>
<locals; separator="\n">

8
tool/src/org/antlr/v4/codegen/target/PHPTarget.java
View File

@ -102,4 +102,12 @@ public class PHPTarget extends Target {
protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) {
UnicodeEscapes.appendPythonStyleEscapedCodePoint(codePoint, sb);
}
@Override
public String getTargetStringLiteralFromANTLRStringLiteral(CodeGenerator generator, String literal, boolean addQuotes) {
String targetStringLiteral = super.getTargetStringLiteralFromANTLRStringLiteral(generator, literal, addQuotes);
targetStringLiteral = targetStringLiteral.replace("$", "\\$");
return targetStringLiteral;
}
}